export var tokenize = initializeDocument import codes from '../character/codes.mjs' import markdownLineEnding from '../character/markdown-line-ending.mjs' import constants from '../constant/constants.mjs' import types from '../constant/types.mjs' import spaceFactory from '../tokenize/factory-space.mjs' import blank from '../tokenize/partial-blank-line.mjs' var containerConstruct = {tokenize: tokenizeContainer} var lazyFlowConstruct = {tokenize: tokenizeLazyFlow} function initializeDocument(effects) { var self = this var stack = [] var continued = 0 var inspectConstruct = {tokenize: tokenizeInspect, partial: true} var inspectResult var childFlow var childToken return start function start(code) { if (continued < stack.length) { self.containerState = stack[continued][1] return effects.attempt( stack[continued][0].continuation, documentContinue, documentContinued )(code) } return documentContinued(code) } function documentContinue(code) { continued++ return start(code) } function documentContinued(code) { // If we’re in a concrete construct (such as when expecting another line of // HTML, or we resulted in lazy content), we can immediately start flow. if (inspectResult && inspectResult.flowContinue) { return flowStart(code) } self.interrupt = childFlow && childFlow.currentConstruct && childFlow.currentConstruct.interruptible self.containerState = {} return effects.attempt( containerConstruct, containerContinue, flowStart )(code) } function containerContinue(code) { stack.push([self.currentConstruct, self.containerState]) self.containerState = undefined return documentContinued(code) } function flowStart(code) { if (code === codes.eof) { exitContainers(0, true) effects.consume(code) return } childFlow = childFlow || self.parser.flow(self.now()) effects.enter(types.chunkFlow, { contentType: constants.contentTypeFlow, previous: childToken, _tokenizer: childFlow }) return flowContinue(code) } function flowContinue(code) { if (code === codes.eof) { continueFlow(effects.exit(types.chunkFlow)) return flowStart(code) } if (markdownLineEnding(code)) { effects.consume(code) continueFlow(effects.exit(types.chunkFlow)) return effects.check(inspectConstruct, documentAfterPeek) } effects.consume(code) return flowContinue } function documentAfterPeek(code) { exitContainers( inspectResult.continued, inspectResult && inspectResult.flowEnd ) continued = 0 return start(code) } function continueFlow(token) { if (childToken) childToken.next = token childToken = token childFlow.lazy = inspectResult && inspectResult.lazy childFlow.defineSkip(token.start) childFlow.write(self.sliceStream(token)) } function exitContainers(size, end) { var index = stack.length // Close the flow. if (childFlow && end) { childFlow.write([codes.eof]) childToken = childFlow = undefined } // Exit open containers. while (index-- > size) { self.containerState = stack[index][1] stack[index][0].exit.call(self, effects) } stack.length = size } function tokenizeInspect(effects, ok) { var subcontinued = 0 inspectResult = {} return inspectStart function inspectStart(code) { if (subcontinued < stack.length) { self.containerState = stack[subcontinued][1] return effects.attempt( stack[subcontinued][0].continuation, inspectContinue, inspectLess )(code) } // If we’re continued but in a concrete flow, we can’t have more // containers. if (childFlow.currentConstruct && childFlow.currentConstruct.concrete) { inspectResult.flowContinue = true return inspectDone(code) } self.interrupt = childFlow.currentConstruct && childFlow.currentConstruct.interruptible self.containerState = {} return effects.attempt( containerConstruct, inspectFlowEnd, inspectDone )(code) } function inspectContinue(code) { subcontinued++ return self.containerState._closeFlow ? inspectFlowEnd(code) : inspectStart(code) } function inspectLess(code) { if (childFlow.currentConstruct && childFlow.currentConstruct.lazy) { // Maybe another container? self.containerState = {} return effects.attempt( containerConstruct, inspectFlowEnd, // Maybe flow, or a blank line? effects.attempt( lazyFlowConstruct, inspectFlowEnd, effects.check(blank, inspectFlowEnd, inspectLazy) ) )(code) } // Otherwise we’re interrupting. return inspectFlowEnd(code) } function inspectLazy(code) { // Act as if all containers are continued. subcontinued = stack.length inspectResult.lazy = true inspectResult.flowContinue = true return inspectDone(code) } // We’re done with flow if we have more containers, or an interruption. function inspectFlowEnd(code) { inspectResult.flowEnd = true return inspectDone(code) } function inspectDone(code) { inspectResult.continued = subcontinued self.interrupt = self.containerState = undefined return ok(code) } } } function tokenizeContainer(effects, ok, nok) { return spaceFactory( effects, effects.attempt(this.parser.constructs.document, ok, nok), types.linePrefix, this.parser.constructs.disable.null.indexOf('codeIndented') > -1 ? undefined : constants.tabSize ) } function tokenizeLazyFlow(effects, ok, nok) { return spaceFactory( effects, effects.lazy(this.parser.constructs.flow, ok, nok), types.linePrefix, this.parser.constructs.disable.null.indexOf('codeIndented') > -1 ? undefined : constants.tabSize ) }