123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239 |
- export var tokenize = initializeDocument
-
- import codes from '../character/codes.mjs'
- import markdownLineEnding from '../character/markdown-line-ending.mjs'
- import constants from '../constant/constants.mjs'
- import types from '../constant/types.mjs'
- import spaceFactory from '../tokenize/factory-space.mjs'
- import blank from '../tokenize/partial-blank-line.mjs'
-
- var containerConstruct = {tokenize: tokenizeContainer}
- var lazyFlowConstruct = {tokenize: tokenizeLazyFlow}
-
- function initializeDocument(effects) {
- var self = this
- var stack = []
- var continued = 0
- var inspectConstruct = {tokenize: tokenizeInspect, partial: true}
- var inspectResult
- var childFlow
- var childToken
-
- return start
-
- function start(code) {
- if (continued < stack.length) {
- self.containerState = stack[continued][1]
- return effects.attempt(
- stack[continued][0].continuation,
- documentContinue,
- documentContinued
- )(code)
- }
-
- return documentContinued(code)
- }
-
- function documentContinue(code) {
- continued++
- return start(code)
- }
-
- function documentContinued(code) {
- // If we’re in a concrete construct (such as when expecting another line of
- // HTML, or we resulted in lazy content), we can immediately start flow.
- if (inspectResult && inspectResult.flowContinue) {
- return flowStart(code)
- }
-
- self.interrupt =
- childFlow &&
- childFlow.currentConstruct &&
- childFlow.currentConstruct.interruptible
- self.containerState = {}
- return effects.attempt(
- containerConstruct,
- containerContinue,
- flowStart
- )(code)
- }
-
- function containerContinue(code) {
- stack.push([self.currentConstruct, self.containerState])
- self.containerState = undefined
- return documentContinued(code)
- }
-
- function flowStart(code) {
- if (code === codes.eof) {
- exitContainers(0, true)
- effects.consume(code)
- return
- }
-
- childFlow = childFlow || self.parser.flow(self.now())
-
- effects.enter(types.chunkFlow, {
- contentType: constants.contentTypeFlow,
- previous: childToken,
- _tokenizer: childFlow
- })
-
- return flowContinue(code)
- }
-
- function flowContinue(code) {
- if (code === codes.eof) {
- continueFlow(effects.exit(types.chunkFlow))
- return flowStart(code)
- }
-
- if (markdownLineEnding(code)) {
- effects.consume(code)
- continueFlow(effects.exit(types.chunkFlow))
- return effects.check(inspectConstruct, documentAfterPeek)
- }
-
- effects.consume(code)
- return flowContinue
- }
-
- function documentAfterPeek(code) {
- exitContainers(
- inspectResult.continued,
- inspectResult && inspectResult.flowEnd
- )
- continued = 0
- return start(code)
- }
-
- function continueFlow(token) {
- if (childToken) childToken.next = token
- childToken = token
- childFlow.lazy = inspectResult && inspectResult.lazy
- childFlow.defineSkip(token.start)
- childFlow.write(self.sliceStream(token))
- }
-
- function exitContainers(size, end) {
- var index = stack.length
-
- // Close the flow.
- if (childFlow && end) {
- childFlow.write([codes.eof])
- childToken = childFlow = undefined
- }
-
- // Exit open containers.
- while (index-- > size) {
- self.containerState = stack[index][1]
- stack[index][0].exit.call(self, effects)
- }
-
- stack.length = size
- }
-
- function tokenizeInspect(effects, ok) {
- var subcontinued = 0
-
- inspectResult = {}
-
- return inspectStart
-
- function inspectStart(code) {
- if (subcontinued < stack.length) {
- self.containerState = stack[subcontinued][1]
- return effects.attempt(
- stack[subcontinued][0].continuation,
- inspectContinue,
- inspectLess
- )(code)
- }
-
- // If we’re continued but in a concrete flow, we can’t have more
- // containers.
- if (childFlow.currentConstruct && childFlow.currentConstruct.concrete) {
- inspectResult.flowContinue = true
- return inspectDone(code)
- }
-
- self.interrupt =
- childFlow.currentConstruct && childFlow.currentConstruct.interruptible
- self.containerState = {}
- return effects.attempt(
- containerConstruct,
- inspectFlowEnd,
- inspectDone
- )(code)
- }
-
- function inspectContinue(code) {
- subcontinued++
- return self.containerState._closeFlow
- ? inspectFlowEnd(code)
- : inspectStart(code)
- }
-
- function inspectLess(code) {
- if (childFlow.currentConstruct && childFlow.currentConstruct.lazy) {
- // Maybe another container?
- self.containerState = {}
- return effects.attempt(
- containerConstruct,
- inspectFlowEnd,
- // Maybe flow, or a blank line?
- effects.attempt(
- lazyFlowConstruct,
- inspectFlowEnd,
- effects.check(blank, inspectFlowEnd, inspectLazy)
- )
- )(code)
- }
-
- // Otherwise we’re interrupting.
- return inspectFlowEnd(code)
- }
-
- function inspectLazy(code) {
- // Act as if all containers are continued.
- subcontinued = stack.length
- inspectResult.lazy = true
- inspectResult.flowContinue = true
- return inspectDone(code)
- }
-
- // We’re done with flow if we have more containers, or an interruption.
- function inspectFlowEnd(code) {
- inspectResult.flowEnd = true
- return inspectDone(code)
- }
-
- function inspectDone(code) {
- inspectResult.continued = subcontinued
- self.interrupt = self.containerState = undefined
- return ok(code)
- }
- }
- }
-
- function tokenizeContainer(effects, ok, nok) {
- return spaceFactory(
- effects,
- effects.attempt(this.parser.constructs.document, ok, nok),
- types.linePrefix,
- this.parser.constructs.disable.null.indexOf('codeIndented') > -1
- ? undefined
- : constants.tabSize
- )
- }
-
- function tokenizeLazyFlow(effects, ok, nok) {
- return spaceFactory(
- effects,
- effects.lazy(this.parser.constructs.flow, ok, nok),
- types.linePrefix,
- this.parser.constructs.disable.null.indexOf('codeIndented') > -1
- ? undefined
- : constants.tabSize
- )
- }
|