diff --git a/.codecov.yml b/.codecov.yml index 1d192e6..639dd60 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -90,6 +90,6 @@ ignore: profiling: critical_files_paths: - - src/constructs/eof.ts - src/constructs/initialize.ts - src/lexer.ts + - src/preprocess.ts diff --git a/.dictionary.txt b/.dictionary.txt index 21016f5..0719d72 100644 --- a/.dictionary.txt +++ b/.dictionary.txt @@ -13,6 +13,7 @@ fbca ggshield gpgsign hmarr +iife jchen kaisugi lcov @@ -33,3 +34,4 @@ vates vfile vitest yarnrc +zwnj diff --git a/.dprint.jsonc b/.dprint.jsonc index 585cdf6..5242e67 100644 --- a/.dprint.jsonc +++ b/.dprint.jsonc @@ -24,7 +24,8 @@ "CHANGELOG.md", "LICENSE.md", "RELEASE_NOTES.md", - "yarn.lock" + "yarn.lock", + "__fixtures__/markdown/*.md" ], "exec": { "commands": [ diff --git a/.markdownlintignore b/.markdownlintignore index 4a4c8c7..3a22d74 100644 --- a/.markdownlintignore +++ b/.markdownlintignore @@ -5,3 +5,4 @@ **/CHANGELOG.md **/LICENSE.md **/RELEASE_NOTES.md +__fixtures__/markdown/*.md diff --git a/__fixtures__/hello.txt b/__fixtures__/hello.txt deleted file mode 100644 index 33c08f5..0000000 --- a/__fixtures__/hello.txt +++ /dev/null @@ -1,2 +0,0 @@ -const 你好 = "hello 👋"; -console.log(\u4f60\u597d); // hello 👋 diff --git a/__fixtures__/inline-tag.txt b/__fixtures__/inline-tag.txt deleted file mode 100644 index e7c646a..0000000 --- a/__fixtures__/inline-tag.txt +++ /dev/null @@ -1 +0,0 @@ -{@linkcode Code} diff --git a/__fixtures__/markdown/code-fenced.md b/__fixtures__/markdown/code-fenced.md new file mode 100644 index 0000000..f844db6 --- /dev/null +++ b/__fixtures__/markdown/code-fenced.md @@ -0,0 +1,31 @@ +``` +fenced code +``` + +```js +fenced code with a language +``` + +```js line=1 +fenced code with meta +``` + +~~~ +fenced code with tildes +~~~ + +```not fenced code``` + +~~~fenced code~~~ +asd +~~~ + +``` + +asd +``` + +``` +asd + +``` diff --git a/__fixtures__/markdown/code-indented.md b/__fixtures__/markdown/code-indented.md new file mode 100644 index 0000000..8dffc03 --- /dev/null +++ b/__fixtures__/markdown/code-indented.md @@ -0,0 +1,18 @@ + indented code + + + more indented code +Not indented code + + more + indent + +Not code. + + tabs + and mixed with spaces + extra spaces + +Not code. + + a tab diff --git a/__fixtures__/markdown/code-text.md b/__fixtures__/markdown/code-text.md new file mode 100644 index 0000000..b6fd13a --- /dev/null +++ b/__fixtures__/markdown/code-text.md @@ -0,0 +1,8 @@ +A couple of code examples: `a`, ` b`, `c `, ` d `, ` e +`, ` f +`, `g +`, ` +h`. + +And: `alpha bravo charlie + delta echo`. diff --git a/__fixtures__/markdown/empty.md b/__fixtures__/markdown/empty.md new file mode 100644 index 0000000..e69de29 diff --git a/__fixtures__/markdown/html-flow.md b/__fixtures__/markdown/html-flow.md new file mode 100644 index 0000000..d33624e --- /dev/null +++ b/__fixtures__/markdown/html-flow.md @@ -0,0 +1,16 @@ + + + + + + + + + + +
()
})
- it('should match [currentConstruct?: Construct | null | undefined]', () => {
+ it('should match [currentConstruct?: Construct | undefined]', () => {
expectTypeOf()
.toHaveProperty('currentConstruct')
- .toEqualTypeOf>()
+ .toEqualTypeOf>()
})
- it('should match [disabled: readonly string[]]', () => {
+ it('should match [defineSkip: DefineSkip]', () => {
expectTypeOf()
- .toHaveProperty('disabled')
- .toEqualTypeOf()
+ .toHaveProperty('defineSkip')
+ .toEqualTypeOf()
})
it('should match [events: Event[]]', () => {
@@ -43,59 +39,37 @@ describe('unit-d:interfaces/TokenizeContext', () => {
.toEqualTypeOf()
})
- it('should match [includes: CodeReader["includes"]]', () => {
- expectTypeOf()
- .toHaveProperty('includes')
- .toEqualTypeOf()
- })
-
- it('should match [interrupt?: boolean | null | undefined]', () => {
+ it('should match [interrupt?: boolean | undefined]', () => {
expectTypeOf()
.toHaveProperty('interrupt')
- .toEqualTypeOf>()
+ .toEqualTypeOf>()
})
it('should match [next: Code]', () => {
expectTypeOf().toHaveProperty('next').toEqualTypeOf()
})
- it('should match [now: CodeReader["now"]]', () => {
- expectTypeOf()
- .toHaveProperty('now')
- .toEqualTypeOf()
- })
-
- it('should match [peek: CodeReader["peek"]]', () => {
- expectTypeOf()
- .toHaveProperty('peek')
- .toEqualTypeOf()
+ it('should match [now: Now]', () => {
+ expectTypeOf().toHaveProperty('now').toEqualTypeOf()
})
it('should match [previous: Code]', () => {
expectTypeOf().toHaveProperty('previous').toEqualTypeOf()
})
- it('should match [serialize: CodeReader["serialize"]]', () => {
- expectTypeOf()
- .toHaveProperty('serialize')
- .toEqualTypeOf()
- })
-
- it('should match [slice: CodeReader["slice"]]', () => {
+ it('should match [sliceSerialize: SliceSerialize]', () => {
expectTypeOf()
- .toHaveProperty('slice')
- .toEqualTypeOf()
+ .toHaveProperty('sliceSerialize')
+ .toEqualTypeOf()
})
- it('should match [sliceSerialize: CodeReader["sliceSerialize"]]', () => {
+ it('should match [sliceStream: SliceStream]', () => {
expectTypeOf()
- .toHaveProperty('sliceSerialize')
- .toEqualTypeOf()
+ .toHaveProperty('sliceStream')
+ .toEqualTypeOf()
})
- it('should match [token: Readonly]', () => {
- expectTypeOf()
- .toHaveProperty('token')
- .toEqualTypeOf>()
+ it('should match [write: Write]', () => {
+ expectTypeOf().toHaveProperty('write').toEqualTypeOf()
})
})
diff --git a/src/interfaces/construct-record.ts b/src/interfaces/construct-record.ts
new file mode 100644
index 0000000..0a95445
--- /dev/null
+++ b/src/interfaces/construct-record.ts
@@ -0,0 +1,27 @@
+/**
+ * @file Type Aliases - ConstructRecord
+ * @module vfile-lexer/interfaces/ConstructRecord
+ */
+
+import type { ConstructPack } from '#src/types'
+
+/**
+ * Several constructs, mapped from their initial codes.
+ */
+interface ConstructRecord {
+ /**
+ * Try tokenizing constructs that start with the specified character code.
+ *
+ * @see {@linkcode ConstructPack}
+ */
+ [code: `${number}` | number]: ConstructPack | null | undefined
+
+ /**
+ * Try tokenizing constructs that start with any character code.
+ *
+ * @see {@linkcode ConstructPack}
+ */
+ null?: ConstructPack | null | undefined
+}
+
+export type { ConstructRecord as default }
diff --git a/src/interfaces/construct.ts b/src/interfaces/construct.ts
index 6d5fdcd..e147385 100644
--- a/src/interfaces/construct.ts
+++ b/src/interfaces/construct.ts
@@ -12,26 +12,26 @@ interface Construct {
/**
* Name of the construct, used to toggle constructs off.
*/
- name?: string | null | undefined
+ name?: string | undefined
/**
* Whether this construct represents a partial construct.
*/
- partial?: boolean | null | undefined
+ partial?: boolean | undefined
/**
* Check if the previous character code can come before this construct.
*
* @see {@linkcode Guard}
*/
- previous?: Guard | null | undefined
+ previous?: Guard | undefined
/**
* Resolve the events parsed by {@linkcode tokenize}.
*
* @see {@linkcode Resolver}
*/
- resolve?: Resolver | null | undefined
+ resolve?: Resolver | undefined
/**
* Resolve all events when the content is complete, from the start to the end.
@@ -39,7 +39,7 @@ interface Construct {
*
* @see {@linkcode Resolver}
*/
- resolveAll?: Resolver | null | undefined
+ resolveAll?: Resolver | undefined
/**
* Resolve the events from the start of the content (which may include other
@@ -47,14 +47,14 @@ interface Construct {
*
* @see {@linkcode Resolver}
*/
- resolveTo?: Resolver | null | undefined
+ resolveTo?: Resolver | undefined
/**
* Check if the current character code can start this construct.
*
* @see {@linkcode Guard}
*/
- test?: Guard | null | undefined
+ test?: Guard | undefined
/**
* Set up a state machine to handle character codes streaming in.
diff --git a/src/types/effects.ts b/src/interfaces/effects.ts
similarity index 77%
rename from src/types/effects.ts
rename to src/interfaces/effects.ts
index d78baa7..cc6854f 100644
--- a/src/types/effects.ts
+++ b/src/interfaces/effects.ts
@@ -1,17 +1,14 @@
/**
- * @file Type Aliases - Effects
- * @module vfile-lexer/types/Effects
+ * @file Interfaces - Effects
+ * @module vfile-lexer/interfaces/Effects
*/
-import type Attempt from './attempt'
-import type Consume from './consume'
-import type Enter from './enter'
-import type Exit from './exit'
+import type { Attempt, Consume, Enter, Exit } from '#src/types'
/**
* Context object to transition the state machine.
*/
-type Effects = {
+interface Effects {
/**
* Try to tokenize a construct.
*
diff --git a/src/interfaces/index.ts b/src/interfaces/index.ts
index e30fcf2..e061627 100644
--- a/src/interfaces/index.ts
+++ b/src/interfaces/index.ts
@@ -3,10 +3,17 @@
* @module vfile-lexer/interfaces
*/
-export type { Point, Position } from '@flex-development/vfile-reader'
+export type { Point } from '@flex-development/vfile-location'
export type { default as Construct } from './construct'
export type { default as InitialConstruct } from './construct-initial'
+export type { default as ConstructRecord } from './construct-record'
+export type { default as Effects } from './effects'
export type { default as Options } from './options'
+export type { default as PreprocessOptions } from './options-preprocess'
+export type { default as Place } from './place'
+export type { default as Position } from './position'
export type { default as Token } from './token'
+export type { default as TokenFields } from './token-fields'
+export type { default as TokenInfo } from './token-info'
export type { default as TokenTypeMap } from './token-type-map'
export type { default as TokenizeContext } from './tokenize-context'
diff --git a/src/interfaces/options-preprocess.ts b/src/interfaces/options-preprocess.ts
new file mode 100644
index 0000000..7627810
--- /dev/null
+++ b/src/interfaces/options-preprocess.ts
@@ -0,0 +1,18 @@
+/**
+ * @file Interfaces - PreprocessOptions
+ * @module vfile-lexer/interfaces/PreprocessOptions
+ */
+
+/**
+ * Preprocessor configuration options.
+ */
+interface PreprocessOptions {
+ /**
+ * Number of spaces a tab is equivalent to.
+ *
+ * @default 2
+ */
+ tabSize?: number | undefined
+}
+
+export type { PreprocessOptions as default }
diff --git a/src/interfaces/options.ts b/src/interfaces/options.ts
index 2fa3003..2cb555d 100644
--- a/src/interfaces/options.ts
+++ b/src/interfaces/options.ts
@@ -3,8 +3,16 @@
* @module vfile-lexer/interfaces/Options
*/
-import type { Constructs, FinalizeContext, TokenFactory } from '#src/types'
-import type { Point } from '@flex-development/vfile-reader'
+import type {
+ CodeCheck,
+ Constructs,
+ FinalizeContext,
+ Preprocessor,
+ Resolver,
+ TokenFactory
+} from '#src/types'
+import type { u } from '@flex-development/unist-util-builder'
+import type { Point } from '@flex-development/vfile-location'
import type InitialConstruct from './construct-initial'
/**
@@ -18,13 +26,6 @@ interface Options {
*/
constructs?: Constructs | null | undefined
- /**
- * Finalize the tokenization context.
- *
- * @see {@linkcode FinalizeContext}
- */
- context?: FinalizeContext | null | undefined
-
/**
* Debug logger name.
*
@@ -37,6 +38,20 @@ interface Options {
*/
disabled?: readonly string[] | null | undefined
+ /**
+ * Line ending code check.
+ *
+ * @see {@linkcode CodeCheck}
+ */
+ eol?: CodeCheck | null | undefined
+
+ /**
+ * Finalize the tokenization context.
+ *
+ * @see {@linkcode FinalizeContext}
+ */
+ finalizeContext?: FinalizeContext | null | undefined
+
/**
* Point before first character in file.
*
@@ -47,18 +62,35 @@ interface Options {
from?: Point | null | undefined
/**
- * Initialization construct.
+ * Initial construct.
*
* @see {@linkcode InitialConstruct}
*/
initialize?: InitialConstruct | null | undefined
+ /**
+ * Turn a value into character code chunks.
+ *
+ * @see {@linkcode Preprocessor}
+ */
+ preprocess?: Preprocessor | null | undefined
+
+ /**
+ * End of stream resolvers.
+ *
+ * @see {@linkcode Resolver}
+ */
+ resolvers?: readonly Resolver[] | null | undefined
+
/**
* Create a new token.
*
* @see {@linkcode TokenFactory}
+ * @see {@linkcode u}
+ *
+ * @default u
*/
- token: TokenFactory
+ token?: TokenFactory | null | undefined
}
export type { Options as default }
diff --git a/src/interfaces/place.ts b/src/interfaces/place.ts
new file mode 100644
index 0000000..65271e3
--- /dev/null
+++ b/src/interfaces/place.ts
@@ -0,0 +1,22 @@
+/**
+ * @file Interfaces - Place
+ * @module vfile-lexer/interfaces/Place
+ */
+
+import type { Point } from '@flex-development/vfile-location'
+
+/**
+ * One place in a file, with additional chunk metadata.
+ *
+ * @see {@linkcode Point}
+ *
+ * @extends {Point}
+ */
+interface Place extends Point {
+ /**
+ * Index of character code chunk.
+ */
+ _index: number
+}
+
+export type { Place as default }
diff --git a/src/interfaces/position.ts b/src/interfaces/position.ts
new file mode 100644
index 0000000..b95aba2
--- /dev/null
+++ b/src/interfaces/position.ts
@@ -0,0 +1,27 @@
+/**
+ * @file Type Aliases - Position
+ * @module vfile-lexer/types/Position
+ */
+
+import type Place from './place'
+
+/**
+ * Range between two points in a source file.
+ */
+interface Position {
+ /**
+ * Place of last character code in range.
+ *
+ * @see {@linkcode Place}
+ */
+ end: Place
+
+ /**
+ * Place of first character code in range.
+ *
+ * @see {@linkcode Place}
+ */
+ start: Place
+}
+
+export type { Position as default }
diff --git a/src/interfaces/token-fields.ts b/src/interfaces/token-fields.ts
new file mode 100644
index 0000000..046207a
--- /dev/null
+++ b/src/interfaces/token-fields.ts
@@ -0,0 +1,20 @@
+/**
+ * @file Interfaces - TokenFields
+ * @module vfile-lexer/interfaces/TokenFields
+ */
+
+/**
+ * Token fields registry.
+ *
+ * This interface can be augmented to register custom token fields.
+ *
+ * @example
+ * declare module '@flex-development/vfile-lexer' {
+ * interface TokenFields {
+ * value?: string | null
+ * }
+ * }
+ */
+interface TokenFields {}
+
+export type { TokenFields as default }
diff --git a/src/interfaces/token-info.ts b/src/interfaces/token-info.ts
new file mode 100644
index 0000000..fb4287b
--- /dev/null
+++ b/src/interfaces/token-info.ts
@@ -0,0 +1,35 @@
+/**
+ * @file Interfaces - TokenInfo
+ * @module vfile-lexer/interfaces/TokenInfo
+ */
+
+import type Position from './position'
+import type Token from './token'
+import type TokenFields from './token-fields'
+
+/**
+ * Token data.
+ *
+ * @see {@linkcode Position}
+ * @see {@linkcode TokenFields}
+ *
+ * @extends {Position}
+ * @extends {TokenFields}
+ */
+interface TokenInfo extends Position, TokenFields {
+ /**
+ * Next token.
+ *
+ * @see {@linkcode Token}
+ */
+ next?: Token | undefined
+
+ /**
+ * Previous token.
+ *
+ * @see {@linkcode Token}
+ */
+ previous?: Token | undefined
+}
+
+export type { TokenInfo as default }
diff --git a/src/interfaces/token-type-map.ts b/src/interfaces/token-type-map.ts
index cc74152..8063df1 100644
--- a/src/interfaces/token-type-map.ts
+++ b/src/interfaces/token-type-map.ts
@@ -3,8 +3,6 @@
* @module vfile-lexer/interfaces/TokenTypeMap
*/
-import type { tt } from '#src/enums'
-
/**
* Token type registry.
*
@@ -13,13 +11,10 @@ import type { tt } from '#src/enums'
* @example
* declare module '@flex-development/vfile-lexer' {
* interface TokenTypeMap {
- * type: TokenType
+ * whitespace: tt.whitespace
* }
* }
*/
-interface TokenTypeMap {
- eof: tt.eof
- sof: tt.sof
-}
+interface TokenTypeMap {}
export type { TokenTypeMap as default }
diff --git a/src/interfaces/token.ts b/src/interfaces/token.ts
index 2d52fcd..1873daa 100644
--- a/src/interfaces/token.ts
+++ b/src/interfaces/token.ts
@@ -3,8 +3,8 @@
* @module vfile-lexer/interfaces/Token
*/
-import type { TokenType } from '#src/types'
-import type { Code, Position } from '@flex-development/vfile-reader'
+import type { Code, TokenType } from '#src/types'
+import type TokenInfo from './token-info'
/**
* A span of one (`1`) or more character codes.
@@ -26,24 +26,14 @@ import type { Code, Position } from '@flex-development/vfile-reader'
* }
*
* @see {@linkcode Code}
- * @see {@linkcode Position}
+ * @see {@linkcode TokenInfo}
* @see {@linkcode TokenType}
*
* @template {TokenType} [T=TokenType] - Token type
*
- * @extends {Position}
+ * @extends {TokenInfo}
*/
-interface Token extends Position {
- /**
- * Next token in linked token list.
- */
- next?: Token | undefined
-
- /**
- * Previous token in linked token list.
- */
- previous?: Token | undefined
-
+interface Token extends TokenInfo {
/**
* Token type.
*/
diff --git a/src/interfaces/tokenize-context.ts b/src/interfaces/tokenize-context.ts
index 1ccd45c..2fdf93b 100644
--- a/src/interfaces/tokenize-context.ts
+++ b/src/interfaces/tokenize-context.ts
@@ -3,26 +3,21 @@
* @module vfile-lexer/interfaces/TokenizeContext
*/
-import type { Event } from '#src/types'
import type {
Code,
- CodeCheckFactory,
- CodeReader
-} from '@flex-development/vfile-reader'
+ DefineSkip,
+ Event,
+ Now,
+ SliceSerialize,
+ SliceStream,
+ Write
+} from '#src/types'
import type Construct from './construct'
-import type Token from './token'
/**
* Context object to assist with tokenization.
*/
interface TokenizeContext {
- /**
- * Create a code check from a regular expression.
- *
- * @see {@linkcode CodeCheckFactory}
- */
- check: CodeCheckFactory
-
/**
* Get the current character code.
*
@@ -39,32 +34,26 @@ interface TokenizeContext {
*
* @see {@linkcode Construct}
*/
- currentConstruct?: Construct | null | undefined
+ currentConstruct?: Construct | undefined
/**
- * Disabled construct names.
+ * Define a skip.
+ *
+ * @see {@linkcode DefineSkip}
*/
- disabled: readonly string[]
+ defineSkip: DefineSkip
/**
- * Current list of events.
+ * List of events.
*
* @see {@linkcode Event}
*/
events: Event[]
/**
- * Check if the file contains the given search value, relative to the current
- * reader position.
- *
- * @see {@linkcode CodeReader.includes}
+ * Boolean indicating a construct is interrupting another construct.
*/
- includes: CodeReader['includes']
-
- /**
- * Boolean indicating the a construct is interrupting another construct.
- */
- interrupt?: boolean | null | undefined
+ interrupt?: boolean | undefined
/**
* Get the next character code.
@@ -78,20 +67,13 @@ interface TokenizeContext {
/**
* Get the current point in the file.
*
- * @see {@linkcode CodeReader.now}
- */
- now: CodeReader['now']
-
- /**
- * Get the next `k`-th code point from the file without changing the position
- * of the reader, with `null` denoting end of file.
- *
- * @see {@linkcode CodeReader.peek}
+ * @see {@linkcode Now}
*/
- peek: CodeReader['peek']
+ now: Now
/**
- * Get the previous character code.
+ * Get the previous character code without changing the position of the
+ * reader.
*
* @see {@linkcode Code}
*
@@ -100,34 +82,28 @@ interface TokenizeContext {
get previous(): Code
/**
- * Convert the specified sequence of character codes to a string.
+ * Get the text spanning the specified range without changing the position of
+ * the reader.
*
- * @see {@linkcode CodeReader.serialize}
+ * @see {@linkcode SliceSerialize}
*/
- serialize: CodeReader['serialize']
+ sliceSerialize: SliceSerialize
/**
- * Get the character codes spanning the specified range without changing the
- * position of the reader.
+ * Get the chunks spanning the specified range.
*
- * @see {@linkcode CodeReader.slice}
+ * @see {@linkcode SliceStream}
*/
- slice: CodeReader['slice']
+ sliceStream: SliceStream
/**
- * Get the text spanning the specified range without changing the position of
- * the reader.
+ * Write a slice of chunks.
*
- * @see {@linkcode CodeReader.sliceSerialize}
- */
- sliceSerialize: CodeReader['sliceSerialize']
-
- /**
- * Current tail token.
+ * The eof code (`null`) can be used to signal the end of the stream.
*
- * @see {@linkcode Token}
+ * @see {@linkcode Write}
*/
- get token(): Readonly
+ write: Write
}
export type { TokenizeContext as default }
diff --git a/src/lexer.ts b/src/lexer.ts
index e2b4d00..e3acafa 100644
--- a/src/lexer.ts
+++ b/src/lexer.ts
@@ -3,65 +3,90 @@
* @module vfile-lexer/lexer
*/
-import {
- CodeReader as Reader,
- codes,
- type Code,
- type Offset,
- type Point
-} from '@flex-development/vfile-reader'
+import { u } from '@flex-development/unist-util-builder'
+import { Location } from '@flex-development/vfile-location'
import debug from 'debug'
import { ok as assert } from 'devlop'
-import { splice } from 'micromark-util-chunked'
-import type { VFile, Value } from 'vfile'
+import { push, splice } from 'micromark-util-chunked'
import { initialize } from './constructs'
-import { ev } from './enums'
+import { chars, codes, ev } from './enums'
import type {
Construct,
+ ConstructRecord,
+ Effects,
InitialConstruct,
Options,
+ Place,
+ Point,
+ Position,
Token,
+ TokenFields,
+ TokenInfo,
TokenizeContext
} from './interfaces'
+import preprocess from './preprocess'
import type {
Attempt,
- ConstructRecord,
+ Chunk,
+ Code,
+ CodeCheck,
+ Column,
+ ConstructPack,
Constructs,
- Effects,
+ DefineSkip,
Event,
+ Line,
+ Offset,
+ Preprocessor,
+ Resolver,
ReturnHandle,
State,
TokenFactory,
- TokenFields,
TokenType
} from './types'
-import { resolveAll } from './utils'
+import { isLineEnding, resolveAll } from './utils'
/**
* Source file tokenizer.
*
+ * @see {@linkcode Location}
+ * @see {@linkcode TokenizeContext}
+ *
* @class
+ * @extends {Location}
+ * @implements {TokenizeContext}
*/
-class Lexer {
+class Lexer extends Location implements TokenizeContext {
/**
- * Expected character code, used for tracking bugs.
+ * Character code chunks.
*
* @see {@linkcode Code}
*
* @protected
* @instance
+ * @member {Code[]} chunks
+ */
+ protected chunks: Code[]
+
+ /**
+ * Expected character code, used for tracking bugs.
+ *
+ * @see {@linkcode Code}
+ *
+ * @private
+ * @instance
* @member {Code} code
*/
- protected code: Code
+ #code: Code
/**
* Character code consumption state, used for tracking bugs.
*
- * @protected
+ * @private
* @instance
* @member {boolean | null} consumed
*/
- protected consumed: boolean | null
+ #consumed: boolean | null
/**
* Tokenize context.
@@ -86,13 +111,13 @@ class Lexer {
protected debug: debug.Debugger
/**
- * Disabled construct names.
+ * List of disabled constructs.
*
- * @public
+ * @protected
* @instance
* @member {ReadonlyArray} disabled
*/
- public disabled: readonly string[]
+ protected disabled: readonly string[]
/**
* Context object to transition the state machine.
@@ -106,13 +131,15 @@ class Lexer {
protected effects: Effects
/**
- * Boolean indicating end of file has been reached.
+ * Line ending code check.
+ *
+ * @see {@linkcode CodeCheck}
*
* @protected
* @instance
- * @member {boolean} eof
+ * @member {CodeCheck} eol
*/
- protected eof: boolean
+ protected eol: CodeCheck
/**
* List of events.
@@ -126,18 +153,7 @@ class Lexer {
public events: Event[]
/**
- * Head token.
- *
- * @see {@linkcode Token}
- *
- * @public
- * @instance
- * @member {Token} head
- */
- public head!: Token
-
- /**
- * Initialization construct.
+ * Initial construct.
*
* @see {@linkcode InitialConstruct}
*
@@ -154,51 +170,62 @@ class Lexer {
*
* @protected
* @instance
- * @member {Construct | null | undefined} lastConstruct
+ * @member {Construct | undefined} lastConstruct
*/
- protected lastConstruct: Construct | null | undefined
+ protected lastConstruct: Construct | undefined
/**
* Last {@linkcode events} length.
*
- * @see {@linkcode Offset}
- *
* @protected
* @instance
- * @member {Offset} lastEvent
+ * @member {number} lastEvent
*/
- protected lastEvent: Offset
+ protected lastEvent: number
/**
- * Last reader index.
+ * Last place.
*
- * @see {@linkcode Offset}
+ * @see {@linkcode Place}
*
* @protected
* @instance
- * @member {Offset} lastIndex
+ * @member {Place} lastPlace
*/
- protected lastIndex: Offset
+ protected lastPlace: Place
/**
- * Last tail token.
+ * Last token stack.
+ *
+ * @see {@linkcode Token}
*
* @protected
* @instance
- * @member {Token | null} lastToken
+ * @member {Token[]} lastStack
*/
- protected lastToken: Token | null
+ protected lastStack: Token[]
/**
- * Source file reader.
+ * Current point in file.
*
- * @see {@linkcode Reader}
+ * @see {@linkcode Place}
+ *
+ * @public
+ * @instance
+ * @member {Place} place
+ */
+ declare public place: Place
+
+ /**
+ * Turn a value into character code chunks.
+ *
+ * @see {@linkcode Preprocessor}
*
* @protected
* @instance
- * @member {Reader} reader
+ * @member {Preprocessor} preprocess
*/
- protected reader: Reader
+ protected preprocess: Preprocessor
/**
* Constructs with `resolveAll` handlers.
@@ -212,26 +239,50 @@ class Lexer {
protected resolveAll: Construct[]
/**
- * Current state.
+ * End of stream resolvers.
*
- * @see {@linkcode State}
+ * @see {@linkcode Resolver}
*
* @protected
* @instance
- * @member {State | undefined} state
+ * @member {Resolver[]} resolvers
*/
- protected state: State | undefined
+ protected resolvers: Resolver[]
+
+ /**
+ * Map, where each key is a line number and each value a column to be skipped
+ * to when the internal reader is on that line.
+ *
+ * @see {@linkcode Column}
+ * @see {@linkcode Line}
+ *
+ * @protected
+ * @instance
+ * @member {Record} skips
+ */
+ protected skips: Record
/**
- * Tail token.
+ * Token stack.
*
* @see {@linkcode Token}
*
- * @public
+ * @protected
+ * @instance
+ * @member {Token[]} stack
+ */
+ protected stack: Token[]
+
+ /**
+ * Current state.
+ *
+ * @see {@linkcode State}
+ *
+ * @protected
* @instance
- * @member {Token} tail
+ * @member {State | undefined} state
*/
- public tail!: Token
+ protected state: State | undefined
/**
* Token factory.
@@ -248,30 +299,53 @@ class Lexer {
* Create a new file tokenizer.
*
* @see {@linkcode Options}
- * @see {@linkcode VFile}
- * @see {@linkcode Value}
*
- * @param {Value | VFile | null | undefined} file - File to tokenize
- * @param {Options} options - Tokenization options
+ * @param {Options | null | undefined} [options] - Tokenize options
*/
- constructor(file: Value | VFile | null | undefined, options: Options) {
- assert(options.token, 'expected token factory')
-
- this.debug = debug(options.debug ?? 'vfile-lexer')
- this.disabled = Object.freeze(options.disabled ?? [])
- this.reader = new Reader(file, options.from)
- this.token = options.token
+ constructor(options?: Options | null | undefined) {
+ super(null, options?.from)
+ this.place._index = 0
+
+ const {
+ constructs,
+ debug: debugName,
+ disabled,
+ eol,
+ finalizeContext,
+ initialize: initializer,
+ preprocess: preprocessor,
+ resolvers,
+ token
+ } = options ?? {}
+
+ this.debug = debug(debugName ?? 'vfile-lexer')
+ this.disabled = Object.freeze(disabled ?? [])
+ this.initialize = initializer ?? initialize(constructs ?? {})
+ this.preprocess = preprocessor ?? preprocess()
+ this.resolvers = resolvers ? [...resolvers] : []
+ this.token = token ?? function token(
+ type: TokenType,
+ info: TokenInfo
+ ): Token {
+ return Object.defineProperties(u(type, info), {
+ next: { enumerable: false, writable: true },
+ previous: { enumerable: false, writable: true }
+ })
+ }
- this.code = this.reader.read()
- this.consumed = true
- this.eof = false
+ assert(typeof this.token, 'expected token factory')
+ this.#code = codes.eof
+ this.#consumed = true
+ this.chunks = []
+ this.eol = eol ?? isLineEnding
this.events = []
- this.initialize = options.initialize ?? initialize(options.constructs ?? [])
- this.lastConstruct = null
+ this.lastConstruct = undefined
this.lastEvent = 0
- this.lastIndex = 0
- this.lastToken = null
+ this.lastPlace = this.now()
+ this.lastStack = []
this.resolveAll = []
+ this.skips = {}
+ this.stack = []
/**
* Base context object.
@@ -279,33 +353,23 @@ class Lexer {
* @const {TokenizeContext} context
*/
const context: TokenizeContext = Object.defineProperties({
- check: this.reader.check.bind(this.reader),
- code: this.code,
+ code: codes.eof,
currentConstruct: this.lastConstruct,
- disabled: this.disabled,
+ defineSkip: this.defineSkip.bind(this),
events: this.events,
- includes: this.reader.includes.bind(this.reader),
- next: this.reader.peek(),
+ next: codes.eof,
now: this.now.bind(this),
- peek: this.reader.peek.bind(this.reader),
- previous: this.reader.previous,
- serialize: this.reader.serialize.bind(this.reader),
- slice: this.reader.slice.bind(this.reader),
- sliceSerialize: this.reader.sliceSerialize.bind(this.reader),
- token: this.tail
+ previous: codes.eof,
+ sliceSerialize: this.sliceSerialize.bind(this),
+ sliceStream: this.sliceStream.bind(this),
+ write: this.write.bind(this)
}, {
/* c8 ignore next 6 */
- code: { configurable: false, get: (): Code => this.reader.output },
- next: { configurable: false, get: (): Code => this.reader.peek() },
- previous: { configurable: false, get: (): Code => this.reader.previous },
- token: {
- configurable: false,
- get: (): Readonly => Object.freeze(Object.assign({}, this.tail))
- }
+ code: { configurable: false, get: (): Code => this.code },
+ next: { configurable: false, get: (): Code => this.next },
+ previous: { configurable: false, get: (): Code => this.previous }
})
- this.context = options.context?.(context) ?? context
-
this.effects = {
attempt: this.constructFactory(this.resolve.bind(this)),
check: this.constructFactory(this.restore.bind(this)),
@@ -315,26 +379,72 @@ class Lexer {
interrupt: this.constructFactory(this.restore.bind(this), true)
}
- if (this.initialize.resolveAll) this.resolveAll.push(this.initialize)
+ this.context = context
+ this.context = finalizeContext?.call(this, this.context) ?? this.context
+
this.state = this.initialize.tokenize.call(this.context, this.effects)
+ this.initialize.resolveAll && this.resolveAll.push(this.initialize)
}
/**
- * Create a new file tokenizer for `file`.
+ * Get the current character code without changing the position of the reader.
*
- * @see {@linkcode Options}
- * @see {@linkcode VFile}
- * @see {@linkcode Value}
+ * > 👉 Equivalent to `this.peek(0)`.
+ *
+ * @see {@linkcode Code}
+ *
+ * @public
+ * @instance
+ *
+ * @return {Code} Current character code
+ */
+ public get code(): Code {
+ return this.peek(0)
+ }
+
+ /**
+ * Check if end of stream has been reached.
+ *
+ * @public
+ * @instance
+ *
+ * @return {boolean} `true` if at end of stream, `false` otherwise
+ */
+ public get eos(): boolean {
+ return this.chunks[this.chunks.length - 1] === codes.eof
+ }
+
+ /**
+ * Get the next character code without changing the position of the reader.
+ *
+ * > 👉 Equivalent to `this.peek()`.
+ *
+ * @see {@linkcode Code}
*
* @public
- * @static
+ * @instance
*
- * @param {Value | VFile} file - File to tokenize
- * @param {Options} options - Tokenization options
- * @return {Lexer} New lexer instance
+ * @return {Code} Next character code
*/
- public static create(file: Value | VFile, options: Options): Lexer {
- return new Lexer(file, options)
+ public get next(): Code {
+ return this.peek()
+ }
+
+ /**
+ * Get the previous character code without changing the position of the
+ * reader.
+ *
+ * > 👉 Equivalent to `this.peek(-1)`.
+ *
+ * @see {@linkcode Code}
+ *
+ * @public
+ * @instance
+ *
+ * @return {Code} Previous character code
+ */
+ public get previous(): Code {
+ return this.peek(-1)
}
/**
@@ -348,7 +458,7 @@ class Lexer {
*
* @param {ReturnHandle} onreturn - Successful construct callback
* @param {boolean | null | undefined} [interrupt] - Interrupting?
- * @return {Attempt} attempt/check/interrupt state
+ * @return {Attempt} attempt/check/interrupt
*/
protected constructFactory(
onreturn: ReturnHandle,
@@ -361,7 +471,18 @@ class Lexer {
*/
const self: this = this
- return function hook(
+ return hook
+
+ /**
+ * Handle either an object mapping codes to constructs, a list of
+ * constructs, or a single construct.
+ *
+ * @param {Constructs} construct - Constructs to try
+ * @param {State | undefined} [succ] - Successful tokenization state
+ * @param {State | undefined} [fail] - Failed tokenization state
+ * @return {State} Next state
+ */
+ function hook(
construct: Constructs,
succ: State = /* c8 ignore next */ () => undefined,
fail?: State
@@ -383,22 +504,24 @@ class Lexer {
/**
* Construct list.
*
- * @var {ReadonlyArray} list
+ * @var {Construct[]} list
*/
- let list: readonly Construct[]
+ let list: Construct[]
- // handle a single construct, list of constructs, or map of constructs
- return 'tokenize' in construct || Array.isArray(construct)
- ? handleConstructList([construct].flat())
- : handleConstructMap(construct)
+ // handle list of constructs, single construct, or map of constructs
+ return Array.isArray(construct)
+ ? handleConstructList(construct)
+ : 'tokenize' in construct
+ ? handleConstructList([construct])
+ : handleConstructRecord(construct)
/**
* Handle a list of constructs.
*
- * @param {ReadonlyArray} constructs - Constructs to try
+ * @param {Construct[]} constructs - Constructs to try
* @return {State} Next state
*/
- function handleConstructList(constructs: readonly Construct[]): State {
+ function handleConstructList(constructs: Construct[]): State {
list = constructs
j = 0
@@ -416,30 +539,27 @@ class Lexer {
* @param {ConstructRecord} map - Constructs to try
* @return {State} Next state
*/
- function handleConstructMap(map: ConstructRecord): State {
- return start
+ function handleConstructRecord(map: ConstructRecord): State {
+ return run
/**
* Check if `value` looks like a construct, or list of constructs.
*
* @param {unknown} value - Value to check
- * @return {value is Construct | ReadonlyArray} `true` if
- * value is an object
+ * @return {value is ConstructPack} `true` if value is an object
*/
- function is(value: unknown): value is Construct | readonly Construct[] {
+ function is(value: unknown): value is ConstructPack {
return typeof value === 'object'
}
/**
- * Start construct tokenization.
- *
* @param {Code} code - Current character code
* @return {State | undefined} Next state
*/
- function start(code: Code): State | undefined {
+ function run(code: Code): State | undefined {
return handleConstructList([
...[code !== null && map[code]].flat().filter(value => is(value)),
- ...[code !== null && map.null].flat().filter(value => is(value))
+ ...[map.null].flat().filter(value => is(value))
])(code)
}
}
@@ -454,8 +574,6 @@ class Lexer {
return start
/**
- * Start construct tokenization.
- *
* @param {Code} code - Current character code
* @return {State | undefined} Next state
*/
@@ -463,12 +581,10 @@ class Lexer {
const { context, disabled, effects } = self
const { name, partial, previous, test, tokenize } = construct
+ self.store()
currentConstruct = construct
-
if (!partial) context.currentConstruct = construct
- if (fail) self.store()
-
- context.interrupt = interrupt
+ context.interrupt = interrupt
switch (true) {
case !!name && disabled.includes(name):
@@ -488,10 +604,10 @@ class Lexer {
* @return {State} Next state
*/
function ok(code: Code): State {
- assert(code === self.code, 'expected `code` to equal expected code')
+ assert(code === self.#code, 'expected `code` to equal expected code')
self.debug('ok: `%o`', code)
- self.consumed = true
+ self.#consumed = true
onreturn(currentConstruct)
return succ
@@ -505,10 +621,10 @@ class Lexer {
*/
function nok(code: Code): State | undefined {
assert(list, 'expected construct list')
- assert(code === self.code, 'expected `code` to equal expected code')
+ assert(code === self.#code, 'expected `code` to equal expected code')
self.debug('nok: `%o`', code)
- self.consumed = true
+ self.#consumed = true
self.restore()
return ++j < list.length ? handleConstruct(list[j]!) : fail
@@ -528,13 +644,37 @@ class Lexer {
* @return {undefined} Nothing
*/
protected consume(code: Code): undefined {
- assert(code === this.code, 'expected `code` to equal expected code')
+ assert(code === this.#code, 'expected `code` to equal expected code')
this.debug('consume: `%o`', code)
- assert(this.consumed === null, 'expected unconsumed code')
- code !== codes.eof ? this.reader.read() : (this.eof = true)
- return void (this.consumed = true)
+ assert(this.#consumed === null, 'expected unconsumed code')
+ this.read()
+ this.#consumed = true
+ return void code
+ }
+
+ /* c8 ignore start */
+
+ /**
+ * Define a skip.
+ *
+ * @see {@linkcode DefineSkip}
+ * @see {@linkcode Point}
+ *
+ * @todo test
+ *
+ * @public
+ * @instance
+ *
+ * @param {Pick} point - Skip point
+ * @return {undefined} Nothing
+ */
+ public defineSkip(point: Pick): undefined {
+ this.skips[point.line] = point.column
+ return this.skip(), void this.debug('position: define skip: `%j`', point)
}
+ /* c8 ignore stop */
+
/**
* Start a new token.
*
@@ -546,12 +686,12 @@ class Lexer {
* @instance
*
* @param {TokenType} type - Token type
- * @param {(Partial | null)?} fields - Token fields
+ * @param {TokenFields | null | undefined} [fields] - Token fields
* @return {Token} Open token
*/
protected enter(
type: TokenType,
- fields?: Partial | null
+ fields?: TokenFields | null | undefined
): Token {
/**
* New token.
@@ -560,21 +700,16 @@ class Lexer {
*/
const token: Token = this.token(type, {
...fields,
- end: this.reader.point(-1),
- start: this.now()
+ start: this.now(), // eslint-disable-next-line sort-keys
+ end: this.now()
})
- // shift/replace/init tail
- if ((this.head)) {
- token.previous = this.tail
- this.tail.next = token
- this.tail = this.tail.next
- } else {
- this.head = this.tail = token
- }
-
+ assert(typeof type === 'string', 'expected `type` to be a string')
+ assert(type.length > 0, 'expected `type` to be a non-empty string')
this.debug('enter: `%s`; %o', type, token.start)
+
this.events.push([ev.enter, token, this.context])
+ this.stack.push(token)
return token
}
@@ -594,28 +729,13 @@ class Lexer {
protected exit(type: TokenType): Token {
assert(typeof type === 'string', 'expected `type` to be a string')
assert(type.length > 0, 'expected `type` to be a non-empty string')
- assert(this.events.length, 'expected events')
/**
* Token to close.
*
- * @var {Token | undefined} token
+ * @const {Token | undefined} token
*/
- let token: Token | undefined = this.tail
-
- // find open token
- while (token) {
- if (
- !!token.start.column &&
- !!token.start.line &&
- token.start.offset >= 0 &&
- token.end.column + token.end.line + token.end.offset === -3
- ) {
- break
- }
-
- token = token.previous
- }
+ const token: Token | undefined = this.stack.pop()
assert(token, 'cannot exit without open token')
assert(type === token.type, 'expected exit token to match current token')
@@ -641,10 +761,10 @@ class Lexer {
* @return {undefined} Nothing
*/
protected go(code: Code): undefined {
- assert(this.consumed, `expected code \`${code}\` to be consumed`)
- this.consumed = null
+ assert(this.#consumed, `expected code \`${code}\` to be consumed`)
+ this.#consumed = null
this.debug('go: `%o`, %j', code, /* c8 ignore next */ this.state?.name)
- this.code = code
+ this.#code = code
assert(typeof this.state === 'function', 'expected state function')
this.state = this.state(code)
return void code
@@ -653,15 +773,70 @@ class Lexer {
/**
* Get the current point in the file.
*
- * @see {@linkcode Point}
+ * @see {@linkcode Place}
+ *
+ * @public
+ * @instance
+ *
+ * @return {Place} Current point in file, relative to {@linkcode start}
+ */
+ public now(): Place {
+ const { _index, column, line, offset } = this.place
+ // eslint-disable-next-line sort-keys
+ return { line, column, offset, _index }
+ }
+
+ /**
+ * Get the next `k`-th character code from the file without changing the
+ * position of the reader.
+ *
+ * @see {@linkcode Code}
+ *
+ * @public
+ * @instance
+ *
+ * @param {number?} [k=1] - Difference between index of next `k`-th character
+ * code and index of current character code
+ * @return {Code} Peeked character code
+ */
+ public peek(k: number = 1): Code {
+ return this.chunks[this.place._index + k] ?? codes.eof
+ }
+
+ /**
+ * Get the next character code.
+ *
+ * Unlike {@linkcode peek}, this method changes the position of the reader.
+ *
+ * @see {@linkcode Code}
*
* @protected
* @instance
*
- * @return {Point} Current point in file
+ * @return {Code} Next character code
*/
- protected now(): Point {
- return this.reader.now()
+ protected read(): Code {
+ /**
+ * Current character code.
+ *
+ * @const {Code} code
+ */
+ const code: Code = this.code
+
+ if (this.eol(code)) {
+ this.place.column = 1
+ this.place.line++
+ this.place.offset += code === codes.crlf ? 2 : 1
+ this.skip()
+ this.debug('position after eol: %o', this.place)
+ } else if (code !== codes.vs && code !== codes.eof) {
+ this.place.column++
+ this.place.offset++
+ } else if (code === codes.vs && this.previous === codes.vht) {
+ this.place.column++
+ }
+
+ return this.chunks[++this.place._index] ?? codes.eof
}
/**
@@ -686,8 +861,6 @@ class Lexer {
}
if (construct.resolve) {
- assert(lastEvent >= 0, 'expected last event index')
-
splice(
this.events,
lastEvent,
@@ -702,8 +875,8 @@ class Lexer {
assert(
/* c8 ignore next 3 */ !!construct.partial ||
- !this.context.events.length ||
- this.context.events[this.context.events.length - 1]![0] === ev.exit,
+ !this.events.length ||
+ this.events[this.events.length - 1]![0] === ev.exit,
'expected last token to end'
)
@@ -711,7 +884,8 @@ class Lexer {
}
/**
- * Restore the last construct, event index, reader position, and tail token.
+ * Restore the last construct, event index, location, tail token, and token
+ * stack.
*
* @protected
* @instance
@@ -719,22 +893,117 @@ class Lexer {
* @return {undefined} Nothing
*/
protected restore(): undefined {
- assert(this.lastEvent >= 0, 'expected last event index')
- assert(this.lastIndex >= 0, 'expected last reader position')
- assert(this.lastToken, 'expected last token')
-
- this.reader.read(this.lastIndex - this.reader.index)
this.context.currentConstruct = this.lastConstruct
this.events.length = this.lastEvent
- this.tail = this.lastToken
- this.tail.next = undefined
+ this.place = { ...this.lastPlace }
+ this.stack = [...this.lastStack]
+ return void this.debug('restore: %o', this.now())
+ }
- this.debug('restore: %o', this.now())
- return void this
+ /**
+ * Move the current point a bit forward in the line when on a column skip.
+ *
+ * @todo test
+ *
+ * @protected
+ * @instance
+ *
+ * @return {undefined} Nothing
+ */
+ protected skip(): undefined {
+ /* c8 ignore next 4 */
+ if (this.place.line in this.skips && this.place.column < 2) {
+ this.place.column = this.skips[this.place.line]!
+ this.place.offset += this.place.column - 1
+ }
+
+ return void this.place
+ }
+
+ /**
+ * Get the text spanning `range` without changing the position of the reader.
+ *
+ * @see {@linkcode Position}
+ *
+ * @public
+ * @instance
+ *
+ * @param {Position} range - Position in stream
+ * @param {boolean | null | undefined} [expandTabs] - Expand tabs?
+ * @return {string} Serialized slice
+ */
+ public sliceSerialize(
+ range: Position,
+ expandTabs?: boolean | null | undefined
+ ): string {
+ /**
+ * Character code slice.
+ *
+ * @const {Code[]} slice
+ */
+ const slice: Code[] = this.sliceStream(range)
+
+ /**
+ * Serialized character code array.
+ *
+ * @const {string[]} result
+ */
+ const result: string[] = []
+
+ /**
+ * Current code represents horizontal tab?
+ *
+ * @var {boolean} tab
+ */
+ let tab: boolean = false
+
+ for (const code of slice) {
+ switch (code) {
+ case codes.crlf:
+ result.push(chars.crlf)
+ break
+ case codes.vcr:
+ result.push(chars.cr)
+ break
+ case codes.vht:
+ result.push(expandTabs ? chars.space : chars.ht)
+ break
+ case codes.vlf:
+ result.push(chars.lf)
+ break
+ case codes.vs:
+ if (!expandTabs && tab) continue
+ result.push(chars.space)
+ break
+ default:
+ result.push(String.fromCodePoint(code!))
+ }
+
+ tab = code === codes.vht
+ }
+
+ return result.join(chars.empty)
+ }
+
+ /**
+ * Get the chunks spanning `range`.
+ *
+ * @see {@linkcode Code}
+ * @see {@linkcode Position}
+ *
+ * @public
+ * @instance
+ *
+ * @param {Position} range - Position in stream
+ * @return {Code[]} List of chunks
+ */
+ public sliceStream(range: Position): Code[] {
+ return this.chunks.slice(range.start._index, range.end._index)
}
/**
- * Store the current construct, event index reader position, and tail token.
+ * Store the current construct, event index, location, tail token, and token
+ * stack.
*
* @protected
* @instance
@@ -744,27 +1013,74 @@ class Lexer {
protected store(): undefined {
this.lastConstruct = this.context.currentConstruct
this.lastEvent = this.events.length
- this.lastIndex = this.reader.index
- this.lastToken = this.tail
-
+ this.lastPlace = this.now()
+ this.lastStack = [...this.stack]
return void this
}
/**
- * Tokenize the file.
+ * Main loop to walk through {@linkcode chunks}.
*
- * @public
+ * > 👉 The {@linkcode read} method modifies `_index` in {@linkcode place} to
+ * > advance the loop until end of stream.
+ *
+ * @protected
* @instance
*
* @return {this} `this` lexer
*/
- public tokenize(): this {
- while (!this.eof) this.go(this.reader.output)
+ protected tokenize(): this {
+ while (this.place._index < this.chunks.length) this.go(this.code)
+ this.eos && this.state && this.go(this.code)
+ return this
+ }
- this.resolve(this.initialize, 0)
- this.events = resolveAll(this.resolveAll, this.events, this.context)
+ /**
+ * Write a slice of chunks.
+ *
+ * The eof code (`null`) can be used to signal end of stream.
+ *
+ * @see {@linkcode Chunk}
+ * @see {@linkcode Event}
+ *
+ * @public
+ * @instance
+ *
+ * @param {Chunk[]} slice - Chunks
+ * @return {Event[]} List of events
+ */
+ public write(slice: Chunk[]): Event[] {
+ /**
+ * New chunks.
+ *
+ * @const {Code[]} chunks
+ */
+ const chunks: Code[] = slice.flatMap(chunk => {
+ /* c8 ignore next 2 */ return typeof chunk === 'string'
+ ? this.preprocess(chunk)
+ : chunk
+ })
- return this
+ this.chunks = push(this.chunks, chunks)
+ this.tokenize()
+
+ // exit if not done, resolvers might change stuff
+ /* c8 ignore next */ if (!this.eos) return []
+
+ /**
+ * Constructs with `resolveAll` handlers.
+ *
+ * > 👉 Includes partial constructs.
+ *
+ * @const {Partial[]} constructs
+ */
+ const constructs: Partial[] = [
+ ...this.resolveAll,
+ ...this.resolvers.map(resolveAll => ({ resolveAll }))
+ ]
+
+ this.resolve(this.initialize, 0)
+ return this.events = resolveAll(constructs, this.events, this.context)
}
}
diff --git a/src/preprocess.ts b/src/preprocess.ts
new file mode 100644
index 0000000..fca6271
--- /dev/null
+++ b/src/preprocess.ts
@@ -0,0 +1,136 @@
+/**
+ * @file preprocess
+ * @module vfile-lexer/preprocess
+ */
+
+import { codes } from './enums'
+import type { PreprocessOptions } from './interfaces'
+import type {
+ Code,
+ Column,
+ Encoding,
+ FileLike,
+ Preprocessor,
+ Value
+} from './types'
+
+/**
+ * Create a preprocessor to turn a value into character code chunks.
+ *
+ * @see {@linkcode PreprocessOptions}
+ * @see {@linkcode Preprocessor}
+ *
+ * @param {PreprocessOptions | null | undefined} [options] - Configuration
+ * @return {Preprocessor} Character code preprocessor
+ */
+function preprocess(
+ this: void,
+ options?: PreprocessOptions | null | undefined
+): Preprocessor {
+ const { tabSize = 2 } = options ?? {}
+ return preprocessor
+
+ /**
+ * Turn `value` into character code chunks.
+ *
+ * @param {FileLike | Value | null | undefined} value - Value to preprocess
+ * @param {Encoding | null | undefined} [encoding] - Character encoding to use
+ * when value or its contents is {@linkcode Uint8Array}
+ * @param {boolean | null | undefined} [end] - End of stream?
+ * @return {Code[]} Character code chunks
+ */
+ function preprocessor(
+ value: FileLike | Value | null | undefined,
+ encoding?: Encoding | null | undefined,
+ end?: boolean | null | undefined
+ ): Code[] {
+ /**
+ * Character code chunks.
+ *
+ * @const {Code[]} chunks
+ */
+ const chunks: Code[] = []
+
+ if (
+ (typeof value === 'string' && value) ||
+ (typeof value === 'object' && value)
+ ) {
+ value = typeof value === 'object' && 'value' in value
+ ? value.value
+ : value
+
+ value = typeof value === 'string'
+ ? value.toString()
+ : new TextDecoder(encoding ?? undefined).decode(value)
+
+ /**
+ * Current column.
+ *
+ * @var {Column} column
+ */
+ let column: Column = 1
+
+ /**
+ * Index of current character code.
+ *
+ * @var {number} index
+ */
+ let index: number = 0
+
+ while (index < value.length) {
+ /**
+ * Character code.
+ *
+ * @var {NonNullable} code
+ */
+ let code: NonNullable = value[index]!.codePointAt(0)!
+
+ /**
+ * Difference between next column and current column.
+ *
+ * @var {number} k
+ */
+ let k: number = 1
+
+ switch (true) {
+ case code === codes.cr:
+ if (value[index + 1]?.codePointAt(0) === codes.lf) {
+ chunks.push(codes.crlf)
+ k++
+ } else {
+ chunks.push(codes.vcr)
+ }
+
+ column = 1
+ break
+ case code === codes.ht:
+ /**
+ * Next column.
+ *
+ * @const {number} n
+ */
+ const n: number = Math.ceil(column / tabSize) * tabSize
+
+ chunks.push(codes.vht)
+ while (column++ < n) chunks.push(codes.vs)
+
+ break
+ case code === codes.lf:
+ chunks.push(codes.vlf)
+ column = 1
+ break
+ default:
+ chunks.push(code)
+ column++
+ break
+ }
+
+ index += k
+ }
+ }
+
+ return end && chunks.push(codes.eof), chunks
+ }
+}
+
+export default preprocess
diff --git a/src/tokenize.ts b/src/tokenize.ts
index b6032d3..efb0cce 100644
--- a/src/tokenize.ts
+++ b/src/tokenize.ts
@@ -3,24 +3,80 @@
* @module vfile-lexer/tokenize
*/
-import type { Options } from '#src/interfaces'
-import type { VFile, Value } from 'vfile'
+import type {
+ Encoding,
+ Event,
+ FileLike,
+ TokenizeOptions,
+ Value
+} from '#src/types'
import Lexer from './lexer'
+import preprocess from './preprocess'
/**
- * Tokenize `file`.
+ * Tokenize `value`.
*
- * @see {@linkcode Lexer}
- * @see {@linkcode Options}
- * @see {@linkcode VFile}
+ * @see {@linkcode Encoding}
+ * @see {@linkcode Event}
+ * @see {@linkcode FileLike}
+ * @see {@linkcode TokenizeOptions}
* @see {@linkcode Value}
*
- * @param {Value | VFile} file - File to tokenize
- * @param {Options} options - Lexer options
- * @return {Lexer} Lexer instance
+ * @param {FileLike | Value | null | undefined} value - Value to tokenize
+ * @param {Encoding | null | undefined} [encoding] - Character encoding to use
+ * when `value` or its contents is {@linkcode Uint8Array}
+ * @param {TokenizeOptions | null | undefined} [options] - Configuration options
+ * @return {Event[]} List of events
*/
-function tokenize(file: Value | VFile, options: Options): Lexer {
- return Lexer.create(file, options).tokenize()
+function tokenize(
+ value: FileLike | Value | null | undefined,
+ encoding?: Encoding | null | undefined,
+ options?: TokenizeOptions | null | undefined
+): Event[]
+
+/**
+ * Tokenize `value`.
+ *
+ * @see {@linkcode FileLike}
+ * @see {@linkcode Event}
+ * @see {@linkcode TokenizeOptions}
+ * @see {@linkcode Value}
+ *
+ * @param {FileLike | Value | null | undefined} value - Value to tokenize
+ * @param {TokenizeOptions | null | undefined} [options] - Tokenize options
+ * @return {Event[]} List of events
+ */
+function tokenize(
+ value: FileLike | Value | null | undefined,
+ options?: TokenizeOptions | null | undefined
+): Event[]
+
+/**
+ * Tokenize `value`.
+ *
+ * @see {@linkcode FileLike}
+ * @see {@linkcode Event}
+ * @see {@linkcode TokenizeOptions}
+ * @see {@linkcode Value}
+ *
+ * @param {FileLike | Value | null | undefined} value - Value to tokenize
+ * @param {Encoding | TokenizeOptions | null | undefined} [encoding] - Character
+ * encoding to use when `value` or its contents is {@linkcode Uint8Array}, or
+ * configuration options
+ * @param {TokenizeOptions | null | undefined} [options] - Configuration options
+ * @return {Event[]} List of events
+ */
+function tokenize(
+ value: FileLike | Value | null | undefined,
+ encoding?: Encoding | TokenizeOptions | null | undefined,
+ options?: TokenizeOptions | null | undefined
+): Event[] {
+ if (typeof encoding === 'object' && encoding) {
+ options = encoding
+ encoding = undefined
+ }
+
+ return new Lexer(options).write(preprocess(options)(value, encoding, true))
}
export default tokenize
diff --git a/src/types/__tests__/chunk.spec-d.ts b/src/types/__tests__/chunk.spec-d.ts
new file mode 100644
index 0000000..ab5ffb9
--- /dev/null
+++ b/src/types/__tests__/chunk.spec-d.ts
@@ -0,0 +1,17 @@
+/**
+ * @file Type Tests - Chunk
+ * @module vfile-lexer/types/tests/unit-d/Chunk
+ */
+
+import type TestSubject from '../chunk'
+import type Code from '../code'
+
+describe('unit-d:types/Chunk', () => {
+ it('should extract Code', () => {
+ expectTypeOf().extract().not.toBeNever()
+ })
+
+ it('should extract string', () => {
+ expectTypeOf().extract().not.toBeNever()
+ })
+})
diff --git a/src/types/__tests__/code-check.spec-d.ts b/src/types/__tests__/code-check.spec-d.ts
new file mode 100644
index 0000000..45a7859
--- /dev/null
+++ b/src/types/__tests__/code-check.spec-d.ts
@@ -0,0 +1,21 @@
+/**
+ * @file Type Tests - CodeCheck
+ * @module vfile-reader/types/tests/unit-d/CodeCheck
+ */
+
+import type Code from '../code'
+import type TestSubject from '../code-check'
+
+describe('unit-d:types/CodeCheck', () => {
+ describe('parameters', () => {
+ it('should be callable with [Code]', () => {
+ expectTypeOf().parameters.toEqualTypeOf<[Code]>()
+ })
+ })
+
+ describe('returns', () => {
+ it('should return boolean', () => {
+ expectTypeOf().returns.toEqualTypeOf()
+ })
+ })
+})
diff --git a/src/types/__tests__/code.spec-d.ts b/src/types/__tests__/code.spec-d.ts
new file mode 100644
index 0000000..eac6059
--- /dev/null
+++ b/src/types/__tests__/code.spec-d.ts
@@ -0,0 +1,16 @@
+/**
+ * @file Type Tests - Code
+ * @module vfile-lexer/types/tests/unit-d/Code
+ */
+
+import type TestSubject from '../code'
+
+describe('unit-d:types/Code', () => {
+ it('should extract null', () => {
+ expectTypeOf().extract().not.toBeNever()
+ })
+
+ it('should extract number', () => {
+ expectTypeOf().extract().not.toBeNever()
+ })
+})
diff --git a/src/types/__tests__/construct-pack.spec-d.ts b/src/types/__tests__/construct-pack.spec-d.ts
new file mode 100644
index 0000000..6e00b43
--- /dev/null
+++ b/src/types/__tests__/construct-pack.spec-d.ts
@@ -0,0 +1,17 @@
+/**
+ * @file Type Tests - ConstructPack
+ * @module vfile-lexer/types/tests/unit-d/ConstructPack
+ */
+
+import type { Construct } from '#src/interfaces'
+import type TestSubject from '../construct-pack'
+
+describe('unit-d:types/ConstructPack', () => {
+ it('should extract Construct', () => {
+ expectTypeOf().extract().not.toBeNever()
+ })
+
+ it('should extract Construct[]', () => {
+ expectTypeOf().extract().not.toBeNever()
+ })
+})
diff --git a/src/types/__tests__/construct-record.spec-d.ts b/src/types/__tests__/construct-record.spec-d.ts
deleted file mode 100644
index 5b8cdb4..0000000
--- a/src/types/__tests__/construct-record.spec-d.ts
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * @file Type Tests - ConstructRecord
- * @module vfile-lexer/types/tests/unit-d/ConstructRecord
- */
-
-import type { Nilable } from '@flex-development/tutils'
-import { codes } from '@flex-development/vfile-reader'
-import type TestSubject from '../construct-record'
-import type RecordConstructs from '../constructs-record'
-
-describe('unit-d:types/ConstructRecord', () => {
- type Value = Nilable
-
- it('should match [[x: `${number}`]: RecordConstructs | null | undefined]', () => {
- expectTypeOf()
- .toHaveProperty(`${codes.backtick}`)
- .toEqualTypeOf()
- })
-
- it('should match [[x: "null"]: RecordConstructs | null | undefined]', () => {
- expectTypeOf()
- .toHaveProperty(`${codes.eof}`)
- .toEqualTypeOf()
- })
-
- it('should match [[x: number]: RecordConstructs | null | undefined]', () => {
- expectTypeOf()
- .toHaveProperty(codes.at)
- .toEqualTypeOf()
- })
-})
diff --git a/src/types/__tests__/constructs-record.spec-d.ts b/src/types/__tests__/constructs-record.spec-d.ts
deleted file mode 100644
index 05b7840..0000000
--- a/src/types/__tests__/constructs-record.spec-d.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * @file Type Tests - RecordConstructs
- * @module vfile-lexer/types/tests/unit-d/RecordConstructs
- */
-
-import type { Construct } from '#src/interfaces'
-import type TestSubject from '../constructs-record'
-
-describe('unit-d:types/RecordConstructs', () => {
- it('should extract Construct', () => {
- expectTypeOf().extract().not.toBeNever()
- })
-
- it('should extract Construct[]', () => {
- expectTypeOf().extract().not.toBeNever()
- })
-
- it('should extract readonly Construct[]', () => {
- expectTypeOf().extract().not.toBeNever()
- })
-})
diff --git a/src/types/__tests__/constructs.spec-d.ts b/src/types/__tests__/constructs.spec-d.ts
index 6185033..84a6888 100644
--- a/src/types/__tests__/constructs.spec-d.ts
+++ b/src/types/__tests__/constructs.spec-d.ts
@@ -3,16 +3,16 @@
* @module vfile-lexer/types/tests/unit-d/Constructs
*/
-import type ConstructRecord from '../construct-record'
+import type { ConstructRecord } from '#src/interfaces'
+import type ConstructPack from '../construct-pack'
import type TestSubject from '../constructs'
-import type RecordConstructs from '../constructs-record'
describe('unit-d:types/Constructs', () => {
it('should extract ConstructRecord', () => {
expectTypeOf().extract().not.toBeNever()
})
- it('should extract RecordConstructs', () => {
- expectTypeOf().extract().not.toBeNever()
+ it('should extract ConstructPack', () => {
+ expectTypeOf().extract().not.toBeNever()
})
})
diff --git a/src/types/__tests__/consume.spec-d.ts b/src/types/__tests__/consume.spec-d.ts
index 2634685..edfb494 100644
--- a/src/types/__tests__/consume.spec-d.ts
+++ b/src/types/__tests__/consume.spec-d.ts
@@ -3,7 +3,7 @@
* @module vfile-lexer/types/tests/unit-d/Consume
*/
-import type { Code } from '@flex-development/vfile-reader'
+import type Code from '../code'
import type TestSubject from '../consume'
describe('unit-d:types/Consume', () => {
diff --git a/src/types/__tests__/define-skip.spec-d.ts b/src/types/__tests__/define-skip.spec-d.ts
new file mode 100644
index 0000000..aa1f160
--- /dev/null
+++ b/src/types/__tests__/define-skip.spec-d.ts
@@ -0,0 +1,25 @@
+/**
+ * @file Type Tests - DefineSkip
+ * @module vfile-lexer/types/tests/unit-d/DefineSkip
+ */
+
+import type { Point } from '#src/interfaces'
+import type TestSubject from '../define-skip'
+
+describe('unit-d:types/DefineSkip', () => {
+ describe('parameters', () => {
+ it('should be callable with [Pick]', () => {
+ // Arrange
+ type P = [Pick]
+
+ // Expect
+ expectTypeOf().parameters.toEqualTypeOf()
+ })
+ })
+
+ describe('returns', () => {
+ it('should return undefined', () => {
+ expectTypeOf().returns.toEqualTypeOf()
+ })
+ })
+})
diff --git a/src/types/__tests__/encoding.spec-d.ts b/src/types/__tests__/encoding.spec-d.ts
new file mode 100644
index 0000000..2773bb9
--- /dev/null
+++ b/src/types/__tests__/encoding.spec-d.ts
@@ -0,0 +1,32 @@
+/**
+ * @file Type Tests - Encoding
+ * @module vfile-lexer/types/tests/unit-d/Encoding
+ */
+
+import type TestSubject from '../encoding'
+
+describe('unit-d:types/Encoding', () => {
+ it('should extract "unicode-1-1-utf-8"', () => {
+ expectTypeOf().extract<'unicode-1-1-utf-8'>().not.toBeNever()
+ })
+
+ it('should extract "utf-16be"', () => {
+ expectTypeOf().extract<'utf-16be'>().not.toBeNever()
+ })
+
+ it('should extract "utf-16le"', () => {
+ expectTypeOf().extract<'utf-16le'>().not.toBeNever()
+ })
+
+ it('should extract "utf-8"', () => {
+ expectTypeOf().extract<'utf-8'>().not.toBeNever()
+ })
+
+ it('should extract "utf16"', () => {
+ expectTypeOf().extract<'utf16'>().not.toBeNever()
+ })
+
+ it('should extract "utf8"', () => {
+ expectTypeOf().extract<'utf8'>().not.toBeNever()
+ })
+})
diff --git a/src/types/__tests__/enter.spec-d.ts b/src/types/__tests__/enter.spec-d.ts
index 3b4a1d5..4850d43 100644
--- a/src/types/__tests__/enter.spec-d.ts
+++ b/src/types/__tests__/enter.spec-d.ts
@@ -3,16 +3,15 @@
* @module vfile-lexer/types/tests/unit-d/Enter
*/
-import type { Token } from '#src/interfaces'
+import type { Token, TokenFields } from '#src/interfaces'
import type TestSubject from '../enter'
-import type TokenFields from '../token-fields'
import type TokenType from '../token-type'
describe('unit-d:types/Enter', () => {
describe('parameters', () => {
- it('should be callable with [TokenType, (Partial | null | undefined)?]', () => {
+ it('should be callable with [TokenType, (TokenFields | null | undefined)?]', () => {
// Arrange
- type P = [TokenType, (Partial | null | undefined)?]
+ type P = [TokenType, (TokenFields | null | undefined)?]
// Expect
expectTypeOf().parameters.toEqualTypeOf()
diff --git a/src/types/__tests__/event-type.spec-d.ts b/src/types/__tests__/event-type.spec-d.ts
index 29f13b8..4c18096 100644
--- a/src/types/__tests__/event-type.spec-d.ts
+++ b/src/types/__tests__/event-type.spec-d.ts
@@ -7,10 +7,6 @@ import type { ev } from '#src/enums'
import type TestSubject from '../event-type'
describe('unit-d:types/EventType', () => {
- it('should extract ev', () => {
- expectTypeOf().extract().not.toBeNever()
- })
-
it('should extract keyof typeof ev', () => {
expectTypeOf().extract().not.toBeNever()
})
diff --git a/src/types/__tests__/event.spec-d.ts b/src/types/__tests__/event.spec-d.ts
index 60d645c..589f119 100644
--- a/src/types/__tests__/event.spec-d.ts
+++ b/src/types/__tests__/event.spec-d.ts
@@ -3,13 +3,13 @@
* @module vfile-lexer/types/tests/unit-d/Event
*/
-import type tk from '#fixtures/tk'
+import type tt from '#fixtures/tt'
import type { Token, TokenizeContext } from '#src/interfaces'
import type TestSubject from '../event'
import type EventType from '../event-type'
describe('unit-d:types/Event', () => {
- type T = tk.whitespace
+ type T = tt.typeMetadata
type Subject = TestSubject
it('should match [0: EventType]', () => {
diff --git a/src/types/__tests__/file-like.spec-d.ts b/src/types/__tests__/file-like.spec-d.ts
new file mode 100644
index 0000000..5ea7fe7
--- /dev/null
+++ b/src/types/__tests__/file-like.spec-d.ts
@@ -0,0 +1,13 @@
+/**
+ * @file Type Tests - FileLike
+ * @module vfile-lexer/types/tests/unit-d/FileLike
+ */
+
+import type TestSubject from '../file-like'
+import type Value from '../value'
+
+describe('unit-d:types/FileLike', () => {
+ it('should match [value: Value]', () => {
+ expectTypeOf().toHaveProperty('value').toMatchTypeOf()
+ })
+})
diff --git a/src/types/__tests__/finalize-context.spec-d.ts b/src/types/__tests__/finalize-context.spec-d.ts
index 1140a55..02cc688 100644
--- a/src/types/__tests__/finalize-context.spec-d.ts
+++ b/src/types/__tests__/finalize-context.spec-d.ts
@@ -18,9 +18,9 @@ describe('unit-d:types/FinalizeContext', () => {
})
describe('returns', () => {
- it('should return TokenizeContext | null | undefined | void', () => {
+ it('should return TokenizeContext | null | undefined', () => {
// Arrange
- type Expect = TokenizeContext | null | undefined | void
+ type Expect = TokenizeContext | null | undefined
// Expect
expectTypeOf().returns.toEqualTypeOf()
diff --git a/src/types/__tests__/guard.spec-d.ts b/src/types/__tests__/guard.spec-d.ts
index dfbdf8e..aac623e 100644
--- a/src/types/__tests__/guard.spec-d.ts
+++ b/src/types/__tests__/guard.spec-d.ts
@@ -4,7 +4,7 @@
*/
import type { TokenizeContext } from '#src/interfaces'
-import type { Code } from '@flex-development/vfile-reader'
+import type Code from '../code'
import type TestSubject from '../guard'
describe('unit-d:types/Guard', () => {
diff --git a/src/types/__tests__/initializer.spec-d.ts b/src/types/__tests__/initializer.spec-d.ts
index ec6057a..a792cdb 100644
--- a/src/types/__tests__/initializer.spec-d.ts
+++ b/src/types/__tests__/initializer.spec-d.ts
@@ -3,8 +3,7 @@
* @module vfile-lexer/types/tests/unit-d/Initializer
*/
-import type { TokenizeContext } from '#src/interfaces'
-import type Effects from '../effects'
+import type { Effects, TokenizeContext } from '#src/interfaces'
import type TestSubject from '../initializer'
import type State from '../state'
diff --git a/src/types/__tests__/now.spec-d.ts b/src/types/__tests__/now.spec-d.ts
new file mode 100644
index 0000000..47c8ec6
--- /dev/null
+++ b/src/types/__tests__/now.spec-d.ts
@@ -0,0 +1,22 @@
+/**
+ * @file Type Tests - Now
+ * @module vfile-lexer/types/tests/unit-d/Now
+ */
+
+import type { Place } from '#src/interfaces'
+import type { EmptyArray } from '@flex-development/tutils'
+import type TestSubject from '../now'
+
+describe('unit-d:types/Now', () => {
+ describe('parameters', () => {
+ it('should be callable with []', () => {
+ expectTypeOf().parameters.toEqualTypeOf()
+ })
+ })
+
+ describe('returns', () => {
+ it('should return Place', () => {
+ expectTypeOf().returns.toEqualTypeOf()
+ })
+ })
+})
diff --git a/src/types/__tests__/preprocessor.spec-d.ts b/src/types/__tests__/preprocessor.spec-d.ts
new file mode 100644
index 0000000..2dea09c
--- /dev/null
+++ b/src/types/__tests__/preprocessor.spec-d.ts
@@ -0,0 +1,32 @@
+/**
+ * @file Type Tests - Preprocessor
+ * @module vfile-lexer/types/tests/unit-d/Preprocessor
+ */
+
+import type Code from '../code'
+import type Encoding from '../encoding'
+import type FileLike from '../file-like'
+import type TestSubject from '../preprocessor'
+import type Value from '../value'
+
+describe('unit-d:types/Preprocessor', () => {
+ describe('parameters', () => {
+ it('should be callable with [FileLike | Value | null | undefined, (Encoding | null | undefined)?, (boolean | null | undefined)?]', () => {
+ // Arrange
+ type P = [
+ value: FileLike | Value | null | undefined,
+ encoding?: Encoding | null | undefined,
+ end?: boolean | null | undefined
+ ]
+
+ // Expect
+ expectTypeOf().parameters.toEqualTypeOf()
+ })
+ })
+
+ describe('returns', () => {
+ it('should return Code[]', () => {
+ expectTypeOf().returns.toEqualTypeOf()
+ })
+ })
+})
diff --git a/src/types/__tests__/slice-serialize.spec-d.ts b/src/types/__tests__/slice-serialize.spec-d.ts
new file mode 100644
index 0000000..07ed1ac
--- /dev/null
+++ b/src/types/__tests__/slice-serialize.spec-d.ts
@@ -0,0 +1,28 @@
+/**
+ * @file Type Tests - SliceSerialize
+ * @module vfile-lexer/types/tests/unit-d/SliceSerialize
+ */
+
+import type { Position } from '#src/interfaces'
+import type TestSubject from '../slice-serialize'
+
+describe('unit-d:types/SliceSerialize', () => {
+ describe('parameters', () => {
+ it('should be callable with [Position, (boolean | null | undefined)?]', () => {
+ // Arrange
+ type P = [
+ range: Position,
+ expandTabs?: boolean | null | undefined
+ ]
+
+ // Expect
+ expectTypeOf().parameters.toEqualTypeOf()
+ })
+ })
+
+ describe('returns', () => {
+ it('should return string', () => {
+ expectTypeOf().returns.toEqualTypeOf()
+ })
+ })
+})
diff --git a/src/types/__tests__/slice-stream.spec-d.ts b/src/types/__tests__/slice-stream.spec-d.ts
new file mode 100644
index 0000000..2d7a185
--- /dev/null
+++ b/src/types/__tests__/slice-stream.spec-d.ts
@@ -0,0 +1,22 @@
+/**
+ * @file Type Tests - SliceStream
+ * @module vfile-lexer/types/tests/unit-d/SliceStream
+ */
+
+import type { Position } from '#src/interfaces'
+import type Code from '../code'
+import type TestSubject from '../slice-stream'
+
+describe('unit-d:types/SliceStream', () => {
+ describe('parameters', () => {
+ it('should be callable with [Position]', () => {
+ expectTypeOf().parameters.toEqualTypeOf<[Position]>()
+ })
+ })
+
+ describe('returns', () => {
+ it('should return Code[]', () => {
+ expectTypeOf().returns.toEqualTypeOf()
+ })
+ })
+})
diff --git a/src/types/__tests__/state.spec-d.ts b/src/types/__tests__/state.spec-d.ts
index 8291f11..a7f5d4e 100644
--- a/src/types/__tests__/state.spec-d.ts
+++ b/src/types/__tests__/state.spec-d.ts
@@ -4,7 +4,7 @@
*/
import type { Optional } from '@flex-development/tutils'
-import type { Code } from '@flex-development/vfile-reader'
+import type Code from '../code'
import type TestSubject from '../state'
describe('unit-d:types/State', () => {
diff --git a/src/types/__tests__/token-factory.spec-d.ts b/src/types/__tests__/token-factory.spec-d.ts
index 8e616e6..89c7f70 100644
--- a/src/types/__tests__/token-factory.spec-d.ts
+++ b/src/types/__tests__/token-factory.spec-d.ts
@@ -3,16 +3,15 @@
* @module vfile-lexer/types/tests/unit-d/TokenFactory
*/
-import type { Token } from '#src/interfaces'
+import type { Token, TokenInfo } from '#src/interfaces'
import type TestSubject from '../token-factory'
-import type TokenFields from '../token-fields'
import type TokenType from '../token-type'
describe('unit-d:types/TokenFactory', () => {
describe('parameters', () => {
- it('should be callable with [TokenType, TokenFields]', () => {
+ it('should be callable with [TokenType, TokenInfo]', () => {
// Arrange
- type P = [TokenType, TokenFields]
+ type P = [TokenType, TokenInfo]
// Expect
expectTypeOf().parameters.toEqualTypeOf()
diff --git a/src/types/__tests__/token-fields.spec-d.ts b/src/types/__tests__/token-fields.spec-d.ts
deleted file mode 100644
index 1703478..0000000
--- a/src/types/__tests__/token-fields.spec-d.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-/**
- * @file Type Tests - TokenFields
- * @module vfile-lexer/types/tests/unit-d/TokenFields
- */
-
-import type { Token } from '#src/interfaces'
-import type TestSubject from '../token-fields'
-
-describe('unit-d:types/TokenFields', () => {
- it('should equal Omit', () => {
- expectTypeOf().toEqualTypeOf>()
- })
-})
diff --git a/src/types/__tests__/tokenize-options.spec-d.ts b/src/types/__tests__/tokenize-options.spec-d.ts
new file mode 100644
index 0000000..ec44bba
--- /dev/null
+++ b/src/types/__tests__/tokenize-options.spec-d.ts
@@ -0,0 +1,17 @@
+/**
+ * @file Type Tests - TokenizeOptions
+ * @module vfile-lexer/types/tests/unit-d/TokenizeOptions
+ */
+
+import type { Options, PreprocessOptions } from '#src/interfaces'
+import type TestSubject from '../tokenize-options'
+
+describe('unit-d:types/TokenizeOptions', () => {
+ it('should match Options', () => {
+ expectTypeOf().toMatchTypeOf()
+ })
+
+ it('should match PreprocessOptions', () => {
+ expectTypeOf().toMatchTypeOf()
+ })
+})
diff --git a/src/types/__tests__/tokenizer.spec-d.ts b/src/types/__tests__/tokenizer.spec-d.ts
index 308614d..8f8bc35 100644
--- a/src/types/__tests__/tokenizer.spec-d.ts
+++ b/src/types/__tests__/tokenizer.spec-d.ts
@@ -3,8 +3,7 @@
* @module vfile-lexer/types/tests/unit-d/Tokenizer
*/
-import type { TokenizeContext } from '#src/interfaces'
-import type Effects from '../effects'
+import type { Effects, TokenizeContext } from '#src/interfaces'
import type State from '../state'
import type TestSubject from '../tokenizer'
diff --git a/src/types/__tests__/value.spec-d.ts b/src/types/__tests__/value.spec-d.ts
new file mode 100644
index 0000000..b23785e
--- /dev/null
+++ b/src/types/__tests__/value.spec-d.ts
@@ -0,0 +1,16 @@
+/**
+ * @file Type Tests - Value
+ * @module vfile-lexer/types/tests/unit-d/Value
+ */
+
+import type TestSubject from '../value'
+
+describe('unit-d:types/Value', () => {
+ it('should extract Uint8Array', () => {
+ expectTypeOf().extract().not.toBeNever()
+ })
+
+ it('should extract string', () => {
+ expectTypeOf().extract().not.toBeNever()
+ })
+})
diff --git a/src/types/__tests__/write.spec-d.ts b/src/types/__tests__/write.spec-d.ts
new file mode 100644
index 0000000..92c2b99
--- /dev/null
+++ b/src/types/__tests__/write.spec-d.ts
@@ -0,0 +1,22 @@
+/**
+ * @file Type Tests - Write
+ * @module vfile-lexer/types/tests/unit-d/Write
+ */
+
+import type Chunk from '../chunk'
+import type Event from '../event'
+import type TestSubject from '../write'
+
+describe('unit-d:types/Write', () => {
+ describe('parameters', () => {
+ it('should be callable with [Chunk[]]', () => {
+ expectTypeOf().parameters.toEqualTypeOf<[Chunk[]]>()
+ })
+ })
+
+ describe('returns', () => {
+ it('should return Event[]', () => {
+ expectTypeOf().returns.toEqualTypeOf()
+ })
+ })
+})
diff --git a/src/types/chunk.ts b/src/types/chunk.ts
new file mode 100644
index 0000000..86a1ca0
--- /dev/null
+++ b/src/types/chunk.ts
@@ -0,0 +1,15 @@
+/**
+ * @file Type Aliases - Chunk
+ * @module vfile-lexer/types/Chunk
+ */
+
+import type Code from './code'
+
+/**
+ * A character code or slice of a buffer in the form of a string.
+ *
+ * @see {@linkcode Code}
+ */
+type Chunk = Code | string
+
+export type { Chunk as default }
diff --git a/src/types/code-check.ts b/src/types/code-check.ts
new file mode 100644
index 0000000..c494a5d
--- /dev/null
+++ b/src/types/code-check.ts
@@ -0,0 +1,18 @@
+/**
+ * @file Type Aliases - DefineSkip
+ * @module vfile-lexer/types/DefineSkip
+ */
+
+import type Code from './code'
+
+/**
+ * Check whether a character code passes a test.
+ *
+ * @see {@linkcode Code}
+ *
+ * @param {Code} code - Character code to check
+ * @return {boolean} `true` if `code` passes test
+ */
+type CodeCheck = (code: Code) => boolean
+
+export type { CodeCheck as default }
diff --git a/src/types/code.ts b/src/types/code.ts
new file mode 100644
index 0000000..dbd670f
--- /dev/null
+++ b/src/types/code.ts
@@ -0,0 +1,17 @@
+/**
+ * @file Type Aliases - Code
+ * @module vfile-lexer/types/Code
+ */
+
+/**
+ * A character code, with `null` denoting end of stream (eof).
+ *
+ * This often the same as what [`String#codePointAt`][codepointat] yields, but
+ * meaning is added to other values as well. Negative integers can be used to
+ * represent line endings and tabs.
+ *
+ * [codepointat]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String/codePointAt
+ */
+type Code = number | null
+
+export type { Code as default }
diff --git a/src/types/construct-pack.ts b/src/types/construct-pack.ts
new file mode 100644
index 0000000..393d5cd
--- /dev/null
+++ b/src/types/construct-pack.ts
@@ -0,0 +1,15 @@
+/**
+ * @file Type Aliases - ConstructPack
+ * @module vfile-lexer/types/ConstructPack
+ */
+
+import type { Construct } from '#src/interfaces'
+
+/**
+ * A single construct or list of constructs.
+ *
+ * @see {@linkcode Construct}
+ */
+type ConstructPack = Construct | Construct[]
+
+export type { ConstructPack as default }
diff --git a/src/types/construct-record.ts b/src/types/construct-record.ts
deleted file mode 100644
index 9fd853f..0000000
--- a/src/types/construct-record.ts
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * @file Type Aliases - ConstructRecord
- * @module vfile-lexer/types/ConstructRecord
- */
-
-import type RecordConstructs from './constructs-record'
-
-/**
- * Several constructs, mapped from their initial codes.
- */
-type ConstructRecord = {
- /**
- * Try tokenizing constructs that start with the specified character code.
- *
- * > 👉 Does not run on end-of-file code (`null`).
- *
- * @see {@linkcode RecordConstructs}
- */
- [code: `${number}` | number]: RecordConstructs | null | undefined
-
- /**
- * Try tokenizing constructs that start with any character code.
- *
- * > 👉 Does not run on end-of-file code (`null`).
- *
- * @see {@linkcode RecordConstructs}
- */
- null?: RecordConstructs | null | undefined
-}
-
-export type { ConstructRecord as default }
diff --git a/src/types/constructs-record.ts b/src/types/constructs-record.ts
deleted file mode 100644
index 8ba1226..0000000
--- a/src/types/constructs-record.ts
+++ /dev/null
@@ -1,15 +0,0 @@
-/**
- * @file Type Aliases - RecordConstructs
- * @module vfile-lexer/types/RecordConstructs
- */
-
-import type { Construct } from '#src/interfaces'
-
-/**
- * A single construct or list of constructs.
- *
- * @see {@linkcode Construct}
- */
-type RecordConstructs = Construct | Construct[] | readonly Construct[]
-
-export type { RecordConstructs as default }
diff --git a/src/types/constructs.ts b/src/types/constructs.ts
index eb49571..8908600 100644
--- a/src/types/constructs.ts
+++ b/src/types/constructs.ts
@@ -3,16 +3,16 @@
* @module vfile-lexer/types/Constructs
*/
-import type ConstructRecord from './construct-record'
-import type RecordConstructs from './constructs-record'
+import type { ConstructRecord } from '#src/interfaces'
+import type ConstructPack from './construct-pack'
/**
* A single construct, list of constructs, or several constructs mapped from
* their initial codes.
*
+ * @see {@linkcode ConstructPack}
* @see {@linkcode ConstructRecord}
- * @see {@linkcode RecordConstructs}
*/
-type Constructs = ConstructRecord | RecordConstructs
+type Constructs = ConstructRecord | ConstructPack
export type { Constructs as default }
diff --git a/src/types/consume.ts b/src/types/consume.ts
index 9ca5209..b255b8d 100644
--- a/src/types/consume.ts
+++ b/src/types/consume.ts
@@ -3,7 +3,7 @@
* @module vfile-lexer/types/Consume
*/
-import type { Code } from '@flex-development/vfile-reader'
+import type Code from './code'
/**
* Deal with a character `code` and move onto the next.
diff --git a/src/types/define-skip.ts b/src/types/define-skip.ts
new file mode 100644
index 0000000..c2f3111
--- /dev/null
+++ b/src/types/define-skip.ts
@@ -0,0 +1,25 @@
+/**
+ * @file Type Aliases - DefineSkip
+ * @module vfile-lexer/types/DefineSkip
+ */
+
+import type { Point } from '#src/interfaces'
+
+/**
+ * Define a skip.
+ *
+ * As containers may "nibble" a prefix from margins, where a line starts after
+ * that prefix can be defined here.
+ *
+ * When a tokenizer moves after consuming a line ending corresponding to
+ * `point.line`, the tokenizer shifts past the prefix based on the column in the
+ * shifted point.
+ *
+ * @see {@linkcode Point}
+ *
+ * @param {Pick} point - Skip point
+ * @return {undefined} Nothing
+ */
+type DefineSkip = (point: Pick) => undefined
+
+export type { DefineSkip as default }
diff --git a/src/types/encoding.ts b/src/types/encoding.ts
new file mode 100644
index 0000000..65975a6
--- /dev/null
+++ b/src/types/encoding.ts
@@ -0,0 +1,23 @@
+/**
+ * @file Type Aliases - Encoding
+ * @module vfile-lexer/types/Encoding
+ */
+
+/**
+ * Encodings supported by {@linkcode TextDecoder}.
+ *
+ * > 👉 Arbitrary encodings can be supported depending on how the engine is
+ * > built, so any string *could* be valid.
+ *
+ * @see https://nodejs.org/api/util.html#whatwg-supported-encodings
+ */
+type Encoding =
+ | 'unicode-1-1-utf-8' // always supported in node
+ | 'utf-16be' // not supported when ICU is disabled
+ | 'utf-16le' // always supported in node
+ | 'utf-8' // always supported in node
+ | 'utf16' // always supported in node
+ | 'utf8' // always supported in node
+ | (string & {}) // everything else (depends on browser, or full ICU data)
+
+export type { Encoding as default }
diff --git a/src/types/enter.ts b/src/types/enter.ts
index 01cd1cc..b0c7886 100644
--- a/src/types/enter.ts
+++ b/src/types/enter.ts
@@ -3,8 +3,7 @@
* @module vfile-lexer/types/Enter
*/
-import type { Token } from '#src/interfaces'
-import type TokenFields from './token-fields'
+import type { Token, TokenFields } from '#src/interfaces'
import type TokenType from './token-type'
/**
@@ -15,9 +14,9 @@ import type TokenType from './token-type'
* @see {@linkcode Token}
*
* @param {TokenType} type - Token type
- * @param {(Partial | null)?} fields - Token fields
+ * @param {TokenFields | null | undefined} [fields] - Token fields
* @return {Token} Open token
*/
-type Enter = (type: TokenType, fields?: Partial | null) => Token
+type Enter = (type: TokenType, fields?: TokenFields | null | undefined) => Token
export type { Enter as default }
diff --git a/src/types/event-type.ts b/src/types/event-type.ts
index 9049cc1..d2adca1 100644
--- a/src/types/event-type.ts
+++ b/src/types/event-type.ts
@@ -3,13 +3,9 @@
* @module vfile-lexer/types/Event
*/
-import type { ev } from '#src/enums'
-
/**
* Union of event types.
- *
- * @see {@linkcode ev}
*/
-type EventType = keyof typeof ev | ev
+type EventType = 'enter' | 'exit'
export type { EventType as default }
diff --git a/src/types/file-like.ts b/src/types/file-like.ts
new file mode 100644
index 0000000..afd35bc
--- /dev/null
+++ b/src/types/file-like.ts
@@ -0,0 +1,20 @@
+/**
+ * @file Type Aliases - FileLike
+ * @module vfile-lexer/types/FileLike
+ */
+
+import type Value from './value'
+
+/**
+ * A file-like structure.
+ */
+type FileLike = {
+ /**
+ * Contents of file.
+ *
+ * @see {@linkcode Value}
+ */
+ value: Value
+}
+
+export type { FileLike as default }
diff --git a/src/types/finalize-context.ts b/src/types/finalize-context.ts
index adb9744..a4d0e3e 100644
--- a/src/types/finalize-context.ts
+++ b/src/types/finalize-context.ts
@@ -4,17 +4,21 @@
*/
import type { TokenizeContext } from '#src/interfaces'
+import type Lexer from '#src/lexer'
/**
* Finalize the tokenization context.
*
* @see {@linkcode TokenizeContext}
*
- * @param {TokenizeContext} context - Base context
- * @return {TokenizeContext | null | undefined | void} Final context
+ * @this {Lexer}
+ *
+ * @param {TokenizeContext} base - Base context
+ * @return {TokenizeContext | null | undefined} Final context
*/
type FinalizeContext = (
+ this: Lexer,
base: TokenizeContext
-) => TokenizeContext | null | undefined | void
+) => TokenizeContext | null | undefined
export type { FinalizeContext as default }
diff --git a/src/types/guard.ts b/src/types/guard.ts
index 7ead90b..0702a81 100644
--- a/src/types/guard.ts
+++ b/src/types/guard.ts
@@ -4,7 +4,7 @@
*/
import type { TokenizeContext } from '#src/interfaces'
-import type { Code } from '@flex-development/vfile-reader'
+import type Code from './code'
/**
* Check the given character `code`.
diff --git a/src/types/index.ts b/src/types/index.ts
index 21dec22..2f64a2c 100644
--- a/src/types/index.ts
+++ b/src/types/index.ts
@@ -3,35 +3,39 @@
* @module vfile-lexer/types
*/
-export type {
- Code,
- CodeCheck,
- Offset,
- Range,
- RangeTuple,
- ReaderSlice
-} from '@flex-development/vfile-reader'
+export type { Column, Line, Offset } from '@flex-development/unist-util-types'
+export type { Indices, SerializedPoint } from '@flex-development/vfile-location'
export type {
default as Attempt,
default as Check,
default as Interrupt
} from './attempt'
-export type { default as ConstructRecord } from './construct-record'
+export type { default as Chunk } from './chunk'
+export type { default as Code } from './code'
+export type { default as CodeCheck } from './code-check'
+export type { default as ConstructPack } from './construct-pack'
export type { default as Constructs } from './constructs'
-export type { default as RecordConstructs } from './constructs-record'
export type { default as Consume } from './consume'
-export type { default as Effects } from './effects'
+export type { default as DefineSkip } from './define-skip'
+export type { default as Encoding } from './encoding'
export type { default as Enter } from './enter'
export type { default as Event } from './event'
export type { default as EventType } from './event-type'
export type { default as Exit } from './exit'
+export type { default as FileLike } from './file-like'
export type { default as FinalizeContext } from './finalize-context'
export type { default as Guard } from './guard'
export type { default as Initializer } from './initializer'
+export type { default as Now } from './now'
+export type { default as Preprocessor } from './preprocessor'
export type { default as Resolver } from './resolver'
export type { default as ReturnHandle } from './return-handle'
+export type { default as SliceSerialize } from './slice-serialize'
+export type { default as SliceStream } from './slice-stream'
export type { default as State } from './state'
export type { default as TokenFactory } from './token-factory'
-export type { default as TokenFields } from './token-fields'
export type { default as TokenType } from './token-type'
+export type { default as TokenizeOptions } from './tokenize-options'
export type { default as Tokenizer } from './tokenizer'
+export type { default as Value } from './value'
+export type { default as Write } from './write'
diff --git a/src/types/initializer.ts b/src/types/initializer.ts
index f667026..5905594 100644
--- a/src/types/initializer.ts
+++ b/src/types/initializer.ts
@@ -3,8 +3,7 @@
* @module vfile-lexer/types/Initializer
*/
-import type { TokenizeContext } from '#src/interfaces'
-import type Effects from './effects'
+import type { Effects, TokenizeContext } from '#src/interfaces'
import type State from './state'
import type Tokenizer from './tokenizer'
diff --git a/src/types/now.ts b/src/types/now.ts
new file mode 100644
index 0000000..dcc3863
--- /dev/null
+++ b/src/types/now.ts
@@ -0,0 +1,17 @@
+/**
+ * @file Type Aliases - Now
+ * @module vfile-lexer/types/Now
+ */
+
+import type { Place } from '#src/interfaces'
+
+/**
+ * Get the current point in the file.
+ *
+ * @see {@linkcode Place}
+ *
+ * @return {Place} Current place in file
+ */
+type Now = () => Place
+
+export type { Now as default }
diff --git a/src/types/preprocessor.ts b/src/types/preprocessor.ts
new file mode 100644
index 0000000..67b009c
--- /dev/null
+++ b/src/types/preprocessor.ts
@@ -0,0 +1,31 @@
+/**
+ * @file Type Aliases - Preprocessor
+ * @module vfile-lexer/types/Preprocessor
+ */
+
+import type Code from './code'
+import type Encoding from './encoding'
+import type FileLike from './file-like'
+import type Value from './value'
+
+/**
+ * Turn `value` into character code chunks.
+ *
+ * @see {@linkcode Code}
+ * @see {@linkcode Encoding}
+ * @see {@linkcode FileLike}
+ * @see {@linkcode Value}
+ *
+ * @param {FileLike | Value | null | undefined} value - Value to preprocess
+ * @param {Encoding | null | undefined} [encoding] - Character encoding to use
+ * when value or its contents is {@linkcode Uint8Array}
+ * @param {boolean | null | undefined} [end] - End of stream?
+ * @return {Code[]} Character code chunks
+ */
+type Preprocessor = (
+ value: FileLike | Value | null | undefined,
+ encoding?: Encoding | null | undefined,
+ end?: boolean | null | undefined
+) => Code[]
+
+export type { Preprocessor as default }
diff --git a/src/types/slice-serialize.ts b/src/types/slice-serialize.ts
new file mode 100644
index 0000000..03d18c5
--- /dev/null
+++ b/src/types/slice-serialize.ts
@@ -0,0 +1,22 @@
+/**
+ * @file Type Aliases - SliceSerialize
+ * @module vfile-lexer/types/SliceSerialize
+ */
+
+import type { Position } from '#src/interfaces'
+
+/**
+ * Get the text spanning `range` without changing the position of the reader.
+ *
+ * @see {@linkcode Position}
+ *
+ * @param {Position} range - Slice position
+ * @param {boolean | null | undefined} [expandTabs] - Expand tabs?
+ * @return {string} Serialized slice
+ */
+type SliceSerialize = (
+ range: Position,
+ expandTabs?: boolean | null | undefined
+) => string
+
+export type { SliceSerialize as default }
diff --git a/src/types/slice-stream.ts b/src/types/slice-stream.ts
new file mode 100644
index 0000000..70ab600
--- /dev/null
+++ b/src/types/slice-stream.ts
@@ -0,0 +1,20 @@
+/**
+ * @file Type Aliases - SliceStream
+ * @module vfile-lexer/types/SliceStream
+ */
+
+import type { Position } from '#src/interfaces'
+import type Code from './code'
+
+/**
+ * Get the chunks spanning `range`.
+ *
+ * @see {@linkcode Code}
+ * @see {@linkcode Token}
+ *
+ * @param {Position} range - Position in stream
+ * @return {Code[]} List of chunks
+ */
+type SliceStream = (range: Position) => Code[]
+
+export type { SliceStream as default }
diff --git a/src/types/state.ts b/src/types/state.ts
index 4c0386e..1c88601 100644
--- a/src/types/state.ts
+++ b/src/types/state.ts
@@ -3,7 +3,7 @@
* @module vfile-lexer/types/State
*/
-import type { Code } from '@flex-development/vfile-reader'
+import type Code from './code'
/**
* The main unit in the state machine: a function that gets a character code and
diff --git a/src/types/token-factory.ts b/src/types/token-factory.ts
index 4e5e65e..1d7db08 100644
--- a/src/types/token-factory.ts
+++ b/src/types/token-factory.ts
@@ -3,21 +3,20 @@
* @module vfile-lexer/types/TokenFactory
*/
-import type { Token } from '#src/interfaces'
-import type TokenFields from './token-fields'
+import type { Token, TokenInfo } from '#src/interfaces'
import type TokenType from './token-type'
/**
* Create a new token.
*
- * @see {@linkcode TokenFields}
+ * @see {@linkcode TokenInfo}
* @see {@linkcode TokenType}
* @see {@linkcode Token}
*
* @param {TokenType} type - Token type
- * @param {TokenFields} fields - Token fields
+ * @param {TokenInfo} info - Token info
* @return {Token} New token
*/
-type TokenFactory = (type: TokenType, fields: TokenFields) => Token
+type TokenFactory = (type: TokenType, info: TokenInfo) => Token
export type { TokenFactory as default }
diff --git a/src/types/token-fields.ts b/src/types/token-fields.ts
deleted file mode 100644
index 92216b4..0000000
--- a/src/types/token-fields.ts
+++ /dev/null
@@ -1,15 +0,0 @@
-/**
- * @file Type Aliases - TokenFields
- * @module vfile-lexer/types/TokenFields
- */
-
-import type { Token } from '#src/interfaces'
-
-/**
- * Additional token fields.
- *
- * @see {@linkcode Token}
- */
-type TokenFields = Omit
-
-export type { TokenFields as default }
diff --git a/src/types/token-type.ts b/src/types/token-type.ts
index 5697bf9..e459e43 100644
--- a/src/types/token-type.ts
+++ b/src/types/token-type.ts
@@ -11,6 +11,6 @@ import type { TokenTypeMap } from '#src/interfaces'
* To register custom token types, augment {@linkcode TokenTypeMap}. They will
* be added to this union automatically.
*/
-type TokenType = keyof TokenTypeMap
+type TokenType = Extract
export type { TokenType as default }
diff --git a/src/types/tokenize-options.ts b/src/types/tokenize-options.ts
new file mode 100644
index 0000000..630eca5
--- /dev/null
+++ b/src/types/tokenize-options.ts
@@ -0,0 +1,16 @@
+/**
+ * @file Type Aliases - TokenizeOptions
+ * @module vfile-lexer/types/TokenizeOptions
+ */
+
+import type { Options, PreprocessOptions } from '#src/interfaces'
+
+/**
+ * Tokenize options.
+ *
+ * @see {@linkcode Options}
+ * @see {@linkcode PreprocessOptions}
+ */
+type TokenizeOptions = Options & PreprocessOptions
+
+export type { TokenizeOptions as default }
diff --git a/src/types/tokenizer.ts b/src/types/tokenizer.ts
index f9b2e0b..089dd9f 100644
--- a/src/types/tokenizer.ts
+++ b/src/types/tokenizer.ts
@@ -3,8 +3,7 @@
* @module vfile-lexer/types/Tokenizer
*/
-import type { TokenizeContext } from '#src/interfaces'
-import type Effects from './effects'
+import type { Effects, TokenizeContext } from '#src/interfaces'
import type State from './state'
/**
diff --git a/src/types/value.ts b/src/types/value.ts
new file mode 100644
index 0000000..1aaddfd
--- /dev/null
+++ b/src/types/value.ts
@@ -0,0 +1,13 @@
+/**
+ * @file Type Aliases - Value
+ * @module vfile-lexer/types/Value
+ */
+
+/**
+ * Contents of a file.
+ *
+ * Can either be text, or a {@linkcode Uint8Array} like structure.
+ */
+type Value = Uint8Array | string
+
+export type { Value as default }
diff --git a/src/types/write.ts b/src/types/write.ts
new file mode 100644
index 0000000..f4c10f5
--- /dev/null
+++ b/src/types/write.ts
@@ -0,0 +1,22 @@
+/**
+ * @file Type Aliases - Write
+ * @module vfile-lexer/types/Write
+ */
+
+import type Chunk from './chunk'
+import type Event from './event'
+
+/**
+ * Write a slice of chunks.
+ *
+ * The eof code (`null`) can be used to signal end of stream.
+ *
+ * @see {@linkcode Chunk}
+ * @see {@linkcode Event}
+ *
+ * @param {Chunk[]} slice - Chunks
+ * @return {Event[]} List of events
+ */
+type Write = (slice: Chunk[]) => Event[]
+
+export type { Write as default }
diff --git a/src/utils/__tests__/is-line-ending.spec.ts b/src/utils/__tests__/is-line-ending.spec.ts
new file mode 100644
index 0000000..4ee8c19
--- /dev/null
+++ b/src/utils/__tests__/is-line-ending.spec.ts
@@ -0,0 +1,25 @@
+/**
+ * @file Unit Tests - isLineEnding
+ * @module vfile-lexer/utils/tests/unit/isLineEnding
+ */
+
+import { codes } from '#src/enums'
+import testSubject from '../is-line-ending'
+
+describe('unit:utils/isLineEnding', () => {
+ it('should return false if `code` is not line ending', () => {
+ expect(testSubject(codes.eof)).to.be.false
+ })
+
+ it.each([
+ 'cr',
+ 'crlf',
+ 'lf',
+ 'ls',
+ 'ps',
+ 'vcr',
+ 'vlf'
+ ])('should return true if `code` is line ending (codes.%s)', key => {
+ expect(testSubject(codes[key])).to.be.true
+ })
+})
diff --git a/src/utils/__tests__/resolve-all.functional.spec.ts b/src/utils/__tests__/resolve-all.functional.spec.ts
index 9ad6993..06e2b3c 100644
--- a/src/utils/__tests__/resolve-all.functional.spec.ts
+++ b/src/utils/__tests__/resolve-all.functional.spec.ts
@@ -3,29 +3,29 @@
* @module vfile-lexer/utils/tests/functional/resolveAll
*/
-import { ev, tt } from '#src/enums'
-import type { Construct, Token, TokenizeContext } from '#src/interfaces'
+import tt from '#fixtures/tt'
+import { ev } from '#src/enums'
+import type { Construct, Place, Token, TokenizeContext } from '#src/interfaces'
import type { Event, Resolver } from '#src/types'
-import type { MockInstance } from '#tests/interfaces'
-import type { Point } from '@flex-development/vfile-reader'
+import type { MockInstance } from 'vitest'
import testSubject from '../resolve-all'
describe('functional:utils/resolveAll', () => {
let constructs: { resolveAll: MockInstance }[]
let context: TokenizeContext
let events: Event[]
- let point: Point
+ let point: Place
let resolveAll: MockInstance
let token: Token
beforeEach(() => {
- point = { column: 1, line: 1, offset: 0 }
- token = { end: point, start: point, type: tt.sof }
+ point = { _index: 0, column: 1, line: 1, offset: 0 }
+ token = { end: point, start: point, type: tt.eof }
context = {} as unknown as TokenizeContext
events = [[ev.enter, token, context], [ev.exit, token, context]]
- resolveAll = vi.fn<[Event[], TokenizeContext], Event[]>(() => events)
+ resolveAll = vi.fn(() => events)
constructs = [{ resolveAll }, { resolveAll }]
})
diff --git a/src/utils/__tests__/resolve-slice.functional.spec.ts b/src/utils/__tests__/resolve-slice.functional.spec.ts
index 20edecc..d77f1be 100644
--- a/src/utils/__tests__/resolve-slice.functional.spec.ts
+++ b/src/utils/__tests__/resolve-slice.functional.spec.ts
@@ -3,30 +3,29 @@
* @module vfile-lexer/utils/tests/functional/resolveSlice
*/
-import tk from '#fixtures/tk'
+import tt from '#fixtures/tt'
import { ev } from '#src/enums'
import type { Token, TokenizeContext } from '#src/interfaces'
-import type { Event } from '#src/types'
-import type { MockInstance } from '#tests/interfaces'
-import type { Range } from '@flex-development/vfile-reader'
+import type { Event, SliceSerialize } from '#src/types'
+import type { MockInstance } from 'vitest'
import testSubject from '../resolve-slice'
describe('functional:utils/resolveSlice', () => {
let context: TokenizeContext
let events: Event[]
- let sliceSerialize: MockInstance
+ let sliceSerialize: MockInstance
let token: Token
let value: string
beforeEach(() => {
token = {
- end: { column: 1, line: 2, offset: 17 },
- start: { column: 17, line: 1, offset: 16 },
- type: tk.whitespace
+ end: { _index: 16, column: 17, line: 1, offset: 16 },
+ start: { _index: 0, column: 1, line: 1, offset: 0 },
+ type: tt.typeMetadata
}
- value = '\n'
- sliceSerialize = vi.fn<[range: Range], string>(() => value)
+ value = '{{ id: string }}'
+ sliceSerialize = vi.fn