From fae10c3fe2e867e8c8842e5a77f60c1a81bf080b Mon Sep 17 00:00:00 2001 From: Lexus Drumgold Date: Wed, 17 Jul 2024 20:28:50 -0400 Subject: [PATCH] feat: construct records Signed-off-by: Lexus Drumgold --- .github/infrastructure.yml | 2 - .github/workflows/ci.yml | 2 - src/__tests__/tokenize.integration.spec.ts | 20 +-- src/constructs/initialize.ts | 120 +++++++----------- src/interfaces/__tests__/options.spec-d.ts | 7 +- .../__tests__/tokenize-context.spec-d.ts | 10 +- src/interfaces/options.ts | 10 +- src/interfaces/tokenize-context.ts | 11 +- src/lexer.ts | 78 ++++++++---- .../__tests__/construct-record.spec-d.ts | 31 +++++ .../__tests__/constructs-record.spec-d.ts | 21 +++ src/types/__tests__/constructs.spec-d.ts | 15 +-- src/types/construct-record.ts | 31 +++++ src/types/constructs-record.ts | 15 +++ src/types/constructs.ts | 11 +- src/types/index.ts | 2 + 16 files changed, 230 insertions(+), 156 deletions(-) create mode 100644 src/types/__tests__/construct-record.spec-d.ts create mode 100644 src/types/__tests__/constructs-record.spec-d.ts create mode 100644 src/types/construct-record.ts create mode 100644 src/types/constructs-record.ts diff --git a/.github/infrastructure.yml b/.github/infrastructure.yml index c7ef872..cd72675 100644 --- a/.github/infrastructure.yml +++ b/.github/infrastructure.yml @@ -39,8 +39,6 @@ branches: - context: test (18) - context: test (19) - context: test (20) - - context: typescript (5.3.3) - - context: typescript (5.4.5) - context: typescript (5.5.2) - context: typescript (latest) strict: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9096712..d5b58e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,8 +245,6 @@ jobs: matrix: typescript-version: - ${{ needs.preflight.outputs.version-typescript }} - - 5.4.5 - - 5.3.3 - latest steps: - id: checkout diff --git a/src/__tests__/tokenize.integration.spec.ts b/src/__tests__/tokenize.integration.spec.ts index e9037b2..7827b81 100644 --- a/src/__tests__/tokenize.integration.spec.ts +++ b/src/__tests__/tokenize.integration.spec.ts @@ -9,6 +9,7 @@ import type { Options } from '#src/interfaces' import { inlineTag, numeric, punctuator, string, ws } from '#tests/constructs' import token from '#tests/utils/token' import { identity } from '@flex-development/tutils' +import { codes } from '@flex-development/vfile-reader' import { readSync as read } from 'to-vfile' import type { VFile, Value } from 'vfile' import type Lexer from '../lexer' @@ -47,18 +48,21 @@ describe('integration:tokenize', () => { describe('non-empty file', () => { it.each<[VFile, (Partial | null | undefined)?]>([ [read('__fixtures__/inline-tag.txt'), { - constructs: [inlineTag, ws], + constructs: { + [codes.cr]: ws, + [codes.leftBrace]: inlineTag, + [codes.lf]: ws, + [codes.space]: ws + }, context: vi.fn(), - disabled: [tk.whitespace], - initialize: { - name: initialize.name, - resolveAll: vi.fn(identity), - tokenize: initialize.tokenize - } + disabled: [tk.whitespace] }], [read('__fixtures__/hello.txt'), { constructs: [string, punctuator], - context: vi.fn(identity) + context: vi.fn(identity), + initialize: Object.assign(initialize([string, punctuator]), { + resolveAll: vi.fn(identity) + }) }], [read('__fixtures__/strings.txt'), { constructs: [string, punctuator] diff --git a/src/constructs/initialize.ts b/src/constructs/initialize.ts index e89bd10..dbf7036 100644 --- a/src/constructs/initialize.ts +++ b/src/constructs/initialize.ts @@ -4,98 +4,64 @@ */ import { tt } from '#src/enums' -import type { - Construct, - InitialConstruct, - TokenizeContext -} from '#src/interfaces' -import type { Effects, State } from '#src/types' -import type { Code } from '@flex-development/vfile-reader' +import type { InitialConstruct, TokenizeContext } from '#src/interfaces' +import type { Constructs, Effects, State } from '#src/types' +import { codes, type Code } from '@flex-development/vfile-reader' import eof from './eof' /** - * Initialization construct. + * Create an initial construct. * - * @const {InitialConstruct} initialize + * @see {@linkcode Constructs} + * @see {@linkcode InitialConstruct} + * + * @param {Constructs} constructs - Construct(s) to try + * @return {InitialConstruct} Initial construct */ -const initialize: InitialConstruct = { - /** - * Construct name. - */ - name: 'vfile-lexer:initialize', - - /** - * Set up a state machine to handle character codes streaming in. - * - * @see {@linkcode Effects} - * @see {@linkcode State} - * @see {@linkcode TokenizeContext} - * - * @this {TokenizeContext} - * - * @param {Effects} effects - Context object to transition state machine - * @return {State} Initial state - */ - tokenize(this: TokenizeContext, effects: Effects): State { +function initialize(constructs: Constructs): InitialConstruct { + return { /** - * Tokenize context. - * - * @const {TokenizeContext} self + * Construct name. */ - const self: TokenizeContext = this + name: 'vfile-lexer:initialize', /** - * List of constructs. + * Set up a state machine to handle character codes streaming in. * - * @const {Construct[]} constructs - */ - const constructs: Construct[] = [eof, ...self.constructs] - - /** - * Try to tokenize a list of constructs. + * @see {@linkcode Effects} + * @see {@linkcode State} + * @see {@linkcode TokenizeContext} * - * @var {State} state - */ - let state: State = effects.attempt(constructs, succ, fail) - - void (effects.enter(tt.sof), effects.exit(tt.sof)) - return succ - - /** - * Eat `code`. + * @this {TokenizeContext} * - * @param {Code} code - Current character code - * @return {State | undefined} Next state + * @param {Effects} effects - Context object to transition state machine + * @return {State} Initial state */ - function eat(code: Code): State | undefined { - return effects.consume(code), state - } + tokenize(this: TokenizeContext, effects: Effects): State { + void (effects.enter(tt.sof), effects.exit(tt.sof)) + return state - /** - * Try tokenizing the next construct, and move onto the next character code - * if all constructs fail. - * - * @param {Code} code - Current character code - * @return {State | undefined} Next state - */ - function fail(code: Code): State | undefined { - return effects.attempt( - constructs, - succ, - constructs.indexOf(self.construct!) === constructs.length - 1 - ? eat - : fail - )(code) - } + /** + * Consume `code` and try tokenizing the next construct. + * + * @param {Code} code - Current character code + * @return {State | undefined} Next state + */ + function eat(code: Code): State | undefined { + return code === codes.eof + ? effects.attempt(eof)(code) + : (effects.consume(code), state) + } - /** - * Try tokenizing the next construct. - * - * @param {Code} code - Current character code - * @return {State | undefined} Next state - */ - function succ(code: Code): State | undefined { - return (state = effects.attempt(constructs, succ, fail))(code) + /** + * Try to tokenize a construct. + * + * @param {Code} code - Current character code + * @return {State | undefined} Next state + */ + function state(code: Code): State | undefined { + return effects.attempt(constructs, state, eat)(code) + } } } } diff --git a/src/interfaces/__tests__/options.spec-d.ts b/src/interfaces/__tests__/options.spec-d.ts index ebc3b93..e3262ab 100644 --- a/src/interfaces/__tests__/options.spec-d.ts +++ b/src/interfaces/__tests__/options.spec-d.ts @@ -3,18 +3,17 @@ * @module vfile-lexer/interfaces/tests/unit-d/Options */ -import type { FinalizeContext, TokenFactory } from '#src/types' +import type { Constructs, FinalizeContext, TokenFactory } from '#src/types' import type { Nilable } from '@flex-development/tutils' import type { Point } from '@flex-development/vfile-reader' -import type Construct from '../construct' import type InitialConstruct from '../construct-initial' import type TestSubject from '../options' describe('unit-d:interfaces/Options', () => { - it('should match [constructs?: readonly Construct[] | null | undefined]', () => { + it('should match [constructs?: Constructs | null | undefined]', () => { expectTypeOf() .toHaveProperty('constructs') - .toEqualTypeOf>() + .toEqualTypeOf>() }) it('should match [context?: FinalizeContext | null | undefined]', () => { diff --git a/src/interfaces/__tests__/tokenize-context.spec-d.ts b/src/interfaces/__tests__/tokenize-context.spec-d.ts index e5095ad..c25bca2 100644 --- a/src/interfaces/__tests__/tokenize-context.spec-d.ts +++ b/src/interfaces/__tests__/tokenize-context.spec-d.ts @@ -25,18 +25,12 @@ describe('unit-d:interfaces/TokenizeContext', () => { expectTypeOf().toHaveProperty('code').toEqualTypeOf() }) - it('should match [construct?: Construct | null | undefined]', () => { + it('should match [currentConstruct?: Construct | null | undefined]', () => { expectTypeOf() - .toHaveProperty('construct') + .toHaveProperty('currentConstruct') .toEqualTypeOf>() }) - it('should match [constructs: readonly Construct[]]', () => { - expectTypeOf() - .toHaveProperty('constructs') - .toEqualTypeOf() - }) - it('should match [disabled: readonly string[]]', () => { expectTypeOf() .toHaveProperty('disabled') diff --git a/src/interfaces/options.ts b/src/interfaces/options.ts index a80059a..2fa3003 100644 --- a/src/interfaces/options.ts +++ b/src/interfaces/options.ts @@ -3,12 +3,8 @@ * @module vfile-lexer/interfaces/Options */ -import type { - FinalizeContext, - TokenFactory -} from '#src/types' +import type { Constructs, FinalizeContext, TokenFactory } from '#src/types' import type { Point } from '@flex-development/vfile-reader' -import type Construct from './construct' import type InitialConstruct from './construct-initial' /** @@ -18,9 +14,9 @@ interface Options { /** * Constructs. * - * @see {@linkcode Construct} + * @see {@linkcode Constructs} */ - constructs?: readonly Construct[] | null | undefined + constructs?: Constructs | null | undefined /** * Finalize the tokenization context. diff --git a/src/interfaces/tokenize-context.ts b/src/interfaces/tokenize-context.ts index a843a39..1ccd45c 100644 --- a/src/interfaces/tokenize-context.ts +++ b/src/interfaces/tokenize-context.ts @@ -33,18 +33,13 @@ interface TokenizeContext { get code(): Code /** - * Current construct. + * The current construct. * - * @see {@linkcode Construct} - */ - construct?: Construct | null | undefined - - /** - * All constructs. + * Constructs that are not `partial` are set here. * * @see {@linkcode Construct} */ - constructs: readonly Construct[] + currentConstruct?: Construct | null | undefined /** * Disabled construct names. diff --git a/src/lexer.ts b/src/lexer.ts index 7e71efd..e2b4d00 100644 --- a/src/lexer.ts +++ b/src/lexer.ts @@ -25,6 +25,7 @@ import type { } from './interfaces' import type { Attempt, + ConstructRecord, Constructs, Effects, Event, @@ -53,17 +54,6 @@ class Lexer { */ protected code: Code - /** - * List of constructs. - * - * @see {@linkcode Construct} - * - * @protected - * @instance - * @member {Readonly} constructs - */ - protected constructs: readonly Construct[] - /** * Character code consumption state, used for tracking bugs. * @@ -78,11 +68,11 @@ class Lexer { * * @see {@linkcode TokenizeContext} * - * @protected + * @public * @instance * @member {TokenizeContext} context */ - protected context: TokenizeContext + public context: TokenizeContext /** * Debug logger. @@ -267,7 +257,6 @@ class Lexer { constructor(file: Value | VFile | null | undefined, options: Options) { assert(options.token, 'expected token factory') - this.constructs = Object.freeze([...(options.constructs ?? [])]) this.debug = debug(options.debug ?? 'vfile-lexer') this.disabled = Object.freeze(options.disabled ?? []) this.reader = new Reader(file, options.from) @@ -277,7 +266,7 @@ class Lexer { this.consumed = true this.eof = false this.events = [] - this.initialize = options.initialize ?? initialize + this.initialize = options.initialize ?? initialize(options.constructs ?? []) this.lastConstruct = null this.lastEvent = 0 this.lastIndex = 0 @@ -292,8 +281,7 @@ class Lexer { const context: TokenizeContext = Object.defineProperties({ check: this.reader.check.bind(this.reader), code: this.code, - construct: this.lastConstruct, - constructs: this.constructs, + currentConstruct: this.lastConstruct, disabled: this.disabled, events: this.events, includes: this.reader.includes.bind(this.reader), @@ -312,7 +300,7 @@ class Lexer { previous: { configurable: false, get: (): Code => this.reader.previous }, token: { configurable: false, - get: (): Readonly => Object.freeze(this.tail) + get: (): Readonly => Object.freeze(Object.assign({}, this.tail)) } }) @@ -381,9 +369,9 @@ class Lexer { /** * Current construct. * - * @var {Construct} current + * @var {Construct} currentConstruct */ - let current: Construct + let currentConstruct: Construct /** * Index of current construct. @@ -399,8 +387,10 @@ class Lexer { */ let list: readonly Construct[] - // handle a single construct, or a list of constructs - return handleConstructList([construct].flat()) + // handle a single construct, list of constructs, or map of constructs + return 'tokenize' in construct || Array.isArray(construct) + ? handleConstructList([construct].flat()) + : handleConstructMap(construct) /** * Handle a list of constructs. @@ -420,6 +410,40 @@ class Lexer { return handleConstruct(constructs[j]!) } + /** + * Handle a construct record. + * + * @param {ConstructRecord} map - Constructs to try + * @return {State} Next state + */ + function handleConstructMap(map: ConstructRecord): State { + return start + + /** + * Check if `value` looks like a construct, or list of constructs. + * + * @param {unknown} value - Value to check + * @return {value is Construct | ReadonlyArray} `true` if + * value is an object + */ + function is(value: unknown): value is Construct | readonly Construct[] { + return typeof value === 'object' + } + + /** + * Start construct tokenization. + * + * @param {Code} code - Current character code + * @return {State | undefined} Next state + */ + function start(code: Code): State | undefined { + return handleConstructList([ + ...[code !== null && map[code]].flat().filter(value => is(value)), + ...[code !== null && map.null].flat().filter(value => is(value)) + ])(code) + } + } + /** * Handle a single construct. * @@ -439,9 +463,9 @@ class Lexer { const { context, disabled, effects } = self const { name, partial, previous, test, tokenize } = construct - current = construct + currentConstruct = construct - if (!partial) context.construct = construct + if (!partial) context.currentConstruct = construct if (fail) self.store() context.interrupt = interrupt @@ -468,7 +492,7 @@ class Lexer { self.debug('ok: `%o`', code) self.consumed = true - onreturn(current) + onreturn(currentConstruct) return succ } @@ -700,7 +724,7 @@ class Lexer { assert(this.lastToken, 'expected last token') this.reader.read(this.lastIndex - this.reader.index) - this.context.construct = this.lastConstruct + this.context.currentConstruct = this.lastConstruct this.events.length = this.lastEvent this.tail = this.lastToken this.tail.next = undefined @@ -718,7 +742,7 @@ class Lexer { * @return {undefined} Nothing */ protected store(): undefined { - this.lastConstruct = this.context.construct + this.lastConstruct = this.context.currentConstruct this.lastEvent = this.events.length this.lastIndex = this.reader.index this.lastToken = this.tail diff --git a/src/types/__tests__/construct-record.spec-d.ts b/src/types/__tests__/construct-record.spec-d.ts new file mode 100644 index 0000000..5b8cdb4 --- /dev/null +++ b/src/types/__tests__/construct-record.spec-d.ts @@ -0,0 +1,31 @@ +/** + * @file Type Tests - ConstructRecord + * @module vfile-lexer/types/tests/unit-d/ConstructRecord + */ + +import type { Nilable } from '@flex-development/tutils' +import { codes } from '@flex-development/vfile-reader' +import type TestSubject from '../construct-record' +import type RecordConstructs from '../constructs-record' + +describe('unit-d:types/ConstructRecord', () => { + type Value = Nilable + + it('should match [[x: `${number}`]: RecordConstructs | null | undefined]', () => { + expectTypeOf() + .toHaveProperty(`${codes.backtick}`) + .toEqualTypeOf() + }) + + it('should match [[x: "null"]: RecordConstructs | null | undefined]', () => { + expectTypeOf() + .toHaveProperty(`${codes.eof}`) + .toEqualTypeOf() + }) + + it('should match [[x: number]: RecordConstructs | null | undefined]', () => { + expectTypeOf() + .toHaveProperty(codes.at) + .toEqualTypeOf() + }) +}) diff --git a/src/types/__tests__/constructs-record.spec-d.ts b/src/types/__tests__/constructs-record.spec-d.ts new file mode 100644 index 0000000..05b7840 --- /dev/null +++ b/src/types/__tests__/constructs-record.spec-d.ts @@ -0,0 +1,21 @@ +/** + * @file Type Tests - RecordConstructs + * @module vfile-lexer/types/tests/unit-d/RecordConstructs + */ + +import type { Construct } from '#src/interfaces' +import type TestSubject from '../constructs-record' + +describe('unit-d:types/RecordConstructs', () => { + it('should extract Construct', () => { + expectTypeOf().extract().not.toBeNever() + }) + + it('should extract Construct[]', () => { + expectTypeOf().extract().not.toBeNever() + }) + + it('should extract readonly Construct[]', () => { + expectTypeOf().extract().not.toBeNever() + }) +}) diff --git a/src/types/__tests__/constructs.spec-d.ts b/src/types/__tests__/constructs.spec-d.ts index 789e9e7..6185033 100644 --- a/src/types/__tests__/constructs.spec-d.ts +++ b/src/types/__tests__/constructs.spec-d.ts @@ -3,19 +3,16 @@ * @module vfile-lexer/types/tests/unit-d/Constructs */ -import type { Construct } from '#src/interfaces' +import type ConstructRecord from '../construct-record' import type TestSubject from '../constructs' +import type RecordConstructs from '../constructs-record' describe('unit-d:types/Constructs', () => { - it('should extract Construct', () => { - expectTypeOf().extract().not.toBeNever() + it('should extract ConstructRecord', () => { + expectTypeOf().extract().not.toBeNever() }) - it('should extract Construct[]', () => { - expectTypeOf().extract().not.toBeNever() - }) - - it('should extract readonly Construct[]', () => { - expectTypeOf().extract().not.toBeNever() + it('should extract RecordConstructs', () => { + expectTypeOf().extract().not.toBeNever() }) }) diff --git a/src/types/construct-record.ts b/src/types/construct-record.ts new file mode 100644 index 0000000..9fd853f --- /dev/null +++ b/src/types/construct-record.ts @@ -0,0 +1,31 @@ +/** + * @file Type Aliases - ConstructRecord + * @module vfile-lexer/types/ConstructRecord + */ + +import type RecordConstructs from './constructs-record' + +/** + * Several constructs, mapped from their initial codes. + */ +type ConstructRecord = { + /** + * Try tokenizing constructs that start with the specified character code. + * + * > 👉 Does not run on end-of-file code (`null`). + * + * @see {@linkcode RecordConstructs} + */ + [code: `${number}` | number]: RecordConstructs | null | undefined + + /** + * Try tokenizing constructs that start with any character code. + * + * > 👉 Does not run on end-of-file code (`null`). + * + * @see {@linkcode RecordConstructs} + */ + null?: RecordConstructs | null | undefined +} + +export type { ConstructRecord as default } diff --git a/src/types/constructs-record.ts b/src/types/constructs-record.ts new file mode 100644 index 0000000..8ba1226 --- /dev/null +++ b/src/types/constructs-record.ts @@ -0,0 +1,15 @@ +/** + * @file Type Aliases - RecordConstructs + * @module vfile-lexer/types/RecordConstructs + */ + +import type { Construct } from '#src/interfaces' + +/** + * A single construct or list of constructs. + * + * @see {@linkcode Construct} + */ +type RecordConstructs = Construct | Construct[] | readonly Construct[] + +export type { RecordConstructs as default } diff --git a/src/types/constructs.ts b/src/types/constructs.ts index c9be20d..eb49571 100644 --- a/src/types/constructs.ts +++ b/src/types/constructs.ts @@ -3,13 +3,16 @@ * @module vfile-lexer/types/Constructs */ -import type { Construct } from '#src/interfaces' +import type ConstructRecord from './construct-record' +import type RecordConstructs from './constructs-record' /** - * A single construct, or a list of constructs. + * A single construct, list of constructs, or several constructs mapped from + * their initial codes. * - * @see {@linkcode Construct} + * @see {@linkcode ConstructRecord} + * @see {@linkcode RecordConstructs} */ -type Constructs = Construct | Construct[] | readonly Construct[] +type Constructs = ConstructRecord | RecordConstructs export type { Constructs as default } diff --git a/src/types/index.ts b/src/types/index.ts index 769a22d..21dec22 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -16,7 +16,9 @@ export type { default as Check, default as Interrupt } from './attempt' +export type { default as ConstructRecord } from './construct-record' export type { default as Constructs } from './constructs' +export type { default as RecordConstructs } from './constructs-record' export type { default as Consume } from './consume' export type { default as Effects } from './effects' export type { default as Enter } from './enter'