From 6327e442dcb77c11e358d866d4fccbd8c883186e Mon Sep 17 00:00:00 2001 From: Jason Dent Date: Tue, 30 Jul 2024 12:40:44 +0200 Subject: [PATCH] fix: Improve the speed of checking text. (#6004) --- cspell.json | 1 + integration-tests/src/sh.ts | 2 +- .../SpellingDictionary/CachingDictionary.ts | 40 +- .../src/__snapshots__/index.test.ts.snap | 1 + packages/cspell-dictionary/src/index.ts | 12 + .../cspell-dictionary/src/perf/has.perf.ts | 157 +- packages/cspell-pipe/package.json | 12 +- packages/cspell-pipe/src/operators/append.ts | 8 +- packages/cspell-pipe/src/operators/buffer.ts | 8 +- .../src/operators/concatMap.test.ts | 4 +- .../cspell-pipe/src/operators/concatMap.ts | 36 +- .../cspell-pipe/src/operators/filter.test.ts | 3 +- packages/cspell-pipe/src/operators/filter.ts | 32 +- .../cspell-pipe/src/operators/map.test.ts | 4 +- packages/cspell-pipe/src/operators/map.ts | 30 +- packages/cspell-pipe/src/perf/gen.perf.ts | 47 + packages/cspell-pipe/tsconfig.esm.json | 12 - packages/cspell-pipe/tsconfig.json | 10 +- packages/cspell-trie-lib/src/perf/has.perf.ts | 5 + packages/cspell/cspell.json | 1 + packages/cspell/package.json | 1 + .../src/app/__snapshots__/app.test.ts.snap | 1 + packages/cspell/src/app/environment.ts | 58 + packages/cspell/src/app/lint/LintRequest.ts | 4 +- .../logging/dictionary-logging.csv | 1509 +++++++++++++++++ packages/cspell/src/app/lint/lint.test.ts | 34 +- packages/cspell/src/app/lint/lint.ts | 29 +- .../cspell/src/app/util/writeFile.test.ts | 85 + packages/cspell/src/app/util/writeFile.ts | 30 + pnpm-lock.yaml | 29 +- 30 files changed, 2145 insertions(+), 60 deletions(-) create mode 100644 packages/cspell-pipe/src/perf/gen.perf.ts delete mode 100644 packages/cspell-pipe/tsconfig.esm.json create mode 100644 packages/cspell/src/app/environment.ts create mode 100644 packages/cspell/src/app/lint/__snapshots__/logging/dictionary-logging.csv create mode 100644 packages/cspell/src/app/util/writeFile.test.ts create mode 100644 packages/cspell/src/app/util/writeFile.ts diff --git a/cspell.json b/cspell.json index 03401a492cc..8804c946101 100644 --- a/cspell.json +++ b/cspell.json @@ -31,6 +31,7 @@ ".pnp.{js,cjs}", ".prettierignore", ".yarn", + "__snapshots__", "*.{png,jpg,pdf,svg}", "*.cpuprofile", "*.heapprofile", diff --git a/integration-tests/src/sh.ts b/integration-tests/src/sh.ts index 6e6e6f3fef5..6f76b12af36 100644 --- a/integration-tests/src/sh.ts +++ b/integration-tests/src/sh.ts @@ -24,7 +24,7 @@ export function execAsync(command: string, options: ExecOptions = {}): Promise((resolve) => { Shell.exec( command /* lgtm[js/shell-command-injection-from-environment] */, - { silent: !echo, fatal: bail }, + { silent: !echo, fatal: bail, env: { ...process.env } }, (code, stdout, stderr) => resolve({ code, stdout, stderr }), ); }); diff --git a/packages/cspell-dictionary/src/SpellingDictionary/CachingDictionary.ts b/packages/cspell-dictionary/src/SpellingDictionary/CachingDictionary.ts index 2c8a7a0232d..658fcefdf0c 100644 --- a/packages/cspell-dictionary/src/SpellingDictionary/CachingDictionary.ts +++ b/packages/cspell-dictionary/src/SpellingDictionary/CachingDictionary.ts @@ -17,6 +17,9 @@ let dictionaryCounter = 0; const DefaultAutoCacheSize = 1000; +let logRequests = false; +const log: LogEntry[] = []; + /** * Caching Dictionary remembers method calls to increase performance. */ @@ -30,6 +33,22 @@ export interface CachingDictionary { getPreferredSuggestions(word: string): PreferredSuggestion[] | undefined; } +interface LogEntryBase extends SearchOptions { + time: number; + method: 'has'; + word: string; + value?: unknown; +} + +interface LogEntryHas extends LogEntryBase { + method: 'has'; + value: boolean; +} + +const startTime = performance.now(); + +export type LogEntry = LogEntryHas; + class CachedDict implements CachingDictionary { readonly name: string; readonly id = ++dictionaryCounter; @@ -41,7 +60,16 @@ class CachedDict implements CachingDictionary { // console.log(`CachedDict for ${this.name}`); } - readonly has = autoCache((word: string) => this.dict.has(word, this.options), DefaultAutoCacheSize); + #has = autoCache((word: string) => this.dict.has(word, this.options), DefaultAutoCacheSize); + has = logRequests + ? (word: string): boolean => { + const time = performance.now() - startTime; + const value = this.#has(word); + log.push({ time, method: 'has', word, value }); + return value; + } + : this.#has; + readonly isNoSuggestWord = autoCache( (word: string) => this.dict.isNoSuggestWord(word, this.options), DefaultAutoCacheSize, @@ -56,7 +84,7 @@ class CachedDict implements CachingDictionary { return { name: this.name, id: this.id, - has: extractStats(this.has), + has: extractStats(this.#has), isNoSuggestWord: extractStats(this.isNoSuggestWord), isForbidden: extractStats(this.isForbidden), getPreferredSuggestions: extractStats(this.getPreferredSuggestions), @@ -90,3 +118,11 @@ export function createCachingDictionary( knownOptions.set(dict, cached); return cached; } + +export function enableLogging(enabled = !logRequests): void { + logRequests = enabled; +} + +export function getLog(): LogEntryBase[] { + return log; +} diff --git a/packages/cspell-dictionary/src/__snapshots__/index.test.ts.snap b/packages/cspell-dictionary/src/__snapshots__/index.test.ts.snap index dfa97715be0..43de431b7fc 100644 --- a/packages/cspell-dictionary/src/__snapshots__/index.test.ts.snap +++ b/packages/cspell-dictionary/src/__snapshots__/index.test.ts.snap @@ -2,6 +2,7 @@ exports[`index > verify api 1`] = ` [ + "_debug", "createCachingDictionary", "createCollection", "createFailedToLoadDictionary", diff --git a/packages/cspell-dictionary/src/index.ts b/packages/cspell-dictionary/src/index.ts index e8305c6c1d6..b35fe150888 100644 --- a/packages/cspell-dictionary/src/index.ts +++ b/packages/cspell-dictionary/src/index.ts @@ -1,3 +1,7 @@ +import { + enableLogging as cacheDictionaryEnableLogging, + getLog as cacheDictionaryGetLog, +} from './SpellingDictionary/CachingDictionary.js'; export type { CachingDictionary, FindOptions, @@ -24,3 +28,11 @@ export { createSuggestDictionary, createSuggestOptions, } from './SpellingDictionary/index.js'; + +/** + * Debugging utilities. + */ +export const _debug = { + cacheDictionaryEnableLogging, + cacheDictionaryGetLog, +}; diff --git a/packages/cspell-dictionary/src/perf/has.perf.ts b/packages/cspell-dictionary/src/perf/has.perf.ts index 03c6190c633..8bb01078155 100644 --- a/packages/cspell-dictionary/src/perf/has.perf.ts +++ b/packages/cspell-dictionary/src/perf/has.perf.ts @@ -4,22 +4,34 @@ import { buildITrieFromWords } from 'cspell-trie-lib'; import { loremIpsum } from 'lorem-ipsum'; import { suite } from 'perf-insight'; +import { createCachingDictionary } from '../SpellingDictionary/CachingDictionary.js'; import { createSpellingDictionary } from '../SpellingDictionary/createSpellingDictionary.js'; import { createCollection } from '../SpellingDictionary/SpellingDictionaryCollection.js'; suite('dictionary has', async (test) => { - const words = genWords(10_000); + const words1 = genWords(10_000); const words2 = genWords(1000); const words3 = genWords(1000); - const iTrie = buildITrieFromWords(words); - const dict = createSpellingDictionary(words, 'test', import.meta.url); + const words = words1; + + const iTrie = buildITrieFromWords(words1); + const dict = createSpellingDictionary(words1, 'test', import.meta.url); const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url); const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url); const dictCol = createCollection([dict, dict2, dict3], 'test-collection'); const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse'); + const cacheDictSingle = createCachingDictionary(dict, {}); + const cacheDictCol = createCachingDictionary(dictCol, {}); + + const dictSet = new Set(words); + + test('Set has 100k words', () => { + checkWords(dictSet, words); + }); + test('dictionary has 100k words', () => { checkWords(dict, words); }); @@ -32,6 +44,14 @@ suite('dictionary has', async (test) => { checkWords(dictColRev, words); }); + test('cache dictionary has 100k words', () => { + checkWords(cacheDictSingle, words); + }); + + test('cache collection has 100k words', () => { + checkWords(cacheDictCol, words); + }); + test('iTrie has 100k words', () => { checkWords(iTrie, words); }); @@ -58,6 +78,12 @@ suite('dictionary has Not', async (test) => { const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url); const dictCol = createCollection([dict, dict2, dict3], 'test-collection'); + const dictSet = new Set(words); + + test('Set has not 100k words', () => { + checkWords(dictSet, missingWords, false); + }); + test('dictionary has not 100k words', () => { checkWords(dict, missingWords, false); }); @@ -80,6 +106,104 @@ suite('dictionary has Not', async (test) => { }); }); +suite('dictionary has sampling', async (test) => { + const words1 = genWords(10_000); + const words2 = genWords(1000); + const words3 = genWords(1000); + + const sampleIdx = genSamples(100_000, words1.length); + const wordsSample = sampleIdx.map((i) => words1[i]); + + const iTrie = buildITrieFromWords(words1); + const dict = createSpellingDictionary(words1, 'test', import.meta.url); + const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url); + const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url); + + const dictCol = createCollection([dict, dict2, dict3], 'test-collection'); + const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse'); + + const cacheDictSingle = createCachingDictionary(dict, {}); + const cacheDictCol = createCachingDictionary(dictCol, {}); + + const dictSet = new Set(words1); + + test('Set has 100k words', () => { + checkWords(dictSet, wordsSample); + }); + + test('dictionary has 100k words', () => { + checkWords(dict, wordsSample); + }); + + test('collection has 100k words', () => { + checkWords(dictCol, wordsSample); + }); + + test('collection reverse has 100k words', () => { + checkWords(dictColRev, wordsSample); + }); + + test('cache dictionary has 100k words', () => { + checkWords(cacheDictSingle, wordsSample); + }); + + test('cache collection has 100k words', () => { + checkWords(cacheDictCol, wordsSample); + }); + + test('iTrie has 100k words', () => { + checkWords(iTrie, wordsSample); + }); + + test('iTrie.hasWord has 100k words', () => { + const dict = { has: (word: string) => iTrie.hasWord(word, true) }; + checkWords(dict, wordsSample); + }); + + test('iTrie.data has 100k words', () => { + checkWords(iTrie.data, wordsSample); + }); +}); + +suite('dictionary isForbidden sampling', async (test) => { + const words1 = genWords(10_000); + const words2 = genWords(1000); + const words3 = genWords(1000); + + const sampleIdx = genSamples(100_000, words1.length); + const wordsSample = sampleIdx.map((i) => words1[i]); + + const dict = createSpellingDictionary(words1, 'test', import.meta.url); + const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url); + const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url); + + const dictCol = createCollection([dict, dict2, dict3], 'test-collection'); + const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse'); + + const cacheDictSingle = createCachingDictionary(dict, {}); + const cacheDictCol = createCachingDictionary(dictCol, {}); + + test('dictionary isForbidden 100k words', () => { + checkForForbiddenWords(dict, wordsSample); + }); + + test('collection isForbidden 100k words', () => { + checkForForbiddenWords(dictCol, wordsSample); + }); + + test('collection reverse isForbidden 100k words', () => { + checkForForbiddenWords(dictColRev, wordsSample); + }); + + test('cache dictionary isForbidden 100k words', () => { + checkForForbiddenWords(cacheDictSingle, wordsSample); + }); + + test('cache collection isForbidden 100k words', () => { + checkForForbiddenWords(cacheDictCol, wordsSample); + }); +}); + function checkWords(dict: { has: (word: string) => boolean }, words: string[], expected = true, totalChecks = 100_000) { let has = true; const len = words.length; @@ -94,6 +218,21 @@ function checkWords(dict: { has: (word: string) => boolean }, words: string[], e assert(has, 'All words should be found in the dictionary'); } +function checkForForbiddenWords( + dict: { isForbidden: (word: string) => boolean }, + words: string[], + totalChecks = 100_000, +) { + let result = true; + const len = words.length; + for (let i = 0; i < totalChecks; ++i) { + const word = words[i % len]; + const r = !dict.isForbidden(word); + result = r && result; + } + assert(result, 'All words should not be forbidden'); +} + function genWords(count: number, includeForbidden = true): string[] { const setOfWords = new Set(loremIpsum({ count }).split(' ')); @@ -122,3 +261,15 @@ function genWords(count: number, includeForbidden = true): string[] { return [...setOfWords]; } + +function genSamples(count: number, max: number, depth = 3) { + const r = Array(count); + for (let j = 0; j < count; ++j) { + let n = Math.random() * max; + for (let i = 1; i < depth; ++i) { + n = Math.random() * n; + } + r[j] = Math.floor(n); + } + return r; +} diff --git a/packages/cspell-pipe/package.json b/packages/cspell-pipe/package.json index 1a79c1965d8..e95b80b20ad 100644 --- a/packages/cspell-pipe/package.json +++ b/packages/cspell-pipe/package.json @@ -99,18 +99,21 @@ "!**/*.tsbuildInfo", "!**/__mocks__", "!**/*.spec.*", + "!**/*.perf.*", "!**/*.test.*", + "!**/perf/**", "!**/test/**", "!**/*.map" ], "scripts": { - "build": "tsc -b . -f", - "watch": "tsc -b . -w -f", + "build": "tsc -p .", + "watch": "tsc -p . -w", "clean": "shx rm -rf dist temp coverage \"*.tsbuildInfo\"", "clean-build": "pnpm run clean && pnpm run build", "coverage": "vitest run --coverage", "test-watch": "vitest", - "test": "vitest run" + "test": "vitest run", + "test:perf": "NODE_ENV=production insight --register ts-node/esm --file \"**/*.perf.{mts,ts}\"" }, "repository": { "type": "git", @@ -124,6 +127,7 @@ "node": ">=18" }, "devDependencies": { - "globby": "^14.0.2" + "globby": "^14.0.2", + "perf-insight": "^1.2.0" } } diff --git a/packages/cspell-pipe/src/operators/append.ts b/packages/cspell-pipe/src/operators/append.ts index 375c9998279..406f5fe0533 100644 --- a/packages/cspell-pipe/src/operators/append.ts +++ b/packages/cspell-pipe/src/operators/append.ts @@ -9,14 +9,14 @@ import type { PipeFn } from '../internalTypes.js'; export function opAppendAsync( ...iterablesToAppend: (AsyncIterable | Iterable)[] ): (iter: AsyncIterable | Iterable) => AsyncIterable { - async function* fn(iter: AsyncIterable | Iterable) { + async function* fnAppend(iter: AsyncIterable | Iterable) { yield* iter; for (const i of iterablesToAppend) { yield* i; } } - return fn; + return fnAppend; } /** @@ -25,14 +25,14 @@ export function opAppendAsync( * @returns */ export function opAppendSync(...iterablesToAppend: Iterable[]): (iter: Iterable) => Iterable { - function* fn(iter: Iterable) { + function* fnAppend(iter: Iterable) { yield* iter; for (const i of iterablesToAppend) { yield* i; } } - return fn; + return fnAppend; } export function opAppend(...iterablesToAppend: Iterable[]): PipeFn { diff --git a/packages/cspell-pipe/src/operators/buffer.ts b/packages/cspell-pipe/src/operators/buffer.ts index 6aa4afa55a4..686bd9a4cc4 100644 --- a/packages/cspell-pipe/src/operators/buffer.ts +++ b/packages/cspell-pipe/src/operators/buffer.ts @@ -7,7 +7,7 @@ import type { PipeFn } from '../internalTypes.js'; * @returns A function that takes an async iterable and returns an async iterable of arrays of the given size. */ export function opBufferAsync(size: number): (iter: AsyncIterable) => AsyncIterable { - async function* fn(iter: Iterable | AsyncIterable) { + async function* fnBuffer(iter: Iterable | AsyncIterable) { let buffer: T[] = []; for await (const v of iter) { buffer.push(v); @@ -22,7 +22,7 @@ export function opBufferAsync(size: number): (iter: AsyncIterable) => Asyn } } - return fn; + return fnBuffer; } /** @@ -30,7 +30,7 @@ export function opBufferAsync(size: number): (iter: AsyncIterable) => Asyn * @returns A function that takes an iterable and returns an iterable of arrays of the given size. */ export function opBufferSync(size: number): (iter: Iterable) => Iterable { - function* fn(iter: Iterable) { + function* fnBuffer(iter: Iterable) { let buffer: T[] = []; for (const v of iter) { buffer.push(v); @@ -45,7 +45,7 @@ export function opBufferSync(size: number): (iter: Iterable) => Iterable(size: number): PipeFn { diff --git a/packages/cspell-pipe/src/operators/concatMap.test.ts b/packages/cspell-pipe/src/operators/concatMap.test.ts index 610001019ac..6dccbf224d3 100644 --- a/packages/cspell-pipe/src/operators/concatMap.test.ts +++ b/packages/cspell-pipe/src/operators/concatMap.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'vitest'; import { toArray } from '../helpers/index.js'; import { pipeAsync, pipeSync } from '../pipe.js'; -import { opConcatMap } from './concatMap.js'; +import { _opConcatMapSync, opConcatMap } from './concatMap.js'; describe('Validate map', () => { test('map', async () => { @@ -17,11 +17,13 @@ describe('Validate map', () => { const s = pipeSync(values, mapToLen, opConcatMap(mapFn2)); const a = pipeAsync(values, mapToLen, opConcatMap(mapFn2)); + const s2 = pipeSync(values, mapToLen, _opConcatMapSync(mapFn2)); const sync = toArray(s); const async = await toArray(a); expect(sync).toEqual(expected); expect(async).toEqual(expected); + expect([...s2]).toEqual(expected); }); }); diff --git a/packages/cspell-pipe/src/operators/concatMap.ts b/packages/cspell-pipe/src/operators/concatMap.ts index d3095fd3b47..5831eda67ae 100644 --- a/packages/cspell-pipe/src/operators/concatMap.ts +++ b/packages/cspell-pipe/src/operators/concatMap.ts @@ -13,12 +13,44 @@ export function opConcatMapAsync( } export function opConcatMapSync(mapFn: (v: T) => Iterable): (iter: Iterable) => Iterable { - function* fn(iter: Iterable) { + function fnConcatMapSync(iterable: Iterable): Iterable { + function opConcatMapIterator() { + const iter = iterable[Symbol.iterator](); + let resultsIter: Iterator | undefined = undefined; + function nextConcatMap() { + while (true) { + if (resultsIter) { + const { done, value } = resultsIter.next(); + if (!done) { + return { value }; + } + resultsIter = undefined; + } + const { done, value } = iter.next(); + if (done) { + return { done, value: undefined }; + } + resultsIter = mapFn(value)[Symbol.iterator](); + } + } + return { + next: nextConcatMap, + }; + } + return { + [Symbol.iterator]: opConcatMapIterator, + }; + } + return fnConcatMapSync; +} + +export function _opConcatMapSync(mapFn: (v: T) => Iterable): (iter: Iterable) => Iterable { + function* fnConcatMapSync(iter: Iterable) { for (const v of iter) { yield* mapFn(v); } } - return fn; + return fnConcatMapSync; } export const opConcatMap = (fn: (v: T) => Iterable) => toPipeFn(opConcatMapSync(fn), opConcatMapAsync(fn)); diff --git a/packages/cspell-pipe/src/operators/filter.test.ts b/packages/cspell-pipe/src/operators/filter.test.ts index 71a85999178..609aed16d33 100644 --- a/packages/cspell-pipe/src/operators/filter.test.ts +++ b/packages/cspell-pipe/src/operators/filter.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'vitest'; import { toArray, toAsyncIterable } from '../helpers/index.js'; import { pipeAsync, pipeSync } from '../pipe.js'; -import { opFilter, opFilterAsync } from './filter.js'; +import { _opFilterSync, opFilter, opFilterAsync } from './filter.js'; describe('Validate filter', () => { test('filter', async () => { @@ -22,6 +22,7 @@ describe('Validate filter', () => { expect(sync).toEqual(expected); expect(async).toEqual(expected); + expect([..._opFilterSync(filterFn)(values)]).toEqual(expected); }); type Primitives = string | number | boolean; diff --git a/packages/cspell-pipe/src/operators/filter.ts b/packages/cspell-pipe/src/operators/filter.ts index a4a11c0d1bf..cd34e4a9c0b 100644 --- a/packages/cspell-pipe/src/operators/filter.ts +++ b/packages/cspell-pipe/src/operators/filter.ts @@ -11,26 +11,50 @@ export function opFilterAsync(filterFn: (v: Awaited) => boolean): (iter: A export function opFilterAsync(filterFn: (v: Awaited) => Promise): (iter: AsyncIterable) => AsyncIterable>; // prettier-ignore export function opFilterAsync(filterFn: (v: Awaited) => boolean | Promise): (iter: AsyncIterable) => AsyncIterable> { - async function* fn(iter: Iterable | AsyncIterable) { + async function* genFilter(iter: Iterable | AsyncIterable) { for await (const v of iter) { const pass = await filterFn(v); if (pass) yield v; } } - return fn; + return genFilter; } export function opFilterSync(filterFn: (v: T) => v is S): (iter: Iterable) => Iterable; export function opFilterSync(filterFn: (v: T) => boolean): (iter: Iterable) => Iterable; export function opFilterSync(filterFn: (v: T) => boolean): (iter: Iterable) => Iterable { - function* fn(iter: Iterable) { + function opFilterIterable(iterable: Iterable) { + function opFilterIterator() { + const iter = iterable[Symbol.iterator](); + function nextOpFilter() { + while (true) { + const { done, value } = iter.next(); + if (done) return { done, value: undefined }; + if (filterFn(value)) return { value }; + } + } + return { + next: nextOpFilter, + }; + } + return { + [Symbol.iterator]: opFilterIterator, + }; + } + return opFilterIterable; +} + +export function _opFilterSync(filterFn: (v: T) => v is S): (iter: Iterable) => Iterable; +export function _opFilterSync(filterFn: (v: T) => boolean): (iter: Iterable) => Iterable; +export function _opFilterSync(filterFn: (v: T) => boolean): (iter: Iterable) => Iterable { + function* genFilter(iter: Iterable) { for (const v of iter) { if (filterFn(v)) yield v; } } - return fn; + return genFilter; } export function opFilter(fn: (v: T) => v is S): PipeFn; diff --git a/packages/cspell-pipe/src/operators/map.test.ts b/packages/cspell-pipe/src/operators/map.test.ts index 108409cf078..39ca44601ed 100644 --- a/packages/cspell-pipe/src/operators/map.test.ts +++ b/packages/cspell-pipe/src/operators/map.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'vitest'; import { toArray } from '../helpers/index.js'; import { pipeAsync, pipeSync } from '../pipe.js'; -import { opMap } from './map.js'; +import { _opMapSync, opMap } from './map.js'; describe('Validate map', () => { test('map', async () => { @@ -17,11 +17,13 @@ describe('Validate map', () => { const s = pipeSync(values, mapToLen, opMap(mapFn2)); const a = pipeAsync(values, mapToLen, opMap(mapFn2)); + const sGen = pipeSync(values, mapToLen, _opMapSync(mapFn2)); const sync = toArray(s); const async = await toArray(a); expect(sync).toEqual(expected); expect(async).toEqual(expected); + expect(toArray(sGen)).toEqual(expected); }); }); diff --git a/packages/cspell-pipe/src/operators/map.ts b/packages/cspell-pipe/src/operators/map.ts index 1426ea026a9..df60256cc01 100644 --- a/packages/cspell-pipe/src/operators/map.ts +++ b/packages/cspell-pipe/src/operators/map.ts @@ -1,22 +1,42 @@ import { toPipeFn } from '../helpers/util.js'; export function opMapAsync(mapFn: (v: T) => U): (iter: AsyncIterable) => AsyncIterable { - async function* fn(iter: Iterable | AsyncIterable) { + async function* genMap(iter: Iterable | AsyncIterable) { for await (const v of iter) { yield mapFn(v); } } - return fn; + return genMap; } -export function opMapSync(mapFn: (v: T) => U): (iter: Iterable) => Iterable { - function* fn(iter: Iterable) { +export function _opMapSync(mapFn: (v: T) => U): (iter: Iterable) => Iterable { + function* genMap(iter: Iterable) { for (const v of iter) { yield mapFn(v); } } - return fn; + return genMap; +} + +export function opMapSync(mapFn: (v: T) => U): (iterable: Iterable) => Iterable { + function opMapIterable(iterable: Iterable) { + function opMapIterator() { + const iter = iterable[Symbol.iterator](); + function nextOpMap() { + const { done, value } = iter.next(); + if (done) return { done, value: undefined }; + return { value: mapFn(value) }; + } + return { + next: nextOpMap, + }; + } + return { + [Symbol.iterator]: opMapIterator, + }; + } + return opMapIterable; } export const opMap = (fn: (v: T) => U) => toPipeFn(opMapSync(fn), opMapAsync(fn)); diff --git a/packages/cspell-pipe/src/perf/gen.perf.ts b/packages/cspell-pipe/src/perf/gen.perf.ts new file mode 100644 index 00000000000..3f088524f48 --- /dev/null +++ b/packages/cspell-pipe/src/perf/gen.perf.ts @@ -0,0 +1,47 @@ +import { suite } from 'perf-insight'; + +suite('generators vs iterators', async (test) => { + const data = Array.from({ length: 10_000 }, (_, i) => i); + + function double(v: number) { + return v * 2; + } + + test('generator', () => { + return testIterable(genValues(data, double)); + }); + + test('iterator', () => { + return testIterable(iterValues(data, double)); + }); + + function testIterable(iter: Iterable) { + let sum = 0; + for (const v of iter) { + sum += v; + } + return sum; + } +}); + +function* genValues(i: Iterable, fnMap: (v: number) => number) { + for (const v of i) { + yield fnMap(v); + } +} + +function iterValues(i: Iterable, fnMap: (v: number) => number): Iterable { + return { + [Symbol.iterator]: () => { + const iter = i[Symbol.iterator](); + function next() { + const { done, value } = iter.next(); + if (done) return { done, value: undefined }; + return { value: fnMap(value) }; + } + return { + next, + }; + }, + }; +} diff --git a/packages/cspell-pipe/tsconfig.esm.json b/packages/cspell-pipe/tsconfig.esm.json deleted file mode 100644 index 459d7f9343e..00000000000 --- a/packages/cspell-pipe/tsconfig.esm.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/tsconfig", - "extends": "../../tsconfig.esm.json", - "compilerOptions": { - "composite": true, - "tsBuildInfoFile": "dist/compile.esm.tsbuildInfo", - "rootDir": "src", - "outDir": "dist", - "types": ["node"] - }, - "include": ["src"] -} diff --git a/packages/cspell-pipe/tsconfig.json b/packages/cspell-pipe/tsconfig.json index 635469e894c..a8f3aeb0024 100644 --- a/packages/cspell-pipe/tsconfig.json +++ b/packages/cspell-pipe/tsconfig.json @@ -1,4 +1,10 @@ { - "files": [], - "references": [{ "path": "./tsconfig.esm.json" }] + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "../../tsconfig.esm.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "types": ["node"] + }, + "include": ["src"] } diff --git a/packages/cspell-trie-lib/src/perf/has.perf.ts b/packages/cspell-trie-lib/src/perf/has.perf.ts index 1e7f28bc2f0..8c0b008f0f2 100644 --- a/packages/cspell-trie-lib/src/perf/has.perf.ts +++ b/packages/cspell-trie-lib/src/perf/has.perf.ts @@ -18,6 +18,11 @@ suite('trie has', async (test) => { const trieBlob = fastTrieBlob.toTrieBlob(); const iTrieFast = new ITrieImpl(fastTrieBlob); const iTrieBlob = new ITrieImpl(trieBlob); + const setOfWords = new Set(words); + + test('set has words', () => { + trieHasWords(setOfWords, words); + }); test('trie has words', () => { trieHasWords(trie, words); diff --git a/packages/cspell/cspell.json b/packages/cspell/cspell.json index 290108a9745..db76bd30220 100644 --- a/packages/cspell/cspell.json +++ b/packages/cspell/cspell.json @@ -9,6 +9,7 @@ "ignorePaths": [ "*.snap", "node_modules", + "__snapshots__", "package-lock.json", "coverage/**", "dist/**", diff --git a/packages/cspell/package.json b/packages/cspell/package.json index d444cbe91a5..9ad8c15d2b0 100644 --- a/packages/cspell/package.json +++ b/packages/cspell/package.json @@ -90,6 +90,7 @@ "chalk": "^5.3.0", "chalk-template": "^1.1.0", "commander": "^12.1.0", + "cspell-dictionary": "workspace:*", "cspell-gitignore": "workspace:*", "cspell-glob": "workspace:*", "cspell-io": "workspace:*", diff --git a/packages/cspell/src/app/__snapshots__/app.test.ts.snap b/packages/cspell/src/app/__snapshots__/app.test.ts.snap index 58a8ab94b91..416cab3f686 100644 --- a/packages/cspell/src/app/__snapshots__/app.test.ts.snap +++ b/packages/cspell/src/app/__snapshots__/app.test.ts.snap @@ -449,6 +449,7 @@ info info Exclusion Globs: info Glob: *.snap from ./cspell.json info Glob: node_modules from ./cspell.json +info Glob: __snapshots__ from ./cspell.json info Glob: package-lock.json from ./cspell.json info Glob: coverage/** from ./cspell.json info Glob: dist/** from ./cspell.json diff --git a/packages/cspell/src/app/environment.ts b/packages/cspell/src/app/environment.ts new file mode 100644 index 00000000000..601d6346c34 --- /dev/null +++ b/packages/cspell/src/app/environment.ts @@ -0,0 +1,58 @@ +export interface CSpellEnvironmentVariables { + /** + * Enable logging ALL dictionary requests. + * Acceptable values are: 'true', 'false', 't', 'f', 'on', 'off', 'yes', 'no', '1', '0' + */ + CSPELL_ENABLE_DICTIONARY_LOGGING?: string; + /** + * The path to the dictionary log file. + */ + CSPELL_ENABLE_DICTIONARY_LOG_FILE?: string; + /** + * A Csv list of fields to log. + * Fields: + * - time: the time the check was made in milliseconds + * - word: the word being checked + * - value: the result of the check + */ + CSPELL_ENABLE_DICTIONARY_LOG_FIELDS?: string; + CSPELL_GLOB_ROOT?: string; +} + +export type EnvironmentKeys = keyof CSpellEnvironmentVariables; + +type EnvironmentKeyNames = { + [K in EnvironmentKeys]: K; +}; + +export const environmentKeys: EnvironmentKeyNames = { + CSPELL_ENABLE_DICTIONARY_LOGGING: 'CSPELL_ENABLE_DICTIONARY_LOGGING', + CSPELL_ENABLE_DICTIONARY_LOG_FILE: 'CSPELL_ENABLE_DICTIONARY_LOG_FILE', + CSPELL_ENABLE_DICTIONARY_LOG_FIELDS: 'CSPELL_ENABLE_DICTIONARY_LOG_FIELDS', + CSPELL_GLOB_ROOT: 'CSPELL_GLOB_ROOT', +}; + +export function getEnvironmentVariables(): CSpellEnvironmentVariables { + return process.env as CSpellEnvironmentVariables; +} + +export function setEnvironmentVariable(key: K, value: CSpellEnvironmentVariables[K]): void { + process.env[key] = value; +} + +export function getEnvironmentVariable(key: K): CSpellEnvironmentVariables[K] | undefined { + return process.env[key]; +} + +export function truthy(value: string | undefined): boolean { + switch (value?.toLowerCase().trim()) { + case 't': + case 'true': + case 'on': + case 'yes': + case '1': { + return true; + } + } + return false; +} diff --git a/packages/cspell/src/app/lint/LintRequest.ts b/packages/cspell/src/app/lint/LintRequest.ts index 92acedd8d2f..df8159fc9f6 100644 --- a/packages/cspell/src/app/lint/LintRequest.ts +++ b/packages/cspell/src/app/lint/LintRequest.ts @@ -2,7 +2,7 @@ import * as path from 'node:path'; import type { Issue } from '@cspell/cspell-types'; -import type { LinterOptions } from '../options.js'; +import type { LinterCliOptions, LinterOptions } from '../options.js'; import type { GlobSrcInfo } from '../util/glob.js'; import { calcExcludeGlobInfo } from '../util/glob.js'; import type { FinalizedReporter } from '../util/reporters.js'; @@ -28,7 +28,7 @@ export class LintRequest { constructor( readonly fileGlobs: string[], - readonly options: LinterOptions & Deprecated, + readonly options: LinterCliOptions & Deprecated, readonly reporter: FinalizedReporter, ) { this.root = path.resolve(options.root || process.cwd()); diff --git a/packages/cspell/src/app/lint/__snapshots__/logging/dictionary-logging.csv b/packages/cspell/src/app/lint/__snapshots__/logging/dictionary-logging.csv new file mode 100644 index 00000000000..db3a83e9167 --- /dev/null +++ b/packages/cspell/src/app/lint/__snapshots__/logging/dictionary-logging.csv @@ -0,0 +1,1509 @@ +word, value +Book, true +Template, true +Version, true +This, true +template, true +been, true +downloaded, true +from, true +Original, true +author, true +Luis, true +Cobo, true +with, true +extensive, true +modifications, true +License, true +cspell, true +locale, true +words, true +Grimms, true +Gothel, true +DOCUMENT, true +CONFIGURATIONS, true +INFORMATION, true +Font, true +size, true +Include, true +file, true +that, true +specifies, true +document, true +structure, true +layout, true +Fairy, true +Tales, true +title, true +Author, true +edition, true +TITLE, true +PAGE, true +Suppress, true +page, true +numbering, true +background, true +image, true +first, true +argument, true +scaling, true +adjust, true +this, true +necessary, true +fits, true +entire, true +rectangle, true +rounded, true +corners, true +fill, true +white, true +opacity, true +anchor, true +south, true +west, true +minimum, true +width, true +height, true +White, true +adjusts, true +position, true +north, true +Text, true +wrapping, true +xshift, true +yshift, true +relative, true +Make, true +sure, true +following, true +content, true +TABLE, true +CONTENTS, true +Prints, true +table, true +contents, true +INTRODUCTION, true +SECTION, true +Introduction, true +Prologue, true +chapter, true +suppressed, true +finer, true +quotations, true +John, true +Smith, true +great, true +place, true +write, true +introduction, true +prologue, true +even, true +footnote, true +seem, true +smarter, true +BOOK, true +PART, true +CHAPTER, true +Little, true +Riding, true +Hood, true +Once, true +upon, true +time, true +there, true +dear, true +little, true +girl, true +loved, true +everyone, true +looked, true +most, true +grandmother, true +nothing, true +would, true +have, true +given, true +child, true +gave, true +velvet, true +which, true +suited, true +well, true +never, true +wear, true +anything, true +else, true +always, true +called, true +mother, true +said, true +Come, true +here, true +piece, true +cake, true +bottle, true +wine, true +take, true +them, true +your, true +weak, true +they, true +will, true +good, true +before, true +gets, true +when, true +going, true +walk, true +nicely, true +quietly, true +path, true +fall, true +break, true +then, true +into, true +room, true +don't, true +forget, true +Good, true +morning, true +peep, true +every, true +corner, true +care, true +hand, true +lived, true +wood, true +half, true +league, true +village, true +just, true +entered, true +wolf, true +know, true +what, true +wicked, true +creature, true +afraid, true +Thank, true +kindly, true +Whither, true +away, true +early, true +grandmother's, true +What, true +apron, true +Cake, true +yesterday, true +baking, true +poor, true +sick, true +something, true +make, true +stronger, true +Where, true +does, true +live, true +quarter, true +farther, true +house, true +stands, true +under, true +three, true +large, true +trees, true +below, true +surely, true +must, true +replied, true +thought, true +himself, true +tender, true +young, true +nice, true +plump, true +mouthful, true +better, true +than, true +woman, true +craftily, true +catch, true +both, true +walked, true +short, true +side, true +pretty, true +flowers, true +about, true +look, true +round, true +believe, true +hear, true +sweetly, true +birds, true +singing, true +gravely, true +along, true +were, true +school, true +while, true +everything, true +merry, true +raised, true +eyes, true +sunbeams, true +dancing, true +through, true +growing, true +everywhere, true +Suppose, true +fresh, true +nosegay, true +please, true +shall, true +still, true +whenever, true +picked, true +fancied, true +prettier, true +after, true +deeper, true +Meanwhile, true +straight, true +knocked, true +door, true +bringing, true +open, true +Lift, true +latch, true +cannot, true +lifted, true +sprang, true +without, true +saying, true +word, true +went, true +devoured, true +Then, true +clothes, true +dressed, true +laid, true +drew, true +curtains, true +however, true +running, true +picking, true +gathered, true +many, true +could, true +carry, true +more, true +remembered, true +surprised, true +find, true +cottage, true +standing, true +such, true +strange, true +feeling, true +herself, true +uneasy, true +feel, true +today, true +other, true +times, true +like, true +being, true +much, true +received, true +answer, true +back, true +There, true +pulled, true +over, true +face, true +looking, true +very, true +ears, true +reply, true +hands, true +terrible, true +mouth, true +scarcely, true +bound, true +swallowed, true +When, true +appeased, true +appetite, true +down, true +again, true +fell, true +asleep, true +began, true +snore, true +loud, true +huntsman, true +passing, true +snoring, true +wants, true +came, true +lying, true +sinner, true +long, true +sought, true +fire, true +occurred, true +might, true +saved, true +took, true +pair, true +scissors, true +stomach, true +sleeping, true +made, true +snips, true +shining, true +crying, true +frightened, true +dark, true +inside, true +aged, true +alive, true +also, true +able, true +breathe, true +quickly, true +fetched, true +stones, true +filled, true +wolf's, true +belly, true +awoke, true +wanted, true +heavy, true +collapsed, true +once, true +dead, true +delighted, true +skin, true +home, true +drank, true +brought, true +revived, true +myself, true +leave, true +forbidden, true +related, true +taking, true +cakes, true +another, true +spoke, true +tried, true +entice, true +guard, true +forward, true +told, true +public, true +road, true +certain, true +eaten, true +Well, true +shut, true +come, true +Soon, true +afterwards, true +cried, true +Open, true +some, true +speak, true +grey, true +beard, true +stole, true +twice, true +thrice, true +last, true +jumped, true +roof, true +intending, true +wait, true +until, true +evening, true +steal, true +devour, true +darkness, true +thoughts, true +front, true +stone, true +trough, true +Take, true +pail, true +sausages, true +water, true +boiled, true +carried, true +quite, true +full, true +smell, true +reached, true +sniffed, true +peeped, true +stretched, true +neck, true +longer, true +keep, true +footing, true +slip, true +slipped, true +drowned, true +joyously, true +ever, true +harm, true +Hansel, true +Gretel, true +Hard, true +forest, true +dwelt, true +cutter, true +wife, true +children, true +bite, true +dearth, true +land, true +procure, true +daily, true +bread, true +night, true +tossed, true +anxiety, true +groaned, true +become, true +feed, true +ourselves, true +I'll, true +tell, true +husband, true +answered, true +tomorrow, true +where, true +thickest, true +light, true +give, true +each, true +work, true +alone, true +They, true +bear, true +wild, true +animals, true +soon, true +tear, true +pieces, true +fool, true +four, true +hunger, true +plane, true +planks, true +coffins, true +left, true +peace, true +consented, true +sorry, true +same, true +sleep, true +heard, true +their, true +stepmother, true +father, true +wept, true +bitter, true +tears, true +quiet, true +distress, true +yourself, true +help, true +folks, true +fallen, true +coat, true +opened, true +crept, true +outside, true +moon, true +shone, true +brightly, true +pebbles, true +glittered, true +real, true +silver, true +pennies, true +stooped, true +stuffed, true +pocket, true +comforted, true +sister, true +forsake, true +dawned, true +risen, true +sluggards, true +fetch, true +dinner, true +together, true +stood, true +staying, true +behind, true +attention, true +legs, true +sitting, true +goodbye, true +Fool, true +chimneys, true +constantly, true +throwing, true +pebble, true +middle, true +pile, true +cold, true +brushwood, true +high, true +hill, true +lighted, true +flames, true +burning, true +yourselves, true +rest, true +done, true +noon, true +strokes, true +believed, true +near, true +branch, true +fastened, true +withered, true +tree, true +wind, true +blowing, true +backwards, true +forwards, true +closed, true +fatigue, true +fast, true +already, true +Just, true +followed, true +newly, true +coined, true +showed, true +whole, true +father's, true +naughty, true +slept, true +coming, true +rejoiced, true +heart, true +throughout, true +Everything, true +loaf, true +means, true +saving, true +man's, true +share, true +listen, true +scolded, true +reproached, true +says, true +likewise, true +yielded, true +second, true +awake, true +conversation, true +pick, true +locked, true +Nevertheless, true +Early, true +beds, true +Their, true +smaller, true +crumbled, true +often, true +threw, true +morsel, true +ground, true +stop, true +pigeon, true +chimney, true +crumbs, true +lives, true +tired, true +shared, true +scattered, true +passed, true +rises, true +strewn, true +show, true +found, true +thousands, true +woods, true +fields, true +next, true +till, true +hungry, true +berries, true +grew, true +weary, true +beneath, true +mornings, true +since, true +weariness, true +beautiful, true +snow, true +bird, true +bough, true +sang, true +delightfully, true +listened, true +song, true +spread, true +wings, true +flew, true +alighted, true +approached, true +built, true +covered, true +windows, true +clear, true +sugar, true +meal, true +window, true +taste, true +sweet, true +above, true +broke, true +tasted, true +leant, true +against, true +nibbled, true +panes, true +soft, true +voice, true +parlour, true +Nibble, true +nibble, true +gnaw, true +nibbling, true +heaven, true +born, true +eating, true +disturbing, true +themselves, true +liked, true +tore, true +pushed, true +pane, true +enjoyed, true +Suddenly, true +hills, true +supported, true +crutches, true +creeping, true +terribly, true +nodded, true +head, true +stay, true +happen, true +food, true +milk, true +pancakes, true +apples, true +nuts, true +Afterwards, true +clean, true +linen, true +only, true +pretended, true +kind, true +reality, true +witch, true +order, true +power, true +killed, true +cooked, true +feast, true +Witches, true +keen, true +scent, true +beasts, true +aware, true +human, true +beings, true +draw, true +neighborhood, true +laughed, true +malice, true +mockingly, true +escape, true +rosy, true +cheeks, true +muttered, true +That, true +dainty, true +seized, true +shrivelled, true +stable, true +grated, true +Scream, true +shook, true +lazy, true +thing, true +cook, true +brother, true +weep, true +bitterly, true +vain, true +forced, true +commanded, true +best, true +crab, true +shells, true +Every, true +stretch, true +finger, true +bone, true +Hansel's, true +astonished, true +fattening, true +weeks, true +gone, true +remained, true +thin, true +impatience, true +stir, true +bring, true +lean, true +kill, true +lament, true +flow, true +Dear, true +should, true +rate, true +died, true +noise, true +won't, true +hang, true +cauldron, true +bake, true +heated, true +oven, true +kneaded, true +dough, true +darting, true +Creep, true +properly, true +intended, true +mind, true +Silly, true +goose, true +enough, true +thrust, true +push, true +drove, true +iron, true +bolt, true +howl, true +horribly, true +godless, true +miserably, true +burnt, true +death, true +lightning, true +cage, true +rejoice, true +embrace, true +dance, true +kiss, true +need, true +fear, true +witch's, true +chests, true +pearls, true +jewels, true +These, true +pockets, true +whatever, true +pinafore, true +hours, true +cross, true +foot, true +plank, true +bridge, true +ferry, true +duck, true +swimming, true +dost, true +thou, true +waiting, true +thee, true +There's, true +sight, true +across, true +seated, true +safely, true +seemed, true +familiar, true +length, true +afar, true +rushed, true +known, true +happy, true +hour, true +emptied, true +precious, true +handful, true +perfect, true +happiness, true +tale, true +runs, true +mouse, true +whosoever, true +catches, true +THREE, true +Rapunzel, true +wished, true +hoped, true +grant, true +desire, true +people, true +splendid, true +garden, true +seen, true +herbs, true +surrounded, true +wall, true +dared, true +because, true +belonged, true +enchantress, true +dreaded, true +world, true +planted, true +rampion, true +rapunzel, true +green, true +longed, true +pined, true +pale, true +miserable, true +alarmed, true +asked, true +ails, true +can't, true +Sooner, true +cost, true +twilight, true +clambered, true +hastily, true +clutched, true +salad, true +greedily, true +descend, true +gloom, true +therefore, true +dare, true +angry, true +thief, true +suffer, true +mercy, true +justice, true +necessity, true +felt, true +longing, true +allowed, true +anger, true +softened, true +case, true +allow, true +condition, true +treated, true +terror, true +appeared, true +name, true +twelve, true +years, true +tower, true +neither, true +stairs, true +placed, true +hair, true +magnificent, true +fine, true +spun, true +gold, true +unfastened, true +braided, true +tresses, true +wound, true +hooks, true +twenty, true +ells, true +climbed, true +After, true +year, true +pass, true +king's, true +rode, true +charming, true +solitude, true +letting, true +resound, true +climb, true +none, true +deeply, true +touched, true +thus, true +braids, true +ladder, true +mounts, true +fortune, true +grow, true +Immediately, true +beheld, true +talk, true +friend, true +stirred, true +lost, true +handsome, true +love, true +Dame, true +willingly, true +Bring, true +skein, true +silk, true +weave, true +ready, true +horse, true +agreed, true +remarked, true +Tell, true +happens, true +heavier, true +moment, true +separated, true +deceived, true +Rapunzel's, true +wrapped, true +right, true +snip, true +snap, true +lovely, true +pitiless, true +desert, true +grief, true +misery, true +cast, true +hook, true +ascended, true +instead, true +finding, true +dearest, true +gazed, true +venomous, true +looks, true +sits, true +nest, true +scratch, true +beside, true +pain, true +despair, true +leapt, true +escaped, true +life, true +thorns, true +pierced, true +wandered, true +blind, true +roots, true +naught, true +loss, true +Thus, true +roamed, true +twins, true +birth, true +wretchedness, true +towards, true +knew, true +wetted, true +kingdom, true +joyfully, true +contented, true +Book, true +Structural, true +Definitions, true +File, true +Version, true +Created, true +This, true +file, true +been, true +downloaded, true +from, true +License, true +REQUIRED, true +PACKAGES, true +Required, true +inputting, true +international, true +characters, true +Output, true +font, true +encoding, true +Libertine, true +Improves, true +character, true +word, true +spacing, true +drawing, true +custom, true +shapes, true +Color, true +used, true +title, true +page, true +setting, true +background, true +images, true +meta, true +information, true +specification, true +PAPER, true +MARGIN, true +HEADER, true +FOOTER, true +SIZES, true +Paper, true +size, true +trims, true +Left, true +right, true +margins, true +bottom, true +Header, true +footer, true +height, true +Extra, true +header, true +space, true +FOOTNOTE, true +CUSTOMIZATION, true +Font, true +settings, true +footnotes, true +Space, true +between, true +footnote, true +number, true +text, true +multiple, true +same, true +Remove, true +rule, true +above, true +first, true +body, true +FORMATS, true +Define, true +style, true +width, true +Footer, true +Pages, true +chapters, true +defined, true +PART, true +FORMAT, true +Part, true +name, true +Whitespace, true +part, true +heading, true +CHAPTER, true +before, true +chapter, true +starts, true +Chapter, true +Tufte, false +Tufte, false +Tufte, false +SECTION, true +Section, true +section, true +after, true +SUBSECTION, true +Subsection, true +subsection, true +SUBSUBSECTION, true +Subsubsection, true +subsubsection, true +CAPTION, true +Caption, true +QUOTATION, true +ENVIRONMENT, true +QUOTE, true +MISCELLANEOUS, true +DOCUMENT, true +SPECIFICATIONS, true +Paragraph, true +indentation, true +Fewer, true +overfull, true +lines, true +memoir, true +class, true +allows, true +somewhere, true +Tell, true +implement, true +cspell, true +words, true +Leslie, true +Lamport, true +document, true +preparation, true +system, true +typesetting, true +program, true +offers, true +programmable, true +desktop, true +publishing, true +features, true +extensive, true +facilities, true +automating, true +most, true +aspects, true +including, true +numbering, true +cross, true +referencing, true +tables, true +figures, true +page, true +layout, true +bibliographies, true +much, true +more, true +originally, true +written, true +become, true +dominant, true +method, true +using, true +people, true +write, true +plain, true +anymore, true +current, true +version, true +This, true +comment, true +shown, true +final, true +output, true +following, true +shows, true +power, true +This, true +sample, true +input, true +file, true +Version, true +August, true +character, true +causes, true +ignore, true +remaining, true +text, true +line, true +used, true +comments, true +like, true +this, true +Specifies, true +document, true +class, true +preamble, true +begins, true +here, true +Example, true +Document, true +Declares, true +document's, true +title, true +author's, true +name, true +Deleting, true +command, true +produces, true +today's, true +date, true +Defines, true +mean, true +alternative, true +definition, true +that, true +commented, true +beginning, true +Produces, true +example, true +Comparing, true +with, true +output, true +generates, true +show, true +produce, true +simple, true +your, true +Ordinary, true +Text, true +section, true +heading, true +Lower, true +level, true +sections, true +begun, true +similar, true +commands, true +ends, true +words, true +sentences, true +marked, true +spaces, true +doesn't, true +matter, true +many, true +type, true +good, true +counts, true +space, true +more, true +blank, true +lines, true +denote, true +paragraph, true +Since, true +number, true +consecutive, true +treated, true +single, true +formatting, true +makes, true +difference, true +logo, true +When, true +making, true +easy, true +read, true +possible, true +will, true +great, true +help, true +write, true +when, true +change, true +shows, true +Because, true +printing, true +different, true +from, true +typewriting, true +there, true +things, true +have, true +differently, true +preparing, true +than, true +were, true +just, true +typing, true +directly, true +Quotation, true +marks, true +handled, true +specially, true +quotes, true +within, true +separates, true +double, true +quote, true +what, true +wrote, true +Dashes, true +come, true +three, true +sizes, true +intra, true +word, true +dash, true +medium, true +ranges, true +punctuation, true +sentence, true +ending, true +should, true +larger, true +between, true +sometimes, true +special, true +conjunction, true +characters, true +right, true +following, true +Gnats, true +gnus, true +inter, true +begin, true +check, true +after, true +periods, true +reading, true +make, true +sure, true +haven't, true +forgotten, true +cases, true +Generating, true +ellipsis, true +needed, true +because, true +ignores, true +names, true +made, true +letters, true +Note, true +these, true +start, true +spacing, true +around, true +requires, true +interprets, true +some, true +common, true +must, true +generate, true +them, true +These, true +include, true +usually, true +emphasized, true +italic, true +style, true +long, true +segment, true +also, true +such, true +given, true +additional, true +emphasis, true +necessary, true +prevent, true +breaking, true +where, true +might, true +otherwise, true +Jones, true +unbreakable, true +interword, true +especially, true +symbol, true +little, true +sense, true +hyphenated, true +across, true +Footnotes, true +footnote, true +pose, true +problem, true +typesetting, true +mathematical, true +formulas, true +formula, true +ignored, true +Remember, true +letter, true +equivalent, true +denotes, true +typed, true +Displayed, true +displayed, true +indenting, true +left, true +margin, true +Quotations, true +commonly, true +There, true +short, true +quotations, true +quotation, true +consists, true +formatted, true +longer, true +ones, true +paragraphs, true +neither, true +which, true +particularly, true +interesting, true +second, true +dull, true +first, true +Another, true +frequently, true +structure, true +list, true +itemized, true +item, true +Each, true +tick, true +don't, true +worry, true +about, true +kind, true +mark, true +contains, true +another, true +nested, true +inside, true +inner, true +enumerated, true +allows, true +nest, true +lists, true +deeper, true +really, true +rest, true +outer, true +other, true +part, true +third, true +even, true +display, true +poetry, true +environment, true +verse, true +Whose, true +features, true +poets, true +stanza, true +curse, true +separate, true +stanzas, true +instead, true +Them, true +they'd, true +rather, true +forced, true +terse, true +Mathematical, true +multiline, true +require, true +instructions, true +Don't, true +equation, true +itself, true \ No newline at end of file diff --git a/packages/cspell/src/app/lint/lint.test.ts b/packages/cspell/src/app/lint/lint.test.ts index 9ca1cdbe7b6..c9441f90e29 100644 --- a/packages/cspell/src/app/lint/lint.test.ts +++ b/packages/cspell/src/app/lint/lint.test.ts @@ -1,8 +1,9 @@ import * as path from 'node:path'; -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; import { CheckFailed } from '../app.mjs'; +import { environmentKeys } from '../environment.js'; import { pathPackageRoot } from '../test/test.helper.js'; import { InMemoryReporter } from '../util/InMemoryReporter.js'; import { runLint } from './lint.js'; @@ -78,9 +79,40 @@ describe('Linter Validation Tests', () => { expect(runResult).toEqual(expectedRunResult); expect(runResult).toEqual(reporter.runResult); }); + + test.each` + files | options + ${[]} | ${{ root: latexSamples }} + `('runLint $files $options', async ({ files, options }) => { + const reporter = new InMemoryReporter(); + process.env[environmentKeys.CSPELL_ENABLE_DICTIONARY_LOGGING] = 'true'; + process.env[environmentKeys.CSPELL_ENABLE_DICTIONARY_LOG_FILE] = 'stdout'; + process.env[environmentKeys.CSPELL_ENABLE_DICTIONARY_LOG_FIELDS] = 'word, value'; + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(mockWrite); + const runResult = await runLint(new LintRequest(files, { ...options }, reporter)); + expect(runResult).toEqual(reporter.runResult); + expect(stdout).toHaveBeenCalledOnce(); + expect(stdout.mock.calls.map(([data]) => data).join('')).toMatchFileSnapshot( + './__snapshots__/logging/dictionary-logging.csv', + ); + }); }); function report(reporter: InMemoryReporter) { const { issues, errorCount, errors } = reporter; return { issues, errorCount, errors }; } + +function mockWrite(buffer: Uint8Array | string, cb?: (err?: Error) => void): boolean; +function mockWrite(str: Uint8Array | string, encoding?: BufferEncoding, cb?: (err?: Error) => void): boolean; +function mockWrite( + _data: unknown, + encodingOrCb?: BufferEncoding | ((err?: Error) => void), + cb?: (err?: Error) => void, +) { + if (typeof encodingOrCb === 'function') { + cb = encodingOrCb; + } + cb?.(); + return true; +} diff --git a/packages/cspell/src/app/lint/lint.ts b/packages/cspell/src/app/lint/lint.ts index 6ce44a9af57..746c1429fd5 100644 --- a/packages/cspell/src/app/lint/lint.ts +++ b/packages/cspell/src/app/lint/lint.ts @@ -7,6 +7,7 @@ import { opMap, pipe } from '@cspell/cspell-pipe/sync'; import type { CSpellSettings, Glob, Issue, RunResult, TextDocumentOffset, TextOffset } from '@cspell/cspell-types'; import { MessageTypes } from '@cspell/cspell-types'; import chalk from 'chalk'; +import { _debug as cspellDictionaryDebug } from 'cspell-dictionary'; import { findRepoRoot, GitIgnore } from 'cspell-gitignore'; import { GlobMatcher, type GlobMatchOptions, type GlobPatternNormalized, type GlobPatternWithRoot } from 'cspell-glob'; import type { Logger, SpellCheckFileResult, ValidationIssue } from 'cspell-lib'; @@ -24,6 +25,7 @@ import { } from 'cspell-lib'; import { console } from '../console.js'; +import { getEnvironmentVariable, setEnvironmentVariable, truthy } from '../environment.js'; import { getFeatureFlags } from '../featureFlags/index.js'; import { CSpellReporterConfiguration } from '../models.js'; import { npmPackage } from '../pkgInfo.js'; @@ -56,6 +58,7 @@ import type { FinalizedReporter } from '../util/reporters.js'; import { loadReporters, mergeReporters } from '../util/reporters.js'; import { getTimeMeasurer } from '../util/timer.js'; import * as util from '../util/util.js'; +import { writeFileOrStream } from '../util/writeFile.js'; import type { LintRequest } from './LintRequest.js'; const version = npmPackage.version; @@ -73,7 +76,17 @@ export async function runLint(cfg: LintRequest): Promise { const timer = getTimeMeasurer(); + const logDictRequests = truthy(getEnvironmentVariable('CSPELL_ENABLE_DICTIONARY_LOGGING')); + if (logDictRequests) { + cspellDictionaryDebug.cacheDictionaryEnableLogging(true); + } + const lintResult = await run(); + + if (logDictRequests) { + await writeDictionaryLog(); + } + await reporter.result(lintResult); const elapsed = timer(); if (getFeatureFlags().getFlag('timer')) { @@ -405,7 +418,7 @@ export async function runLint(cfg: LintRequest): Promise { async function run(): Promise { if (cfg.options.root) { - process.env[ENV_CSPELL_GLOB_ROOT] = cfg.root; + setEnvironmentVariable(ENV_CSPELL_GLOB_ROOT, cfg.options.root); } const configInfo: ConfigInfo = await readConfig(cfg.configFile, cfg.root); @@ -714,3 +727,17 @@ async function* concatAsyncIterables( yield* iter; } } + +async function writeDictionaryLog() { + const fieldsCsv = getEnvironmentVariable('CSPELL_ENABLE_DICTIONARY_LOG_FIELDS') || 'time, word, value'; + const fields = fieldsCsv.split(',').map((f) => f.trim()); + const header = fields.join(', ') + '\n'; + const lines = cspellDictionaryDebug + .cacheDictionaryGetLog() + .filter((d) => d.method === 'has') + .map((d) => fields.map((f) => (f in d ? `${d[f as keyof typeof d]}` : '')).join(', ')); + const data = header + lines.join('\n') + '\n'; + const filename = getEnvironmentVariable('CSPELL_ENABLE_DICTIONARY_LOG_FILE') || 'cspell-dictionary-log.csv'; + + await writeFileOrStream(filename, data); +} diff --git a/packages/cspell/src/app/util/writeFile.test.ts b/packages/cspell/src/app/util/writeFile.test.ts new file mode 100644 index 00000000000..e50983e6356 --- /dev/null +++ b/packages/cspell/src/app/util/writeFile.test.ts @@ -0,0 +1,85 @@ +import fs from 'node:fs/promises'; + +import { afterEach, describe, expect, test, vi } from 'vitest'; + +import { writeFileOrStream, writeStream } from './writeFile.js'; + +describe('writeFile', () => { + afterEach(() => { + vi.resetAllMocks(); + }); + test('writeFileOrStream stdout', async () => { + const s = vi.spyOn(process.stdout, 'write').mockImplementation(mockWrite); + await writeFileOrStream('stdout', 'test'); + expect(s).toHaveBeenCalledTimes(1); + expect(s).toHaveBeenCalledWith('test', expect.any(Function)); + }); + + test('writeFileOrStream stderr', async () => { + const s = vi.spyOn(process.stderr, 'write').mockImplementation(mockWrite); + await writeFileOrStream('stderr', 'test'); + expect(s).toHaveBeenCalledTimes(1); + expect(s).toHaveBeenCalledWith('test', expect.any(Function)); + }); + + test('writeFileOrStream null', async () => { + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(mockWrite); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(mockWrite); + const fsWriteFile = vi.spyOn(fs, 'writeFile'); + await writeFileOrStream('null', 'test'); + expect(stdout).toHaveBeenCalledTimes(0); + expect(stderr).toHaveBeenCalledTimes(0); + expect(fsWriteFile).toHaveBeenCalledTimes(0); + }); + + test('writeFileOrStream file', async () => { + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(mockWrite); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(mockWrite); + const fsWriteFile = vi.spyOn(fs, 'writeFile').mockResolvedValue(undefined); + await writeFileOrStream('myfile.log', 'test'); + expect(stdout).toHaveBeenCalledTimes(0); + expect(stderr).toHaveBeenCalledTimes(0); + expect(fsWriteFile).toHaveBeenCalledOnce(); + expect(fsWriteFile).toHaveBeenCalledWith('myfile.log', 'test'); + }); + + test('writeStream', async () => { + const s = vi.spyOn(process.stdout, 'write').mockImplementation(mockWrite); + await writeStream(process.stdout, 'test'); + expect(s).toHaveBeenCalledTimes(1); + }); + + test('writeStream with Error', async () => { + vi.spyOn(process.stdout, 'write').mockImplementation(mockWriteWithError); + await expect(writeStream(process.stdout, 'test')).rejects.toThrowError(); + }); +}); + +function mockWrite(buffer: Uint8Array | string, cb?: (err?: Error) => void): boolean; +function mockWrite(str: Uint8Array | string, encoding?: BufferEncoding, cb?: (err?: Error) => void): boolean; +function mockWrite( + _data: unknown, + encodingOrCb?: BufferEncoding | ((err?: Error) => void), + cb?: (err?: Error) => void, +) { + if (typeof encodingOrCb === 'function') { + cb = encodingOrCb; + } + cb?.(); + return true; +} + +function mockWriteWithError(buffer: Uint8Array | string, cb?: (err?: Error) => void): boolean; +function mockWriteWithError(str: Uint8Array | string, encoding?: BufferEncoding, cb?: (err?: Error) => void): boolean; +function mockWriteWithError( + _data: unknown, + encodingOrCb?: BufferEncoding | ((err?: Error) => void), + cb?: (err?: Error) => void, +) { + const err = new Error('mock error'); + if (typeof encodingOrCb === 'function') { + cb = encodingOrCb; + } + cb?.(err); + return true; +} diff --git a/packages/cspell/src/app/util/writeFile.ts b/packages/cspell/src/app/util/writeFile.ts new file mode 100644 index 00000000000..1e3be90e02d --- /dev/null +++ b/packages/cspell/src/app/util/writeFile.ts @@ -0,0 +1,30 @@ +import fs from 'node:fs/promises'; + +export async function writeFileOrStream(filename: string, data: string) { + switch (filename) { + case 'stdout': { + await writeStream(process.stdout, data); + return; + } + case 'stderr': { + await writeStream(process.stderr, data); + return; + } + case 'null': { + return; + } + } + return fs.writeFile(filename, data); +} + +export function writeStream(stream: NodeJS.WriteStream, data: string) { + return new Promise((resolve, reject) => { + stream.write(data, (err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1f6aef1beb0..52b15d369c8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -64,10 +64,10 @@ importers: version: 9.8.0 eslint-import-resolver-typescript: specifier: ^3.6.1 - version: 3.6.1(eslint-plugin-import@2.29.1)(eslint@9.8.0) + version: 3.6.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-plugin-import@2.29.1)(eslint@9.8.0) eslint-plugin-jest: specifier: ^28.6.0 - version: 28.6.0(@typescript-eslint/eslint-plugin@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint@9.8.0)(jest@29.7.0(@types/node@18.19.42)(ts-node@10.9.2(@types/node@18.19.42)(typescript@5.5.4)))(typescript@5.5.4) + version: 28.6.0(@typescript-eslint/eslint-plugin@7.18.0(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint@9.8.0)(typescript@5.5.4))(eslint@9.8.0)(jest@29.7.0(@types/node@18.19.42)(ts-node@10.9.2(@types/node@18.19.42)(typescript@5.5.4)))(typescript@5.5.4) eslint-plugin-n: specifier: ^17.10.1 version: 17.10.1(eslint@9.8.0) @@ -219,6 +219,9 @@ importers: commander: specifier: ^12.1.0 version: 12.1.0 + cspell-dictionary: + specifier: workspace:* + version: link:../cspell-dictionary cspell-gitignore: specifier: workspace:* version: link:../cspell-gitignore @@ -752,6 +755,9 @@ importers: globby: specifier: ^14.0.2 version: 14.0.2 + perf-insight: + specifier: ^1.2.0 + version: 1.2.0 packages/cspell-resolver: dependencies: @@ -14952,13 +14958,13 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.6.1(eslint-plugin-import@2.29.1)(eslint@9.8.0): + eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-plugin-import@2.29.1)(eslint@9.8.0): dependencies: debug: 4.3.6(supports-color@8.1.1) enhanced-resolve: 5.17.1 eslint: 9.8.0 - eslint-module-utils: 2.8.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(eslint-plugin-import@2.29.1)(eslint@9.8.0))(eslint@9.8.0) - eslint-plugin-import: 2.29.1(eslint-import-resolver-typescript@3.6.1)(eslint@9.8.0) + eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-plugin-import@2.29.1)(eslint@9.8.0))(eslint@9.8.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@9.8.0) fast-glob: 3.3.2 get-tsconfig: 4.7.6 is-core-module: 2.15.0 @@ -14990,13 +14996,14 @@ snapshots: - bluebird - supports-color - eslint-module-utils@2.8.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(eslint-plugin-import@2.29.1)(eslint@9.8.0))(eslint@9.8.0): + eslint-module-utils@2.8.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-plugin-import@2.29.1)(eslint@9.8.0))(eslint@9.8.0): dependencies: debug: 3.2.7 optionalDependencies: + '@typescript-eslint/parser': 7.18.0(eslint@9.8.0)(typescript@5.5.4) eslint: 9.8.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.1(eslint-plugin-import@2.29.1)(eslint@9.8.0) + eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-plugin-import@2.29.1)(eslint@9.8.0) transitivePeerDependencies: - supports-color @@ -15014,7 +15021,7 @@ snapshots: eslint: 9.8.0 eslint-compat-utils: 0.5.1(eslint@9.8.0) - eslint-plugin-import@2.29.1(eslint-import-resolver-typescript@3.6.1)(eslint@9.8.0): + eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@9.8.0): dependencies: array-includes: 3.1.8 array.prototype.findlastindex: 1.2.5 @@ -15024,7 +15031,7 @@ snapshots: doctrine: 2.1.0 eslint: 9.8.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(eslint-plugin-import@2.29.1)(eslint@9.8.0))(eslint@9.8.0) + eslint-module-utils: 2.8.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint-plugin-import@2.29.1)(eslint@9.8.0))(eslint@9.8.0) hasown: 2.0.2 is-core-module: 2.15.0 is-glob: 4.0.3 @@ -15034,12 +15041,14 @@ snapshots: object.values: 1.2.0 semver: 6.3.1 tsconfig-paths: 3.15.0 + optionalDependencies: + '@typescript-eslint/parser': 7.18.0(eslint@9.8.0)(typescript@5.5.4) transitivePeerDependencies: - eslint-import-resolver-typescript - eslint-import-resolver-webpack - supports-color - eslint-plugin-jest@28.6.0(@typescript-eslint/eslint-plugin@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint@9.8.0)(jest@29.7.0(@types/node@18.19.42)(ts-node@10.9.2(@types/node@18.19.42)(typescript@5.5.4)))(typescript@5.5.4): + eslint-plugin-jest@28.6.0(@typescript-eslint/eslint-plugin@7.18.0(@typescript-eslint/parser@7.18.0(eslint@9.8.0)(typescript@5.5.4))(eslint@9.8.0)(typescript@5.5.4))(eslint@9.8.0)(jest@29.7.0(@types/node@18.19.42)(ts-node@10.9.2(@types/node@18.19.42)(typescript@5.5.4)))(typescript@5.5.4): dependencies: '@typescript-eslint/utils': 7.18.0(eslint@9.8.0)(typescript@5.5.4) eslint: 9.8.0