refactor: be able to record dictionary requests.

integration-tests/src/sh.ts

@@ -24,7 +24,7 @@ export function execAsync(command: string, options: ExecOptions = {}): Promise<S
     return new Promise<Shell.ExecOutputReturnValue>((resolve) => {
         Shell.exec(
             command /* lgtm[js/shell-command-injection-from-environment] */,
-            { silent: !echo, fatal: bail },
+            { silent: !echo, fatal: bail, env: { ...process.env } },
             (code, stdout, stderr) => resolve({ code, stdout, stderr }),
         );
     });

packages/cspell-dictionary/src/SpellingDictionary/CachingDictionary.ts

@@ -17,6 +17,9 @@ let dictionaryCounter = 0;
 
 const DefaultAutoCacheSize = 1000;
 
+let logRequests = false;
+const log: LogEntry[] = [];
+
 /**
  * Caching Dictionary remembers method calls to increase performance.
  */
@@ -30,6 +33,22 @@ export interface CachingDictionary {
     getPreferredSuggestions(word: string): PreferredSuggestion[] | undefined;
 }
 
+interface LogEntryBase extends SearchOptions {
+    time: number;
+    method: 'has';
+    word: string;
+    value?: unknown;
+}
+
+interface LogEntryHas extends LogEntryBase {
+    method: 'has';
+    value: boolean;
+}
+
+const startTime = performance.now();
+
+export type LogEntry = LogEntryHas;
+
 class CachedDict implements CachingDictionary {
     readonly name: string;
     readonly id = ++dictionaryCounter;
@@ -41,7 +60,17 @@ class CachedDict implements CachingDictionary {
         // console.log(`CachedDict for ${this.name}`);
     }
 
-    readonly has = autoCache((word: string) => this.dict.has(word, this.options), DefaultAutoCacheSize);
+    has = (word: string): boolean => {
+        if (logRequests) {
+            const time = performance.now() - startTime;
+            const value = this.#has(word);
+            log.push({ time, method: 'has', word, value });
+            return value;
+        }
+        return this.#has(word);
+    };
+
+    #has = autoCache((word: string) => this.dict.has(word, this.options), DefaultAutoCacheSize);
     readonly isNoSuggestWord = autoCache(
         (word: string) => this.dict.isNoSuggestWord(word, this.options),
         DefaultAutoCacheSize,
@@ -56,7 +85,7 @@ class CachedDict implements CachingDictionary {
         return {
             name: this.name,
             id: this.id,
-            has: extractStats(this.has),
+            has: extractStats(this.#has),
             isNoSuggestWord: extractStats(this.isNoSuggestWord),
             isForbidden: extractStats(this.isForbidden),
             getPreferredSuggestions: extractStats(this.getPreferredSuggestions),
@@ -90,3 +119,11 @@ export function createCachingDictionary(
     knownOptions.set(dict, cached);
     return cached;
 }
+
+export function enableLogging(enabled = !logRequests): void {
+    logRequests = enabled;
+}
+
+export function getLog(): LogEntryBase[] {
+    return log;
+}

packages/cspell-dictionary/src/index.ts

@@ -1,3 +1,7 @@
+import {
+    enableLogging as cacheDictionaryEnableLogging,
+    getLog as cacheDictionaryGetLog,
+} from './SpellingDictionary/CachingDictionary.js';
 export type {
     CachingDictionary,
     FindOptions,
@@ -24,3 +28,11 @@ export {
     createSuggestDictionary,
     createSuggestOptions,
 } from './SpellingDictionary/index.js';
+
+/**
+ * Debugging utilities.
+ */
+export const _debug = {
+    cacheDictionaryEnableLogging,
+    cacheDictionaryGetLog,
+};

packages/cspell-dictionary/src/perf/has.perf.ts

@@ -4,22 +4,34 @@ import { buildITrieFromWords } from 'cspell-trie-lib';
 import { loremIpsum } from 'lorem-ipsum';
 import { suite } from 'perf-insight';
 
+import { createCachingDictionary } from '../SpellingDictionary/CachingDictionary.js';
 import { createSpellingDictionary } from '../SpellingDictionary/createSpellingDictionary.js';
 import { createCollection } from '../SpellingDictionary/SpellingDictionaryCollection.js';
 
 suite('dictionary has', async (test) => {
-    const words = genWords(10_000);
+    const words1 = genWords(10_000);
     const words2 = genWords(1000);
     const words3 = genWords(1000);
 
-    const iTrie = buildITrieFromWords(words);
-    const dict = createSpellingDictionary(words, 'test', import.meta.url);
+    const words = words1;
+
+    const iTrie = buildITrieFromWords(words1);
+    const dict = createSpellingDictionary(words1, 'test', import.meta.url);
     const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
     const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
 
     const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
     const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse');
 
+    const cacheDictSingle = createCachingDictionary(dict, {});
+    const cacheDictCol = createCachingDictionary(dictCol, {});
+
+    const dictSet = new Set(words);
+
+    test('Set has 100k words', () => {
+        checkWords(dictSet, words);
+    });
+
     test('dictionary has 100k words', () => {
         checkWords(dict, words);
     });
@@ -32,6 +44,14 @@ suite('dictionary has', async (test) => {
         checkWords(dictColRev, words);
     });
 
+    test('cache dictionary has 100k words', () => {
+        checkWords(cacheDictSingle, words);
+    });
+
+    test('cache collection has 100k words', () => {
+        checkWords(cacheDictCol, words);
+    });
+
     test('iTrie has 100k words', () => {
         checkWords(iTrie, words);
     });
@@ -58,6 +78,12 @@ suite('dictionary has Not', async (test) => {
     const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
     const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
 
+    const dictSet = new Set(words);
+
+    test('Set has not 100k words', () => {
+        checkWords(dictSet, missingWords, false);
+    });
+
     test('dictionary has not 100k words', () => {
         checkWords(dict, missingWords, false);
     });
@@ -80,6 +106,65 @@ suite('dictionary has Not', async (test) => {
     });
 });
 
+suite('dictionary has sampling', async (test) => {
+    const words1 = genWords(10_000);
+    const words2 = genWords(1000);
+    const words3 = genWords(1000);
+
+    const sampleIdx = genSamples(100_000, words1.length);
+    const wordsSample = sampleIdx.map((i) => words1[i]);
+
+    const iTrie = buildITrieFromWords(words1);
+    const dict = createSpellingDictionary(words1, 'test', import.meta.url);
+    const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
+    const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
+
+    const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
+    const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse');
+
+    const cacheDictSingle = createCachingDictionary(dict, {});
+    const cacheDictCol = createCachingDictionary(dictCol, {});
+
+    const dictSet = new Set(words1);
+
+    test('Set has 100k words', () => {
+        checkWords(dictSet, wordsSample);
+    });
+
+    test('dictionary has 100k words', () => {
+        checkWords(dict, wordsSample);
+    });
+
+    test('collection has 100k words', () => {
+        checkWords(dictCol, wordsSample);
+    });
+
+    test('collection reverse has 100k words', () => {
+        checkWords(dictColRev, wordsSample);
+    });
+
+    test('cache dictionary has 100k words', () => {
+        checkWords(cacheDictSingle, wordsSample);
+    });
+
+    test('cache collection has 100k words', () => {
+        checkWords(cacheDictCol, wordsSample);
+    });
+
+    test('iTrie has 100k words', () => {
+        checkWords(iTrie, wordsSample);
+    });
+
+    test('iTrie.hasWord has 100k words', () => {
+        const dict = { has: (word: string) => iTrie.hasWord(word, true) };
+        checkWords(dict, wordsSample);
+    });
+
+    test('iTrie.data has 100k words', () => {
+        checkWords(iTrie.data, wordsSample);
+    });
+});
+
 function checkWords(dict: { has: (word: string) => boolean }, words: string[], expected = true, totalChecks = 100_000) {
     let has = true;
     const len = words.length;
@@ -122,3 +207,15 @@ function genWords(count: number, includeForbidden = true): string[] {
 
     return [...setOfWords];
 }
+
+function genSamples(count: number, max: number, depth = 3) {
+    const r = Array<number>(count);
+    for (let j = 0; j < count; ++j) {
+        let n = Math.random() * max;
+        for (let i = 1; i < depth; ++i) {
+            n = Math.random() * n;
+        }
+        r[j] = Math.floor(n);
+    }
+    return r;
+}

packages/cspell-trie-lib/src/perf/has.perf.ts

@@ -18,6 +18,11 @@ suite('trie has', async (test) => {
     const trieBlob = fastTrieBlob.toTrieBlob();
     const iTrieFast = new ITrieImpl(fastTrieBlob);
     const iTrieBlob = new ITrieImpl(trieBlob);
+    const setOfWords = new Set(words);
+
+    test('set has words', () => {
+        trieHasWords(setOfWords, words);
+    });
 
     test('trie has words', () => {
         trieHasWords(trie, words);

packages/cspell/package.json

@@ -90,6 +90,7 @@
     "chalk": "^5.3.0",
     "chalk-template": "^1.1.0",
     "commander": "^12.1.0",
+    "cspell-dictionary": "workspace:*",
     "cspell-gitignore": "workspace:*",
     "cspell-glob": "workspace:*",
     "cspell-io": "workspace:*",

packages/cspell/src/app/environment.ts

@@ -0,0 +1,58 @@
+export interface CSpellEnvironmentVariables {
+    /**
+     * Enable logging ALL dictionary requests.
+     * Acceptable values are: 'true', 'false', 't', 'f', 'on', 'off', 'yes', 'no', '1', '0'
+     */
+    CSPELL_ENABLE_DICTIONARY_LOGGING?: string;
+    /**
+     * The path to the dictionary log file.
+     */
+    CSPELL_ENABLE_DICTIONARY_LOG_FILE?: string;
+    /**
+     * A Csv list of fields to log.
+     * Fields:
+     * - time: the time the check was made in milliseconds
+     * - word: the word being checked
+     * - value: the result of the check
+     */
+    CSPELL_ENABLE_DICTIONARY_LOG_FIELDS?: string;
+    CSPELL_GLOB_ROOT?: string;
+}
+
+export type EnvironmentKeys = keyof CSpellEnvironmentVariables;
+
+type EnvironmentKeyNames = {
+    [K in EnvironmentKeys]: K;
+};
+
+export const environmentKeys: EnvironmentKeyNames = {
+    CSPELL_ENABLE_DICTIONARY_LOGGING: 'CSPELL_ENABLE_DICTIONARY_LOGGING',
+    CSPELL_ENABLE_DICTIONARY_LOG_FILE: 'CSPELL_ENABLE_DICTIONARY_LOG_FILE',
+    CSPELL_ENABLE_DICTIONARY_LOG_FIELDS: 'CSPELL_ENABLE_DICTIONARY_LOG_FIELDS',
+    CSPELL_GLOB_ROOT: 'CSPELL_GLOB_ROOT',
+};
+
+export function getEnvironmentVariables(): CSpellEnvironmentVariables {
+    return process.env as CSpellEnvironmentVariables;
+}
+
+export function setEnvironmentVariable<K extends EnvironmentKeys>(key: K, value: CSpellEnvironmentVariables[K]): void {
+    process.env[key] = value;
+}
+
+export function getEnvironmentVariable<K extends EnvironmentKeys>(key: K): CSpellEnvironmentVariables[K] | undefined {
+    return process.env[key];
+}
+
+export function truthy(value: string | undefined): boolean {
+    switch (value?.toLowerCase().trim()) {
+        case 't':
+        case 'true':
+        case 'on':
+        case 'yes':
+        case '1': {
+            return true;
+        }
+    }
+    return false;
+}

packages/cspell/src/app/lint/LintRequest.ts

@@ -2,7 +2,7 @@ import * as path from 'node:path';
 
 import type { Issue } from '@cspell/cspell-types';
 
-import type { LinterOptions } from '../options.js';
+import type { LinterCliOptions, LinterOptions } from '../options.js';
 import type { GlobSrcInfo } from '../util/glob.js';
 import { calcExcludeGlobInfo } from '../util/glob.js';
 import type { FinalizedReporter } from '../util/reporters.js';
@@ -28,7 +28,7 @@ export class LintRequest {
 
     constructor(
         readonly fileGlobs: string[],
-        readonly options: LinterOptions & Deprecated,
+        readonly options: LinterCliOptions & Deprecated,
         readonly reporter: FinalizedReporter,
     ) {
         this.root = path.resolve(options.root || process.cwd());