Make maximum parse length configurable (mark-wiemer-org#337)

package.json

@@ -126,6 +126,11 @@
                     "default": "C:/Program Files/AutoHotkey/AutoHotkey.chm",
                     "description": "Path to the AHK Help document."
                 },
+                "ahk++.file.maximumParseLength": {
+                    "type": "number",
+                    "default": 10000,
+                    "description": "Number of lines to parse for each AHK file. Larger numbers may impact performance. Parsing is used by IntelliSense only, not AHK itself. Changing this value will not affect the behavior of AHK scripts.\n-1: Unlimited parsing.\n0: No parsing."
+                },
                 "ahk++.file.templateSnippetName": {
                     "type": "string",
                     "default": "AhkTemplate",
@@ -135,7 +140,7 @@
                     "type": "integer",
                     "default": 1,
                     "minimum": -1,
-                    "description": "Allowed number of empty lines.\n-1: ignore empty lines.\n0: no empty lines."
+                    "description": "Allowed number of empty lines.\n-1: Ignore empty lines.\n0: No empty lines."
                 },
                 "ahk++.formatter.indentCodeAfterLabel": {
                     "type": "boolean",

src/common/global.ts

@@ -35,6 +35,7 @@ export enum ConfigKey {
     helpPath = 'file.helpPath',
     indentCodeAfterLabel = 'formatter.indentCodeAfterLabel',
     indentCodeAfterSharpDirective = 'formatter.indentCodeAfterSharpDirective',
+    maximumParseLength = 'file.maximumParseLength',
     preserveIndent = 'formatter.preserveIndent',
     templateSnippetName = 'file.templateSnippetName',
     trimExtraSpaces = 'formatter.trimExtraSpaces',

src/parser/parser.ts

@@ -1,9 +1,17 @@
+import { ConfigKey, Global } from '../common/global';
 import * as fs from 'fs';
 import * as vscode from 'vscode';
 import { CodeUtil } from '../common/codeUtil';
 import { Out } from '../common/out';
 import { Script, Method, Ref, Label, Block, Variable } from './model';
 
+export interface BuildScriptOptions {
+    /** Defaults to false. If true, short-circuits when document is in cache. */
+    usingCache?: boolean;
+    /** Lines to parse. Defaults to extension setting. -1 for unlimited parsing. 0 for no parsing. */
+    maximumParseLength?: number;
+}
+
 export class Parser {
     private static documentCache = new Map<string, Script>();
 
@@ -42,27 +50,36 @@ export class Parser {
      */
     public static async buildScript(
         document: vscode.TextDocument,
-        usingCache = false,
+        options: BuildScriptOptions = {},
     ): Promise<Script> {
-        if (usingCache && this.documentCache.get(document.uri.path)) {
+        if (options.usingCache && this.documentCache.get(document.uri.path)) {
             return this.documentCache.get(document.uri.path);
         }
 
+        const maxParseLength =
+            options.maximumParseLength ??
+            Global.getConfig<number>(ConfigKey.maximumParseLength);
+        // limit parse length for performance
+        /** Count of lines to parse */
+        const linesToParse =
+            maxParseLength >= 0
+                ? Math.min(document.lineCount, maxParseLength)
+                : document.lineCount;
+
         const methods: Method[] = [];
         const refs: Ref[] = [];
         const labels: Label[] = [];
         const variables: Variable[] = [];
         const blocks: Block[] = [];
         let currentMethod: Method;
         let deep = 0;
-        const lineCount = Math.min(document.lineCount, 10000);
         let blockComment = false;
-        for (let line = 0; line < lineCount; line++) {
+        for (let line = 0; line < linesToParse; line++) {
             const lineText = document.lineAt(line).text;
-            if (lineText.match(/ *\/\*/)) {
+            if (lineText.match(startBlockComment)) {
                 blockComment = true;
             }
-            if (lineText.match(/ *\*\//)) {
+            if (lineText.match(endBlockComment)) {
                 blockComment = false;
             }
             if (blockComment) {
@@ -277,7 +294,7 @@ export class Parser {
         document: vscode.TextDocument,
         line: number,
         origin?: string,
-    ) {
+    ): Method | Ref | Ref[] {
         origin ??= document.lineAt(line).text;
         const text = CodeUtil.purify(origin);
         // [\u4e00-\u9fa5] Chinese unicode characters
@@ -374,3 +391,6 @@ export class Parser {
         }
     }
 }
+
+const startBlockComment = / *\/\*/;
+const endBlockComment = / *\*\//;

src/providers/completionProvider.ts

@@ -65,7 +65,7 @@ export class CompletionProvider implements vscode.CompletionItemProvider {
             }
         });
 
-        const script = await Parser.buildScript(document, true);
+        const script = await Parser.buildScript(document, { usingCache: true });
         script.variables.forEach((variable) => {
             const completionItem = new vscode.CompletionItem(
                 variable.name,

src/providers/defProvider.ts

@@ -43,7 +43,7 @@ export class DefProvider implements vscode.DefinitionProvider {
             );
         }
 
-        const script = await Parser.buildScript(document, true);
+        const script = await Parser.buildScript(document, { usingCache: true });
 
         for (const method of script.methods) {
             if (

src/providers/symbolProvider.ts

@@ -8,7 +8,7 @@ export class SymbolProvider implements vscode.DocumentSymbolProvider {
     ): Promise<vscode.DocumentSymbol[]> {
         const result = [];
 
-        const script = await Parser.buildScript(document, false);
+        const script = await Parser.buildScript(document);
 
         for (const method of script.methods) {
             result.push(

src/test/suite/format/format.test.ts

@@ -1,3 +1,4 @@
+import { getDocument } from '../../utils';
 import * as assert from 'assert';
 import * as fs from 'fs-extra';
 import * as path from 'path';
@@ -148,6 +149,7 @@ suite('Internal formatter', () => {
 suite('External formatter', () => {
     // test external formatter a few times to make sure the connection is working
     // advanced tests are for internal formatter only
+    // TODO note these tests only support editor settings, not extension settings
     const externalFormatTests: FormatTest[] = [
         { filenameRoot: '25-multiline-string' },
         {
@@ -168,10 +170,9 @@ suite('External formatter', () => {
             const outFileString = fs
                 .readFileSync(path.join(filesParentPath, outFilename))
                 .toString();
-            const unformattedSampleFile =
-                await vscode.workspace.openTextDocument(
-                    path.join(filesParentPath, inFilename),
-                );
+            const unformattedSampleFile = await getDocument(
+                path.join(filesParentPath, inFilename),
+            );
             const originalText = unformattedSampleFile.getText();
             const textEditor = await vscode.window.showTextDocument(
                 unformattedSampleFile,

src/test/suite/parser/parser.test.ts

@@ -1,4 +1,6 @@
+import { getDocument } from '../../utils';
 import * as assert from 'assert';
+import * as path from 'path';
 import * as vscode from 'vscode';
 import { Parser } from '../../../parser/Parser';
 
@@ -72,4 +74,63 @@ suite('Parser', () => {
             });
         });
     });
+
+    suite('buildScript', () => {
+        // Currently in `out` folder, need to get back to main `src` folder
+        const filesParentPath = path.join(
+            __dirname,
+            '..',
+            '..',
+            '..',
+            '..',
+            'src',
+            'test',
+            'suite',
+            'parser',
+            'samples',
+        );
+
+        const myTests: {
+            name: string;
+            maximumParseLength: number;
+            expectedMethodCount: number;
+        }[] = [
+            {
+                name: 'stops at provided max parse length',
+                maximumParseLength: 10_000,
+                expectedMethodCount: 1,
+            },
+            {
+                name: 'respects parse lengths higher than ten thousand',
+                maximumParseLength: 11_000,
+                expectedMethodCount: 2,
+            },
+            {
+                name: '-1 means unlimited parsing',
+                maximumParseLength: -1,
+                expectedMethodCount: 2,
+            },
+            {
+                name: '0 means no parsing',
+                maximumParseLength: 0,
+                expectedMethodCount: 0,
+            },
+        ];
+
+        myTests.forEach((myTest) =>
+            test(myTest.name, async () => {
+                const filename = '117-ten-thousand-lines.ahk';
+                const document = await getDocument(
+                    path.join(filesParentPath, filename),
+                );
+                const result = await Parser.buildScript(document, {
+                    maximumParseLength: myTest.maximumParseLength,
+                });
+                assert.strictEqual(
+                    result.methods.length,
+                    myTest.expectedMethodCount,
+                );
+            }),
+        );
+    });
 });