AG-26623 Optimize CSS tokenizer

Merge in ADGUARD-FILTERS/tsurlfilter from feature/AG-26623-8 to feature/AG-26623-1 Squashed commit of the following: commit 35a2840 Merge: ea95983 ca47032 Author: scripthunter7 <d.tota@adguard.com> Date: Fri Oct 20 10:36:08 2023 +0200 Merge branch 'feature/AG-26623-1' into feature/AG-26623-8 commit ea95983 Author: scripthunter7 <d.tota@adguard.com> Date: Thu Oct 19 14:58:20 2023 +0200 Optimize CSS tokenizer
AdguardTeam · Oct 20, 2023 · b520120 · b520120
1 parent ca47032
commit b520120
Show file tree

Hide file tree

Showing 7 changed files with 313 additions and 296 deletions.
diff --git a/packages/css-tokenizer/rollup.config.ts b/packages/css-tokenizer/rollup.config.ts
@@ -69,14 +69,7 @@ const commonPlugins = [
 // Plugins for Node.js builds
 const nodePlugins = [
     ...commonPlugins,
-    // Minify the output with Terser
-    terser({
-        sourceMap: true,
-        output: {
-            // Keep the banner in the minified output
-            preamble: BANNER,
-        },
-    }),
+    // TODO: Add other plugins if needed
 ];
 
 // Plugins for browser builds

diff --git a/packages/css-tokenizer/src/algorithms/consumers/delim-token.ts b/packages/css-tokenizer/src/algorithms/consumers/delim-token.ts
@@ -14,8 +14,6 @@ import { TokenType } from '../../common/enums/token-types';
  */
 export const consumeDelimToken: TokenizerContextFunction = (context: TokenizerContext): void => {
     // Return a <delim-token> with its value set to the current input code point.
-    const start = context.offset;
-
     context.consumeCodePoint();
-    context.onToken(TokenType.Delim, start, context.offset);
+    context.onToken(TokenType.Delim, context.offset - 1, context.offset);
 };
diff --git a/packages/css-tokenizer/src/algorithms/consumers/string-token.ts b/packages/css-tokenizer/src/algorithms/consumers/string-token.ts
@@ -31,7 +31,7 @@ export const consumeStringToken: TokenizerContextFunction = (context: TokenizerC
 
     // Repeatedly consume the next input code point from the stream:
     // eslint-disable-next-line no-constant-condition
-    while (context.offset <= context.source.length) {
+    while (context.offset <= context.length) {
         // ending code point
         if (context.code() === endingCodePoint) {
             // Consume it

diff --git a/packages/css-tokenizer/src/algorithms/consumers/url-token.ts b/packages/css-tokenizer/src/algorithms/consumers/url-token.ts
@@ -80,7 +80,7 @@ export const consumeUrlToken: TokenizerContextFunction = (context: TokenizerCont
 
     // Repeatedly consume the next input code point from the stream:
     // eslint-disable-next-line no-constant-condition
-    while (context.offset <= context.source.length) {
+    while (context.offset <= context.length) {
         // U+0029 RIGHT PARENTHESIS ())
         if (context.code() === CodePoint.RightParenthesis) {
             // Consume it.

diff --git a/packages/css-tokenizer/src/common/context.ts b/packages/css-tokenizer/src/common/context.ts
@@ -18,6 +18,11 @@ export class TokenizerContext {
      */
     private cursor: number;
 
+    /**
+     * Cached source length
+     */
+    public readonly length: number;
+
     /**
      * Reference to the `onToken` callback function
      */
@@ -44,6 +49,7 @@ export class TokenizerContext {
     constructor(source: string, onToken: OnTokenCallback, onError: OnErrorCallback, shiftOffset: number) {
         // Set the source and offset
         this.source = source;
+        this.length = source.length;
         this.cursor = shiftOffset;
 
         // Set the callback functions
@@ -102,7 +108,7 @@ export class TokenizerContext {
      * @returns `true` if the current offset is at the end of the source, `false` otherwise
      */
     public isEof(): boolean {
-        return this.offset === this.source.length;
+        return this.offset === this.length;
     }
 
     /**
@@ -111,7 +117,7 @@ export class TokenizerContext {
      * @returns `true` if the next code point is EOF, `false` otherwise
      */
     public isNextEof(): boolean {
-        return this.cursor + 1 === this.source.length;
+        return this.cursor + 1 === this.length;
     }
 
     /**

diff --git a/packages/css-tokenizer/src/common/enums/code-points.ts b/packages/css-tokenizer/src/common/enums/code-points.ts
@@ -158,6 +158,46 @@ export const enum CodePoint {
      */
     DigitZero = 0x30,
 
+    /**
+     * U+0031 DIGIT ONE: `1`
+     */
+    DigitOne = 0x31,
+
+    /**
+     * U+0032 DIGIT TWO: `2`
+     */
+    DigitTwo = 0x32,
+
+    /**
+     * U+0033 DIGIT THREE: `3`
+     */
+    DigitThree = 0x33,
+
+    /**
+     * U+0034 DIGIT FOUR: `4`
+     */
+    DigitFour = 0x34,
+
+    /**
+     * U+0035 DIGIT FIVE: `5`
+     */
+    DigitFive = 0x35,
+
+    /**
+     * U+0036 DIGIT SIX: `6`
+     */
+    DigitSix = 0x36,
+
+    /**
+     * U+0037 DIGIT SEVEN: `7`
+     */
+    DigitSeven = 0x37,
+
+    /**
+     * U+0038 DIGIT EIGHT: `8`
+     */
+    DigitEight = 0x38,
+
     /**
      * U+0039 DIGIT NINE: `9`
      */