Merge branch 'feature/AG-26623-1' into feature/AG-26623-4

AdguardTeam · Oct 16, 2023 · f92b92b · f92b92b
2 parents ef2e716 + f531df1
commit f92b92b
Show file tree

Hide file tree

Showing 61 changed files with 3,106 additions and 1,132 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -2,7 +2,7 @@
 name: test
 
 env:
-  NODE_VERSION: 16
+  NODE_VERSION: 18
 
 on:
   push:

diff --git a/bamboo-specs/adguard-api-build.yaml b/bamboo-specs/adguard-api-build.yaml
@@ -5,7 +5,7 @@ plan:
   key: ADGUARDAPIBUILD
   name: Adguard API  - build
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Build:

diff --git a/bamboo-specs/adguard-api-deploy.yaml b/bamboo-specs/adguard-api-deploy.yaml
@@ -9,7 +9,7 @@ environments:
 
 npmjs:
   docker:
-    image: adguard/node-ssh:16.17--1
+    image: adguard/node-ssh:18.13--1
     volumes:
       ${system.YARN_DIR}: "${bamboo.cacheYarn}"
   triggers: [ ]

diff --git a/bamboo-specs/adguard-api-increment.yaml b/bamboo-specs/adguard-api-increment.yaml
@@ -5,7 +5,7 @@ plan:
   key: ADGUARDAPIINCREMENT
   name: Adguard API  - increment
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Increment:

diff --git a/bamboo-specs/agtree-build.yaml b/bamboo-specs/agtree-build.yaml
@@ -5,7 +5,7 @@ plan:
   key: AGTREEBUILD
   name: AGTree - build
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Build:

diff --git a/bamboo-specs/agtree-deploy.yaml b/bamboo-specs/agtree-deploy.yaml
@@ -9,7 +9,7 @@ environments:
 
 npmjs:
   docker:
-    image: adguard/node-ssh:16.17--1
+    image: adguard/node-ssh:18.13--1
     volumes:
       ${system.YARN_DIR}: "${bamboo.cacheYarn}"
   triggers: [ ]

diff --git a/bamboo-specs/agtree-increment.yaml b/bamboo-specs/agtree-increment.yaml
@@ -5,7 +5,7 @@ plan:
   key: AGTREEINCR
   name: AGTree - increment
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Increment:

diff --git a/bamboo-specs/agtree-test.yaml b/bamboo-specs/agtree-test.yaml
@@ -5,7 +5,7 @@ plan:
   key: AGTREETEST
   name: AGTree - tests
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Test:

diff --git a/bamboo-specs/tsurlfilter-build.yaml b/bamboo-specs/tsurlfilter-build.yaml
@@ -5,7 +5,7 @@ plan:
   key: TSURLFILTERBUILD
   name: tsurlfilter - build
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Build:

diff --git a/bamboo-specs/tsurlfilter-deploy.yaml b/bamboo-specs/tsurlfilter-deploy.yaml
@@ -12,7 +12,7 @@ environments:
 
 npmjs • 1.x:
   docker:
-    image: adguard/node-ssh:16.17--1
+    image: adguard/node-ssh:18.13--1
     volumes:
       ${system.YARN_DIR}: "${bamboo.cacheYarn}"
   triggers: [ ]

diff --git a/bamboo-specs/tsurlfilter-increment.yaml b/bamboo-specs/tsurlfilter-increment.yaml
@@ -5,7 +5,7 @@ plan:
   key: TSURLFILTERINCR
   name: tsurlfilter - increment
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Increment:

diff --git a/bamboo-specs/tsurlfilter-tests.yaml b/bamboo-specs/tsurlfilter-tests.yaml
@@ -5,7 +5,7 @@ plan:
   key: TSURLFILTERTEST
   name: tsurlfilter - tests
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Test:

diff --git a/bamboo-specs/tswebextension-build.yaml b/bamboo-specs/tswebextension-build.yaml
@@ -5,7 +5,7 @@ plan:
   key: TSWEBEXTENSIONBUILD
   name: tswebextension - build
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Build:

diff --git a/bamboo-specs/tswebextension-deploy.yaml b/bamboo-specs/tswebextension-deploy.yaml
@@ -9,7 +9,7 @@ environments:
 
 npmjs:
   docker:
-    image: adguard/node-ssh:16.17--1
+    image: adguard/node-ssh:18.13--1
     volumes:
       ${system.YARN_DIR}: "${bamboo.cacheYarn}"
   triggers: [ ]

diff --git a/bamboo-specs/tswebextension-increment.yaml b/bamboo-specs/tswebextension-increment.yaml
@@ -5,7 +5,7 @@ plan:
   key: TSWEBEXTENSIONINCR
   name: tswebextension - increment
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Increment:

diff --git a/bamboo-specs/tswebextension-tests.yaml b/bamboo-specs/tswebextension-tests.yaml
@@ -5,7 +5,7 @@ plan:
   key: TSWEBEXTENSIONTEST
   name: tswebextension - tests
 variables:
-  dockerContainer: adguard/node-ssh:16.17--1
+  dockerContainer: adguard/node-ssh:18.13--1
 
 stages:
   - Test:

diff --git a/packages/tsurlfilter/CHANGELOG.md b/packages/tsurlfilter/CHANGELOG.md
@@ -1,14 +1,26 @@
 # TSUrlFilter Changelog
 
 All notable changes to this project will be documented in this file.
+
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-
 <!-- TODO: manually add compare links for version to the end of the file -->
 <!-- e.g. [1.0.77]: https://github.com/AdguardTeam/tsurlfilter/compare/tsurlfilter-v1.0.76...tsurlfilter-v1.0.77 -->
 
 
+## [2.2.1] - 2023-10-13
+
+### Changed
+- Updated `@adguard/scriptlets` to `v1.9.83`.
+
+
+## [2.2.0] - 2023-10-12
+
+### Added
+- Support for $badfilter rules to Declarative Converter.
+
+
 ## [2.1.12] - 2023-09-25
 
 ### Fixed

diff --git a/packages/tsurlfilter/cli/convertFilters.ts b/packages/tsurlfilter/cli/convertFilters.ts
@@ -3,20 +3,17 @@ import path from 'path';
 import fs from 'fs';
 
 import {
+    type ConversionResult,
+    type IRuleSet,
     DeclarativeFilterConverter,
     Filter,
-    FILTER_LIST_IDS_FILENAME_JSON,
-    REGEXP_RULES_COUNT_FILENAME,
-    RULES_COUNT_FILENAME,
-    SOURCE_MAP_FILENAME_JSON,
+    METADATA_FILENAME,
+    LAZY_METADATA_FILENAME,
 } from '../src/rules/declarative-converter';
 
 const ensureDirSync = (dirPath: string) => {
     if (!fs.existsSync(dirPath)) {
-        fs.mkdirSync(dirPath, {
-            recursive: true,
-            mode: 0o2775,
-        });
+        fs.mkdirSync(dirPath, { recursive: true });
     }
 };
 
@@ -40,94 +37,132 @@ export const convertFilters = async (
     const resourcesPath = path.resolve(process.cwd(), resourcesDir);
     const destRuleSetsPath = path.resolve(process.cwd(), destRuleSetsDir);
 
-    try {
-        ensureDirSync(filtersPath);
-        ensureDirSync(destRuleSetsPath);
+    ensureDirSync(filtersPath);
+    ensureDirSync(destRuleSetsPath);
 
-        const filtersPaths = new Map<number, string>();
+    const filtersPaths = new Map<number, string>();
 
-        const files = fs.readdirSync(filtersPath);
-        const filterList = files
-            .map((filePath: string) => {
-                console.info(`Parsing ${filePath}...`);
-                const index = filePath.match(/\d+/);
+    const files = fs.readdirSync(filtersPath);
+    const filters = files
+        .map((filePath: string) => {
+            console.info(`Parsing ${filePath}...`);
+            const index = filePath.match(/\d+/);
 
-                if (!index) {
-                    console.info(`${filePath} skipped`);
-                    return null;
-                }
+            if (!index) {
+                console.info(`${filePath} skipped`);
+                return null;
+            }
 
-                const filterId = Number(index);
+            const filterId = Number(index);
 
-                filtersPaths.set(filterId, filePath);
-                const data = fs.readFileSync(`${filtersPath}/${filePath}`, { encoding: 'utf-8' });
+            filtersPaths.set(filterId, filePath);
+            const data = fs.readFileSync(
+                path.resolve(filtersPath, filePath),
+                { encoding: 'utf-8' },
+            );
 
-                console.info(`Preparing filter #${filterId} to convert`);
+            console.info(`Preparing filter #${filterId} to convert`);
 
-                return new Filter(filterId, {
-                    getContent: () => Promise.resolve(data.split('\n')),
-                });
-            })
-            .filter((filter): filter is Filter => filter !== null);
+            return new Filter(filterId, {
+                getContent: async () => data.split(/\r?\n/),
+            });
+        })
+        .filter((filter): filter is Filter => filter !== null);
 
-        const converter = new DeclarativeFilterConverter();
-        const {
-            ruleSets,
-            errors,
-            limitations,
-        } = await converter.convert(
-            filterList,
+    const convertedRuleSets: IRuleSet[] = [];
+    let errors: ConversionResult['errors'] = [];
+    let limitations: ConversionResult['limitations'] = [];
+
+    const converter = new DeclarativeFilterConverter();
+
+    for (let i = 0; i < filters.length; i += 1) {
+        const filter = filters[i];
+
+        // eslint-disable-next-line no-await-in-loop
+        const converted = await converter.convertStaticRuleSet(
+            filter,
             { resourcesPath },
         );
 
-        console.log(`Converted with ${errors.length} errors`);
+        convertedRuleSets.push(converted.ruleSet);
+        errors = errors.concat(converted.errors);
+        limitations = limitations.concat(converted.limitations);
+
+        if (debug) {
+            console.log('======================================');
+            console.log(`Filter #${filter.getId()} info`);
+            console.log('======================================');
 
-        if (limitations.length > 0) {
-            console.log(`Skipped ${limitations.length} converted rules`);
+            console.log(`Errors: ${converted.errors.length}`);
+            if (converted.errors.length > 0) {
+                console.log('======================================');
+                console.log('Converted with following errors: ');
+                console.log('======================================');
+                converted.errors.forEach((e) => console.log(e.message));
+            }
 
-            if (debug) {
+            if (converted.limitations.length > 0) {
+                // eslint-disable-next-line max-len
+                console.log(`Some converted rules were discarded to fit within the limits: ${converted.limitations.length}`);
                 console.log('======================================');
                 console.log('Converted with following limitations: ');
                 console.log('======================================');
-                limitations.forEach((e) => console.log(e.message));
+                converted.limitations.forEach((e) => console.log(e.message));
             }
         }
+    }
 
-        if (debug) {
-            console.log('======================================');
-            console.log('Converted with following errors: ');
-            console.log('======================================');
-            errors.forEach((e) => console.log(e.message));
-        }
+    console.log('======================================');
+    console.log('Common info');
+    console.log('======================================');
 
-        const promises = ruleSets.map(async (ruleSet) => {
-            const {
-                id,
-                declarativeRules,
-                regexpRulesCount,
-                rulesCount,
-                sourceMap,
-                filterListsIds,
-            } = await ruleSet.serialize();
-
-            const ruleSetDir = `${destRuleSetsPath}/${id}`;
-            ensureDirSync(ruleSetDir);
-
-            await Promise.all([
-                fs.promises.writeFile(`${ruleSetDir}/${id}.json`, JSON.stringify(declarativeRules, null, '\t')),
-                fs.promises.writeFile(`${ruleSetDir}/${REGEXP_RULES_COUNT_FILENAME}`, regexpRulesCount.toString()),
-                fs.promises.writeFile(`${ruleSetDir}/${RULES_COUNT_FILENAME}`, rulesCount.toString()),
-                fs.promises.writeFile(`${ruleSetDir}/${SOURCE_MAP_FILENAME_JSON}`, sourceMap),
-                fs.promises.writeFile(`${ruleSetDir}/${FILTER_LIST_IDS_FILENAME_JSON}`, JSON.stringify(filterListsIds)),
-            ]);
+    console.log(`Converted rule sets: ${convertedRuleSets.length}`);
 
-            console.log('======================================');
-            console.info(`Rule set with id ${id} and all rule set info (counters, source map, filter list) was saved`);
-            console.info(`to ${destRuleSetsDir}/${id}`);
-            console.log('======================================');
-        });
-        await Promise.all(promises);
-    } catch (e) {
-        console.error(e);
+    console.log(`Errors: ${errors.length}`);
+
+    if (debug && errors.length > 0) {
+        console.log('======================================');
+        console.log('Converted with following errors: ');
+        console.log('======================================');
+        errors.forEach((e) => console.log(e.message));
+    }
+
+    console.log(`Skipped converting for rules: ${limitations.length}`);
+
+    if (debug && limitations.length > 0) {
+        console.log('======================================');
+        console.log('Converted with following limitations: ');
+        console.log('======================================');
+        limitations.forEach((e) => console.log(e.message));
+    }
+
+    for (let i = 0; i < convertedRuleSets.length; i += 1) {
+        const ruleSet = convertedRuleSets[i];
+
+        const {
+            id,
+            data,
+            lazyData,
+            // eslint-disable-next-line no-await-in-loop
+        } = await ruleSet.serialize();
+
+        // eslint-disable-next-line no-await-in-loop
+        const declarativeRules = await ruleSet.getDeclarativeRules();
+
+        const ruleSetDir = `${destRuleSetsPath}/${id}`;
+        ensureDirSync(ruleSetDir);
+
+        // eslint-disable-next-line no-await-in-loop
+        await Promise.all([
+            fs.promises.writeFile(`${ruleSetDir}/${id}.json`, JSON.stringify(declarativeRules, null, '\t')),
+            fs.promises.writeFile(`${ruleSetDir}/${METADATA_FILENAME}`, data),
+            fs.promises.writeFile(`${ruleSetDir}/${LAZY_METADATA_FILENAME}`, lazyData),
+        ]);
+
+        console.log('===============================================');
+        console.info(`Rule set with id ${id} and all rule set info`);
+        console.info('(counters, source map, filter list) was saved');
+        console.info(`to ${destRuleSetsDir}/${id}`);
+        console.log('===============================================');
     }
 };