Fix codebook generation (ghostery#4096)

seia-soto · Jul 15, 2024 · 046ab8c · 046ab8c
1 parent 7ca177c
commit 046ab8c
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 45 deletions.
diff --git a/packages/adblocker/package.json b/packages/adblocker/package.json
@@ -49,16 +49,16 @@
     "build": "tshy && rollup --config ./rollup.config.js",
     "test": "nyc mocha",
     "dev": "mocha --watch",
-    "bench-metadata": "ts-node --project ./tools/tsconfig.json ./tools/bench-metadata.ts",
-    "bump-internal-engine-version": "ts-node --project ./tools/tsconfig.json ./tools/auto-bump-engine-version.ts",
-    "generate-codebooks": "concurrently -n build: \"yarn:codebook-*\" && yarn bump-internal-engine-version",
-    "codebook-network-csp": "ts-node --project ./tools/tsconfig.json ./tools/generate_compression_codebooks.ts -- network-csp",
-    "codebook-network-redirect": "ts-node --project ./tools/tsconfig.json ./tools/generate_compression_codebooks.ts -- network-redirect",
-    "codebook-network-filter": "ts-node --project ./tools/tsconfig.json ./tools/generate_compression_codebooks.ts -- network-filter",
-    "codebook-network-hostname": "ts-node --project ./tools/tsconfig.json ./tools/generate_compression_codebooks.ts -- network-hostname",
-    "codebook-cosmetic-selector": "ts-node --project ./tools/tsconfig.json ./tools/generate_compression_codebooks.ts -- cosmetic-selector",
-    "codebook-raw-network": "ts-node --project ./tools/tsconfig.json ./tools/generate_compression_codebooks.ts -- raw-network",
-    "codebook-raw-cosmetic": "ts-node --project ./tools/tsconfig.json ./tools/generate_compression_codebooks.ts -- raw-cosmetic"
+    "bench-metadata": "TS_NODE_PROJECT=./tools/tsconfig.json node --loader ts-node/esm ./tools/bench-metadata.ts",
+    "bump-internal-engine-version": "TS_NODE_PROJECT=./tools/tsconfig.json node --loader ts-node/esm ./tools/auto-bump-engine-version.ts",
+    "codebook-network-csp": "TS_NODE_PROJECT=./tools/tsconfig.json node --loader ts-node/esm ./tools/generate_compression_codebooks.ts -- network-csp",
+    "codebook-network-redirect": "TS_NODE_PROJECT=./tools/tsconfig.json node --loader ts-node/esm ./tools/generate_compression_codebooks.ts -- network-redirect",
+    "codebook-network-filter": "TS_NODE_PROJECT=./tools/tsconfig.json node --loader ts-node/esm ./tools/generate_compression_codebooks.ts -- network-filter",
+    "codebook-network-hostname": "TS_NODE_PROJECT=./tools/tsconfig.json node --loader ts-node/esm ./tools/generate_compression_codebooks.ts -- network-hostname",
+    "codebook-cosmetic-selector": "TS_NODE_PROJECT=./tools/tsconfig.json node --loader ts-node/esm ./tools/generate_compression_codebooks.ts -- cosmetic-selector",
+    "codebook-raw-network": "TS_NODE_PROJECT=./tools/tsconfig.json node --loader ts-node/esm ./tools/generate_compression_codebooks.ts -- raw-network",
+    "codebook-raw-cosmetic": "TS_NODE_PROJECT=./tools/tsconfig.json node --loader ts-node/esm ./tools/generate_compression_codebooks.ts -- raw-cosmetic",
+    "generate-codebooks": "concurrently -n build: \"yarn:codebook-*\" && yarn bump-internal-engine-version"
   },
   "bugs": {
     "url": "https://github.com/ghostery/adblocker/issues"

diff --git a/packages/adblocker/tools/auto-bump-engine-version.ts b/packages/adblocker/tools/auto-bump-engine-version.ts
@@ -1,26 +1,27 @@
-import { promises as fs } from 'fs';
-import { resolve, join } from 'path';
+import { readFileSync, writeFileSync } from 'node:fs';
+import { resolve, join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
 
-(async () => {
-  const engineSourcePath = resolve(__dirname, join('..', 'src', 'engine', 'engine.ts'));
-  const engineSource = await fs.readFile(engineSourcePath, 'utf-8');
+const __dirname = dirname(fileURLToPath(import.meta.url));
 
-  await fs.writeFile(
-    engineSourcePath,
-    engineSource.replace(
-      /(^export const ENGINE_VERSION =\s+)(\d+)(;$)/m,
-      (match, prefix, version, suffix, offset) => {
-        console.log('Match:', { match, prefix, version, suffix, offset });
+const engineSourcePath = resolve(__dirname, join('..', 'src', 'engine', 'engine.ts'));
+const engineSource = readFileSync(engineSourcePath, 'utf-8');
 
-        const currentVersion = Number(version);
-        console.log('Current version:', currentVersion);
+writeFileSync(
+  engineSourcePath,
+  engineSource.replace(
+    /(^export const ENGINE_VERSION =\s+)(\d+)(;$)/m,
+    (match, prefix, version, suffix, offset) => {
+      console.log('Match:', { match, prefix, version, suffix, offset });
 
-        const nextVersion = currentVersion + 1;
-        console.log('Next version:', nextVersion);
+      const currentVersion = Number(version);
+      console.log('Current version:', currentVersion);
 
-        return `${prefix}${nextVersion}${suffix}`;
-      },
-    ),
-    'utf-8',
-  );
-})();
+      const nextVersion = currentVersion + 1;
+      console.log('Next version:', nextVersion);
+
+      return `${prefix}${nextVersion}${suffix}`;
+    },
+  ),
+  'utf-8',
+);
diff --git a/packages/adblocker/tools/engine-size.ts b/packages/adblocker/tools/engine-size.ts
@@ -6,23 +6,23 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { promises as fs } from 'fs';
-import { join } from 'path';
-import { gzipSync, brotliCompressSync } from 'zlib';
+import { readFileSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { gzipSync, brotliCompressSync } from 'node:zlib';
 
 import { FiltersEngine, adsLists, adsAndTrackingLists, fullLists } from '../src/index.js';
 
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
 const PREFIX =
   'https://raw.githubusercontent.com/ghostery/adblocker/master/packages/adblocker/assets';
 
 async function loadFromLocalAssets(lists: string[]): Promise<string> {
-  return (
-    await Promise.all(
-      lists
-        .map((path) => join(__dirname, '..', 'assets', path.slice(PREFIX.length)))
-        .map((path) => fs.readFile(path, 'utf-8')),
-    )
-  ).join('\n');
+  return lists
+    .map((path) => join(__dirname, '..', 'assets', path.slice(PREFIX.length)))
+    .map((path) => readFileSync(path, 'utf-8'))
+    .join('\n');
 }
 
 function loadAdsLists(): Promise<string> {

diff --git a/packages/adblocker/tools/generate_compression_codebooks.ts b/packages/adblocker/tools/generate_compression_codebooks.ts
@@ -1,5 +1,6 @@
-import { promises as fs } from 'fs';
-import { resolve, join } from 'path';
+import { promises as fs } from 'node:fs';
+import { resolve, join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { generate } from '@remusao/smaz-generate';
 import { Smaz } from '@remusao/smaz';
 
@@ -11,6 +12,8 @@ import {
   hasUnicode,
 } from '../src/index.js';
 
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
 const PREFIX =
   'https://raw.githubusercontent.com/ghostery/adblocker/master/packages/adblocker/assets';
 

diff --git a/packages/adblocker/tools/priorities.ts b/packages/adblocker/tools/priorities.ts
@@ -6,8 +6,9 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { promises as fs } from 'fs';
-import { join } from 'path';
+import { promises as fs } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
 
 import {
   fullLists,
@@ -41,6 +42,8 @@ class Counter<K> {
   }
 }
 
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
 const PREFIX =
   'https://raw.githubusercontent.com/ghostery/adblocker/master/packages/adblocker/assets';