Add SourceProcessor class (#28)

* sourcemap tools: add sourcemap processor for adding source snippets and sourcemap keys

* sourcemap tools: fix Prettier issues

* sourcemap tools: add semicolon to end of source snippet in DebugIdGenerator

* sourcemap tools: renames SourceProcessor tests

* sourcemap tools: add comment about addMapping

* sourcemap tools: add docs to SourceProcessor

* sourcemap tools: add more explanation to offsetting sourcemaps

* sourcemap tools: update SourceProcessor tests to test for number of newlines in modified sourcemaps

---------

Co-authored-by: Sebastian Alex <sebastian.alex@saucelabs.com>

Author: perf2711

package-lock.json

@@ -42,5 +42,8 @@
         "nock": "^13.3.1",
         "ts-jest": "^29.1.0",
         "typescript": "^5.0.4"
+    },
+    "dependencies": {
+        "source-map": "^0.7.4"
     }
 }

tools/sourcemap-tools/package.json

@@ -4,7 +4,7 @@ export const SOURCEMAP_DEBUG_ID_KEY = 'debugId';
 
 export class DebugIdGenerator {
     public generateSourceSnippet(uuid: string) {
-        return `!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{},n=(new Error).stack;n&&(e.${SOURCE_DEBUG_ID_VARIABLE}=e.${SOURCE_DEBUG_ID_VARIABLE}||{},e.${SOURCE_DEBUG_ID_VARIABLE}[n]="${uuid}")}catch(e){}}()`;
+        return `;!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{},n=(new Error).stack;n&&(e.${SOURCE_DEBUG_ID_VARIABLE}=e.${SOURCE_DEBUG_ID_VARIABLE}||{},e.${SOURCE_DEBUG_ID_VARIABLE}[n]="${uuid}")}catch(e){}}();`;
     }
 
     public generateSourceComment(uuid: string) {

tools/sourcemap-tools/src/DebugIdGenerator.ts

@@ -0,0 +1,92 @@
+import fs from 'fs';
+import { BasicSourceMapConsumer, Position, RawSourceMap, SourceMapConsumer, SourceMapGenerator } from 'source-map';
+import { DebugIdGenerator } from './DebugIdGenerator';
+import { stringToUuid } from './helpers/stringToUuid';
+
+export class SourceProcessor {
+    constructor(private readonly _debugIdGenerator: DebugIdGenerator) {}
+
+    /**
+     * Adds required snippets and comments to source, and modifies sourcemap to include debug ID.
+     * @param source Source content.
+     * @param sourceMap Sourcemap object or JSON.
+     * @param debugId Debug ID. If not provided, one will be generated from `source`.
+     * @returns Used debug ID, new source and new sourcemap.
+     */
+    public async processSourceAndSourceMap(source: string, sourceMap: string | RawSourceMap, debugId?: string) {
+        if (!debugId) {
+            debugId = stringToUuid(source);
+        }
+
+        const sourceSnippet = this._debugIdGenerator.generateSourceSnippet(debugId);
+        const sourceComment = this._debugIdGenerator.generateSourceComment(debugId);
+
+        const newSource = sourceSnippet + '\n' + source + '\n' + sourceComment;
+
+        // We need to offset the source map by amount of lines that we're inserting to the source code
+        // Sourcemaps map code like this:
+        // original code X:Y => generated code A:B
+        // So if we add any code to generated code, mappings after that code will become invalid
+        // We need to offset the mapping lines by sourceSnippetNewlineCount:
+        // original code X:Y => generated code (A + sourceSnippetNewlineCount):B
+        const sourceSnippetNewlineCount = sourceSnippet.match(/\n/g)?.length ?? 0;
+        const offsetSourceMap = await this.offsetSourceMap(sourceMap, 0, sourceSnippetNewlineCount + 1);
+        const newSourceMap = this._debugIdGenerator.addSourceMapKey(offsetSourceMap, debugId);
+
+        return { debugId, source: newSource, sourceMap: newSourceMap };
+    }
+
+    /**
+     * Adds required snippets and comments to source, and modifies sourcemap to include debug ID.
+     * Will write modified content to the files.
+     * @param sourcePath Path to the source.
+     * @param sourceMapPath Path to the sourcemap.
+     * @param debugId Debug ID. If not provided, one will be generated from `source`.
+     * @returns Used debug ID.
+     */
+    public async processSourceAndSourceMapFiles(sourcePath: string, sourceMapPath: string, debugId?: string) {
+        const source = await fs.promises.readFile(sourcePath, 'utf8');
+        const sourceMap = await fs.promises.readFile(sourceMapPath, 'utf8');
+
+        const result = await this.processSourceAndSourceMap(source, sourceMap, debugId);
+
+        await fs.promises.writeFile(sourcePath, result.source, 'utf8');
+        await fs.promises.writeFile(sourceMapPath, JSON.stringify(result.sourceMap), 'utf8');
+
+        return result.debugId;
+    }
+
+    private async offsetSourceMap(
+        sourceMap: string | RawSourceMap,
+        fromLine: number,
+        count: number,
+    ): Promise<RawSourceMap> {
+        const consumer = (await new SourceMapConsumer(sourceMap)) as BasicSourceMapConsumer;
+        const newSourceMap = new SourceMapGenerator({
+            file: consumer.file,
+            sourceRoot: consumer.sourceRoot,
+        });
+
+        consumer.eachMapping((m) => {
+            if (m.generatedLine < fromLine) {
+                return;
+            }
+
+            // Despite how the mappings are written, addMapping expects here a null value if the column/line is not set
+            newSourceMap.addMapping({
+                source: m.source,
+                name: m.name,
+                generated:
+                    m?.generatedColumn != null && m?.generatedLine != null
+                        ? { column: m.generatedColumn, line: m.generatedLine + count }
+                        : (null as unknown as Position),
+                original:
+                    m?.originalColumn != null && m?.originalLine != null
+                        ? { column: m.originalColumn, line: m.originalLine }
+                        : (null as unknown as Position),
+            });
+        });
+
+        return newSourceMap.toJSON();
+    }
+}

tools/sourcemap-tools/src/SourceProcessor.ts

@@ -0,0 +1,13 @@
+export function bytesToUuid(bytes: Buffer) {
+    return (
+        bytes.slice(0, 4).toString('hex') +
+        '-' +
+        bytes.slice(4, 6).toString('hex') +
+        '-' +
+        bytes.slice(6, 8).toString('hex') +
+        '-' +
+        bytes.slice(8, 10).toString('hex') +
+        '-' +
+        bytes.slice(10, 16).toString('hex')
+    );
+}

tools/sourcemap-tools/src/helpers/bytesToUuid.ts

@@ -0,0 +1,7 @@
+import crypto from 'crypto';
+import { bytesToUuid } from './bytesToUuid';
+
+export function stringToUuid(str: string) {
+    const bytes = crypto.createHash('sha1').update(str).digest();
+    return bytesToUuid(bytes);
+}

tools/sourcemap-tools/src/helpers/stringToUuid.ts

@@ -1,3 +1,4 @@
 export * from './ContentAppender';
 export * from './DebugIdGenerator';
 export * from './SourceMapUploader';
+export * from './SourceProcessor';

tools/sourcemap-tools/src/index.ts

@@ -0,0 +1,71 @@
+import { SourceMapConsumer } from 'source-map';
+import { DebugIdGenerator, SOURCEMAP_DEBUG_ID_KEY, SourceProcessor } from '../src';
+
+describe('SourceProcessor', () => {
+    const source = `function foo(){console.log("Hello World!")}foo();`;
+    const sourceMap = {
+        version: 3,
+        file: 'source.js',
+        sources: ['source.js'],
+        names: ['foo', 'console', 'log'],
+        mappings: 'AAAA,SAASA,MACLC,QAAQC,IAAI,cAAc,CAC9B,CAEAF,IAAI',
+    };
+
+    it('should append source snippet to the source on the first line', async () => {
+        const expected = 'APPENDED_SOURCE';
+        const debugIdGenerator = new DebugIdGenerator();
+
+        jest.spyOn(debugIdGenerator, 'generateSourceSnippet').mockReturnValue(expected);
+
+        const sourceProcessor = new SourceProcessor(debugIdGenerator);
+        const result = await sourceProcessor.processSourceAndSourceMap(source, sourceMap);
+
+        expect(result.source).toMatch(new RegExp(`^${expected}\n`));
+    });
+
+    it('should append comment snippet to the source on the last line', async () => {
+        const expected = 'APPENDED_COMMENT';
+        const debugIdGenerator = new DebugIdGenerator();
+
+        jest.spyOn(debugIdGenerator, 'generateSourceComment').mockReturnValue(expected);
+
+        const sourceProcessor = new SourceProcessor(debugIdGenerator);
+        const result = await sourceProcessor.processSourceAndSourceMap(source, sourceMap);
+
+        expect(result.source).toMatch(new RegExp(`\n${expected}$`));
+    });
+
+    it('should return sourcemap from DebugIdGenerator', async () => {
+        const expected = { [SOURCEMAP_DEBUG_ID_KEY]: 'debugId' };
+        const debugIdGenerator = new DebugIdGenerator();
+
+        jest.spyOn(debugIdGenerator, 'addSourceMapKey').mockReturnValue(expected);
+
+        const sourceProcessor = new SourceProcessor(debugIdGenerator);
+        const result = await sourceProcessor.processSourceAndSourceMap(source, sourceMap);
+
+        expect(result.sourceMap).toStrictEqual(expected);
+    });
+
+    it('should offset sourcemap lines by number of newlines in source snippet + 1', async () => {
+        const debugIdGenerator = new DebugIdGenerator();
+        const sourceProcessor = new SourceProcessor(debugIdGenerator);
+        const snippet = 'a\nb\nc\nd';
+        const expectedNewLineCount = (snippet.match(/\n/g)?.length ?? 0) + 1;
+
+        jest.spyOn(debugIdGenerator, 'generateSourceSnippet').mockReturnValue(snippet);
+
+        const unmodifiedConsumer = await new SourceMapConsumer(sourceMap);
+        const expectedPosition = unmodifiedConsumer.originalPositionFor({ line: 1, column: source.indexOf('foo();') });
+
+        const result = await sourceProcessor.processSourceAndSourceMap(source, sourceMap);
+
+        const modifiedConsumer = await new SourceMapConsumer(result.sourceMap);
+        const actualPosition = modifiedConsumer.originalPositionFor({
+            line: 1 + expectedNewLineCount,
+            column: source.indexOf('foo();'),
+        });
+
+        expect(actualPosition).toEqual(expectedPosition);
+    });
+});