Database support

examples/sdk/node/.gitignore

@@ -0,0 +1,2 @@
+./database
+./src/consts.ts

examples/sdk/node/src/index.ts

@@ -11,7 +11,7 @@ const reader = readline.createInterface({
 
 const client = BacktraceClient.builder({
     url: SUBMISSION_URL,
-    attachments: [path.join(path.dirname(process.cwd()), 'samplefile.txt')],
+    attachments: [path.join(process.cwd(), 'samplefile.txt')],
     rateLimit: 5,
     userAttributes: {
         'custom-attribute': 'test',
@@ -20,6 +20,10 @@ const client = BacktraceClient.builder({
             prop2: 123,
         },
     },
+    database: {
+        enabled: true,
+        path: path.join(process.cwd(), 'database'),
+    },
 }).build();
 
 console.log('Welcome to the @Backtrace/node demo');

packages/node/src/BacktraceClient.ts

@@ -11,6 +11,7 @@ import { AGENT } from './agentDefinition';
 import { BacktraceConfiguration } from './BacktraceConfiguration';
 import { BacktraceClientBuilder } from './builder/BacktraceClientBuilder';
 import { NodeOptionReader } from './common/NodeOptionReader';
+import { BacktraceDatabaseFileStorageProvider } from './database/BacktraceDatabaseFileStorageProvider';
 
 export class BacktraceClient extends BacktraceCoreClient {
     constructor(
@@ -26,6 +27,8 @@ export class BacktraceClient extends BacktraceCoreClient {
             undefined,
             undefined,
             new VariableDebugIdMapProvider(global as DebugIdContainer),
+            undefined,
+            BacktraceDatabaseFileStorageProvider.createIfValid(options.database),
         );
 
         this.captureUnhandledErrors(options.captureUnhandledErrors, options.captureUnhandledPromiseRejections);

packages/node/src/attachment/BacktraceFileAttachment.ts

@@ -5,14 +5,14 @@ import { Readable } from 'stream';
 
 export class BacktraceFileAttachment implements BacktraceAttachment<Readable> {
     public readonly name: string;
-    constructor(private readonly _filePath: string) {
-        this.name = path.basename(this._filePath);
+    constructor(public readonly filePath: string) {
+        this.name = path.basename(this.filePath);
     }
 
     public get(): fs.ReadStream | undefined {
-        if (!fs.existsSync(this._filePath)) {
+        if (!fs.existsSync(this.filePath)) {
             return undefined;
         }
-        return fs.createReadStream(this._filePath);
+        return fs.createReadStream(this.filePath);
     }
 }

packages/node/src/database/BacktraceDatabaseFileRecord.ts

@@ -0,0 +1,40 @@
+import { BacktraceData, BacktraceDatabaseRecord } from '@backtrace/sdk-core';
+import { BacktraceFileAttachment } from '../attachment';
+
+export class BacktraceDatabaseFileRecord implements BacktraceDatabaseRecord {
+    public readonly data: BacktraceData;
+    public readonly id: string;
+    public readonly count: number;
+    public readonly hash: string;
+    public locked: boolean;
+
+    private constructor(record: BacktraceDatabaseRecord, public readonly attachments: BacktraceFileAttachment[]) {
+        this.data = record.data;
+        this.id = record.id;
+        this.count = record.count;
+        this.hash = record.hash;
+        // make sure the database record stored in the database directory
+        // is never locked. By doing this, we want to be sure once we load
+        // the record once again, the record will be available for future usage
+        this.locked = false;
+    }
+
+    public static fromRecord(record: BacktraceDatabaseRecord) {
+        return new BacktraceDatabaseFileRecord(
+            record,
+            record.attachments.filter((n) => n instanceof BacktraceFileAttachment) as BacktraceFileAttachment[],
+        );
+    }
+
+    public static fromJson(json: string): BacktraceDatabaseFileRecord | undefined {
+        try {
+            const record = JSON.parse(json) as BacktraceDatabaseFileRecord;
+            const attachments = record.attachments
+                ? record.attachments.map((n) => new BacktraceFileAttachment(n.filePath))
+                : [];
+            return new BacktraceDatabaseFileRecord(record, attachments);
+        } catch {
+            return undefined;
+        }
+    }
+}

packages/node/src/database/BacktraceDatabaseFileStorageProvider.ts

@@ -0,0 +1,118 @@
+import {
+    BacktraceDatabaseConfiguration,
+    BacktraceDatabaseRecord,
+    BacktraceDatabaseStorageProvider,
+} from '@backtrace/sdk-core';
+import fs from 'fs';
+import * as fsPromise from 'fs/promises';
+import path from 'path';
+import { BacktraceDatabaseFileRecord } from './BacktraceDatabaseFileRecord';
+export class BacktraceDatabaseFileStorageProvider implements BacktraceDatabaseStorageProvider {
+    private _enabled = true;
+
+    private readonly RECORD_SUFFIX = '-record.json';
+    private constructor(private readonly _path: string, private readonly _createDatabaseDirectory: boolean = false) {}
+
+    /**
+     * Create a provider if provided options are valid
+     * @param options database configuration
+     * @returns database file storage provider
+     */
+    public static createIfValid(
+        options?: BacktraceDatabaseConfiguration,
+    ): BacktraceDatabaseFileStorageProvider | undefined {
+        if (!options) {
+            return undefined;
+        }
+        if (!options.enabled) {
+            return undefined;
+        }
+
+        if (options.enabled && !options.path) {
+            throw new Error(
+                'Missing mandatory path to the database. Please define the database.path option in the configuration.',
+            );
+        }
+        return new BacktraceDatabaseFileStorageProvider(options.path, options.createDatabaseDirectory);
+    }
+
+    public start(): boolean {
+        // make sure by mistake we don't create anything or start any operation
+        if (this._enabled === false) {
+            return false;
+        }
+
+        const databaseDirectoryExists = fs.existsSync(this._path);
+        if (this._createDatabaseDirectory === false) {
+            return databaseDirectoryExists;
+        }
+        if (databaseDirectoryExists) {
+            return true;
+        }
+        fs.mkdirSync(this._path, { recursive: true });
+        return true;
+    }
+
+    public delete(record: BacktraceDatabaseRecord): boolean {
+        const recordPath = this.getRecordPath(record.id);
+        return this.unlinkRecord(recordPath);
+    }
+
+    public add(record: BacktraceDatabaseRecord): boolean {
+        const recordPath = this.getRecordPath(record.id);
+        try {
+            fs.writeFileSync(recordPath, JSON.stringify(BacktraceDatabaseFileRecord.fromRecord(record)), {
+                encoding: 'utf8',
+            });
+            return true;
+        } catch {
+            return false;
+        }
+    }
+
+    public async get(): Promise<BacktraceDatabaseRecord[]> {
+        const databaseFiles = await fsPromise.readdir(this._path, {
+            encoding: 'utf8',
+            withFileTypes: true,
+        });
+
+        const recordNames = databaseFiles
+            .filter((file) => file.isFile() && file.name.endsWith(this.RECORD_SUFFIX))
+            .map((n) => n.name);
+
+        const records: BacktraceDatabaseRecord[] = [];
+        for (const recordName of recordNames) {
+            const recordPath = path.join(this._path, recordName);
+            try {
+                const recordJson = await fsPromise.readFile(recordPath, 'utf8');
+                const record = BacktraceDatabaseFileRecord.fromJson(recordJson);
+                if (!record) {
+                    await fsPromise.unlink(recordPath);
+                    continue;
+                }
+                records.push(record);
+            } catch {
+                this.unlinkRecord(recordPath);
+            }
+        }
+
+        return records;
+    }
+
+    private unlinkRecord(recordPath: string): boolean {
+        if (!fs.existsSync(recordPath)) {
+            return false;
+        }
+
+        try {
+            fs.unlinkSync(recordPath);
+            return true;
+        } catch {
+            return false;
+        }
+    }
+
+    private getRecordPath(id: string): string {
+        return path.join(this._path, `${id}${this.RECORD_SUFFIX}`);
+    }
+}

packages/sdk-core/src/BacktraceCoreClient.ts

@@ -1,6 +1,8 @@
 import {
     BacktraceAttachment,
     BacktraceAttributeProvider,
+    BacktraceDatabaseRecord,
+    BacktraceDatabaseStorageProvider,
     BacktraceSessionProvider,
     BacktraceStackTraceConverter,
     DebugIdMapProvider,
@@ -18,6 +20,7 @@ import { BacktraceBreadcrumbs, BreadcrumbsSetup } from './modules/breadcrumbs';
 import { BreadcrumbsManager } from './modules/breadcrumbs/BreadcrumbsManager';
 import { V8StackTraceConverter } from './modules/converter/V8StackTraceConverter';
 import { BacktraceDataBuilder } from './modules/data/BacktraceDataBuilder';
+import { BacktraceDatabase } from './modules/database/BacktraceDatabase';
 import { BacktraceMetrics } from './modules/metrics/BacktraceMetrics';
 import { MetricsBuilder } from './modules/metrics/MetricsBuilder';
 import { SingleSessionProvider } from './modules/metrics/SingleSessionProvider';
@@ -65,6 +68,13 @@ export abstract class BacktraceCoreClient {
         return this.breadcrumbsManager;
     }
 
+    /**
+     * Report database used by the client
+     */
+    public get database(): BacktraceDatabase | undefined {
+        return this._database;
+    }
+
     /**
      * Client cached attachments
      */
@@ -76,6 +86,7 @@ export abstract class BacktraceCoreClient {
     private readonly _rateLimitWatcher: RateLimitWatcher;
     private readonly _attributeProvider: AttributeManager;
     private readonly _metrics?: BacktraceMetrics;
+    private readonly _database?: BacktraceDatabase;
 
     protected constructor(
         protected readonly options: BacktraceConfiguration,
@@ -86,6 +97,7 @@ export abstract class BacktraceCoreClient {
         private readonly _sessionProvider: BacktraceSessionProvider = new SingleSessionProvider(),
         debugIdMapProvider?: DebugIdMapProvider,
         breadcrumbsSetup?: BreadcrumbsSetup,
+        databaseStorageProvider?: BacktraceDatabaseStorageProvider,
     ) {
         this._dataBuilder = new BacktraceDataBuilder(
             this._sdkOptions,
@@ -94,7 +106,6 @@ export abstract class BacktraceCoreClient {
         );
 
         this._reportSubmission = new BacktraceReportSubmission(options, requestHandler);
-        this._rateLimitWatcher = new RateLimitWatcher(options.rateLimit);
         this._attributeProvider = new AttributeManager([
             new ClientAttributeProvider(
                 _sdkOptions.agent,
@@ -105,6 +116,18 @@ export abstract class BacktraceCoreClient {
             ...(attributeProviders ?? []),
         ]);
         this.attachments = options.attachments ?? [];
+
+        if (databaseStorageProvider && options?.database?.enabled === true) {
+            this._database = new BacktraceDatabase(
+                this.options.database,
+                databaseStorageProvider,
+                this._reportSubmission,
+            );
+            this._database.start();
+        }
+
+        this._rateLimitWatcher = new RateLimitWatcher(options.rateLimit);
+
         const metrics = new MetricsBuilder(options, _sessionProvider, this._attributeProvider, requestHandler).build();
         if (metrics) {
             this._metrics = metrics;
@@ -179,7 +202,35 @@ export abstract class BacktraceCoreClient {
             return;
         }
 
-        await this._reportSubmission.send(backtraceData, this.generateSubmissionAttachments(report, reportAttachments));
+        const submissionAttachments = this.generateSubmissionAttachments(report, reportAttachments);
+        const record = this.addToDatabase(backtraceData, submissionAttachments);
+
+        const submissionResult = await this._reportSubmission.send(backtraceData, submissionAttachments);
+        if (!record) {
+            return;
+        }
+        record.locked = false;
+        if (submissionResult.status === 'Ok') {
+            this._database?.remove(record);
+        }
+    }
+
+    private addToDatabase(
+        data: BacktraceData,
+        attachments: BacktraceAttachment[],
+    ): BacktraceDatabaseRecord | undefined {
+        if (!this._database) {
+            return undefined;
+        }
+
+        const record = this._database.add(data, attachments);
+
+        if (!record || record.locked || record.count !== 1) {
+            return undefined;
+        }
+
+        record.locked = true;
+        return record;
     }
 
     private generateSubmissionData(report: BacktraceReport): BacktraceData | undefined {

packages/sdk-core/src/index.ts

@@ -5,11 +5,13 @@ export * from './common/IdGenerator';
 export * from './model/attachment';
 export * from './model/configuration/BacktraceConfiguration';
 export * from './model/configuration/BacktraceDatabaseConfiguration';
+export * from './model/data/BacktraceData';
 export * from './model/http';
 export * from './model/report/BacktraceErrorType';
 export * from './model/report/BacktraceReport';
 export * from './modules/attribute/BacktraceAttributeProvider';
 export * from './modules/breadcrumbs';
 export * from './modules/converter';
+export * from './modules/database';
 export * from './modules/metrics/BacktraceSessionProvider';
 export * from './sourcemaps/index';

packages/sdk-core/src/model/configuration/BacktraceDatabaseConfiguration.ts

@@ -18,7 +18,7 @@ export enum DeduplicationStrategy {
     /**
      * Aggregates by faulting callstack, exception type, and exception message
      */
-    All = ~(~0 << 4),
+    All = ~(~0 << 4) - 1,
 }
 export interface EnabledBacktraceDatabaseConfiguration {
     /**
@@ -54,17 +54,12 @@ export interface EnabledBacktraceDatabaseConfiguration {
      */
     maximumNumberOfRecords?: number;
 
-    /**
-     * The maximum database size in MB. When the limit is reached, the oldest reports are removed.
-     * If the value is equal to '0', then no limit is set.
-     * The default value is 0 (unlimited)
-     */
-    maximumDatabaseSizeInMb?: number;
     /**
      * The amount of time (in ms) to wait between retries if the database is unable to send a report.
      * The default value is 60 000
      */
     retryInterval?: number;
+
     /**
      * The maximum number of retries to attempt if the database is unable to send a report.
      * The default value is 3