Upload Dom and Cli logs via LSRS

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lambdatest/smartui-cli",
-  "version": "4.1.24",
+  "version": "4.1.25",
   "description": "A command line interface (CLI) to run SmartUI tests on LambdaTest",
   "files": [
     "dist/**/*"

src/lib/ctx.ts

@@ -24,6 +24,7 @@ export default (options: Record<string, string>): Context => {
     let fetchResultsFileObj: string;
     let buildNameObj: string;
     let allowDuplicateSnapshotNames: boolean = false;
+    let useLambdaInternal: boolean = false;
     try {
         if (options.config) {
             config = JSON.parse(fs.readFileSync(options.config, 'utf-8'));
@@ -96,6 +97,9 @@ export default (options: Record<string, string>): Context => {
     if (config.allowDuplicateSnapshotNames) {
         allowDuplicateSnapshotNames = true;
     }
+    if (config.useLambdaInternal) {
+        useLambdaInternal = true;
+    }
 
     return {
         env: env,
@@ -122,6 +126,7 @@ export default (options: Record<string, string>): Context => {
             userAgent: config.userAgent || '',
             requestHeaders: config.requestHeaders || {},
             allowDuplicateSnapshotNames: allowDuplicateSnapshotNames,
+            useLambdaInternal: useLambdaInternal,
         },
         uploadFilePath: '',
         webStaticConfig: [],

src/lib/httpClient.ts

@@ -92,10 +92,10 @@ export default class httpClient {
 
     async request(config: AxiosRequestConfig, log: Logger): Promise<Record<string, any>> {
         log.debug(`http request: ${config.method} ${config.url}`);
-        if (config && config.data && !config.data.name && !config.data.snapshot) {
+        if (config && config.data && !config.data.skipLogging && !config.data.name && !config.data.snapshot) {
             log.debug(config.data);
         }
-        if (config && config.data && config.data.snapshotUuid) {
+        if (config && config.data && !config.data.skipLogging && config.data.snapshotUuid) {
             log.debug(config.data);
         }
         return this.axiosInstance.request(config)
@@ -495,6 +495,38 @@ export default class httpClient {
         }, ctx.log)
     }
 
+    sendDomToLSRS(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string) {
+        return this.request({
+            url: `/upload/dom`,
+            method: 'POST',
+            data: {
+                buildId: ctx.build.id,
+                snapshotName: snapshot.name,
+                snapshotUuid: snapshotUuid,
+                domContent: snapshot,
+                skipLogging: true
+            }
+        }, ctx.log);
+    }
+
+    sendDomToLSRSForCaps(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string, capsBuildId: string, capsProjectToken: string) {
+        return this.request({
+            url: `/upload/dom`,
+            method: 'POST',
+            headers: { 
+                'Content-Type': 'application/json',
+                projectToken: capsProjectToken !== '' ? capsProjectToken : this.projectToken
+            },
+            data: {
+                buildId: capsBuildId,
+                snapshotName: snapshot.name,
+                snapshotUuid: snapshotUuid,
+                domContent: snapshot,
+                skipLogging: true
+            }
+        }, ctx.log);
+    }
+
     uploadLogs(ctx: Context, uploadURL: string) {
         const fileStream = fs.createReadStream(constants.LOG_FILE_PATH);
         const { size } = fs.statSync(constants.LOG_FILE_PATH);
@@ -512,6 +544,20 @@ export default class httpClient {
         }, ctx.log)
     }
 
+    sendCliLogsToLSRS(ctx: Context) {
+        const logContent = fs.readFileSync(constants.LOG_FILE_PATH, 'utf-8');
+
+        return this.request({
+            url: `/upload/logs`,
+            method: 'POST',
+            data: {
+                buildId: ctx.build.id,
+                logContent: logContent,
+                skipLogging: true
+            }
+        }, ctx.log);
+    }
+
     uploadSnapshotToS3(ctx: Context, uploadURL: string, snapshot: Snapshot) {
         return this.request({
             url: uploadURL,

src/lib/schemaValidation.ts

@@ -262,6 +262,10 @@ const ConfigSchema = {
         allowDuplicateSnapshotNames: {
             type: "boolean",
             errorMessage: "Invalid config; allowDuplicateSnapshotNames must be true/false"
+        },
+        useLambdaInternal: {
+            type: "boolean",
+            errorMessage: "Invalid config; useLambdaInternal must be true/false"
         }
     },
     anyOf: [

src/lib/snapshotQueue.ts

@@ -5,6 +5,7 @@ import processSnapshot, {prepareSnapshot} from "./processSnapshot.js"
 import { v4 as uuidv4 } from 'uuid';
 import { startPolling, stopTunnelHelper } from "./utils.js";
 
+const uploadDomToS3ViaEnv = process.env.USE_LAMBDA_INTERNAL || false;
 export default class Queue {
     private snapshots: Array<Snapshot> = [];
     private processedSnapshots: Array<Record<string, any>> = [];
@@ -333,10 +334,16 @@ export default class Queue {
                     if (useCapsBuildId) {
                         if (useKafkaFlowCaps) {
                             const snapshotUuid = uuidv4();
-                            const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUploadCaps(this.ctx, processedSnapshot.name, snapshotUuid, capsBuildId, capsProjectToken);
-                            const uploadUrl = presignedResponse.data.url;
-
-                            await this.ctx.client.uploadSnapshotToS3Caps(this.ctx, uploadUrl, processedSnapshot, capsProjectToken)
+                            let uploadDomToS3 = this.ctx.config.useLambdaInternal || uploadDomToS3ViaEnv;
+                            if (!uploadDomToS3) {
+                                this.ctx.log.debug(`Uploading dom to S3 for snapshot using presigned URL for CAPS`);
+                                const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUploadCaps(this.ctx, processedSnapshot.name, snapshotUuid, capsBuildId, capsProjectToken);
+                                const uploadUrl = presignedResponse.data.url;
+                                await this.ctx.client.uploadSnapshotToS3Caps(this.ctx, uploadUrl, processedSnapshot, capsProjectToken)
+                            } else {
+                                this.ctx.log.debug(`Uploading dom to S3 for snapshot using LSRS`);
+                                await this.ctx.client.sendDomToLSRSForCaps(this.ctx, processedSnapshot, snapshotUuid, capsBuildId, capsProjectToken);
+                            }
                             await this.ctx.client.processSnapshotCaps(this.ctx, processedSnapshot, snapshotUuid, capsBuildId, capsProjectToken, discoveryErrors);
                         } else {
                             await this.ctx.client.uploadSnapshotForCaps(this.ctx, processedSnapshot, capsBuildId, capsProjectToken, discoveryErrors);
@@ -372,10 +379,17 @@ export default class Queue {
                         }
                         if (this.ctx.build && this.ctx.build.useKafkaFlow) {
                             const snapshotUuid = uuidv4();
-                            const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUpload(this.ctx, processedSnapshot.name, snapshotUuid);
-                            const uploadUrl = presignedResponse.data.url;
-
-                            let snapshotUploadResponse = await this.ctx.client.uploadSnapshotToS3(this.ctx, uploadUrl, processedSnapshot);
+                            let snapshotUploadResponse
+                            let uploadDomToS3 = this.ctx.config.useLambdaInternal || uploadDomToS3ViaEnv;
+                            if (!uploadDomToS3) {
+                                this.ctx.log.debug(`Uploading dom to S3 for snapshot using presigned URL`);
+                                const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUpload(this.ctx, processedSnapshot.name, snapshotUuid);
+                                const uploadUrl = presignedResponse.data.url;
+                                snapshotUploadResponse = await this.ctx.client.uploadSnapshotToS3(this.ctx, uploadUrl, processedSnapshot);
+                            } else {
+                                this.ctx.log.debug(`Uploading dom to S3 for snapshot using LSRS`);
+                                snapshotUploadResponse = await this.ctx.client.sendDomToLSRS(this.ctx, processedSnapshot, snapshotUuid);
+                            }
                             if (!snapshotUploadResponse || Object.keys(snapshotUploadResponse).length === 0) {
                                 this.ctx.log.debug(`snapshot failed; Unable to upload dom to S3`);
                                 this.processedSnapshots.push({ name: snapshot?.name, error: `snapshot failed; Unable to upload dom to S3` });

src/tasks/finalizeBuild.ts

@@ -7,6 +7,7 @@ import { unlinkSync } from 'fs';
 import constants from '../lib/constants.js';
 import fs from 'fs';
 
+const uploadDomToS3ViaEnv = process.env.USE_LAMBDA_INTERNAL || false;
 export default (ctx: Context): ListrTask<Context, ListrRendererFactory, ListrRendererFactory>  =>  {
     return {
         title: `Finalizing build`,
@@ -76,9 +77,15 @@ export default (ctx: Context): ListrTask<Context, ListrRendererFactory, ListrRen
                 await ctx.server?.close();
                 ctx.log.debug(`Closed server`);
                 if (ctx.isSnapshotCaptured) {
-                    ctx.log.debug(`Log file to be uploaded`)
-                    let resp = await ctx.client.getS3PreSignedURL(ctx);
-                    await ctx.client.uploadLogs(ctx, resp.data.url);
+                    let uploadCLILogsToS3 = ctx.config.useLambdaInternal || uploadDomToS3ViaEnv;
+                    if (!uploadCLILogsToS3) {
+                        ctx.log.debug(`Log file to be uploaded`)
+                        let resp = await ctx.client.getS3PreSignedURL(ctx);
+                        await ctx.client.uploadLogs(ctx, resp.data.url);
+                    } else {
+                        ctx.log.debug(`Log file to be uploaded via LSRS`)
+                        let resp = ctx.client.sendCliLogsToLSRS(ctx);
+                    }
                 }
             } catch (error: any) {
                 ctx.log.debug(error);

src/types.ts

@@ -37,6 +37,7 @@ export interface Context {
         userAgent?: string;
         requestHeaders?: Array<Record<string, string>>;
         allowDuplicateSnapshotNames?: boolean;
+        useLambdaInternal?: boolean;
     };
     uploadFilePath: string;
     webStaticConfig: WebStaticConfig;