From fbb78cbf7f4493d0f4d42563ad32fb4f952a5a3a Mon Sep 17 00:00:00 2001 From: parthkirsan Date: Sat, 23 Aug 2025 02:39:24 +0530 Subject: [PATCH 1/2] initial commit for upload to s3 via lsrs --- src/lib/ctx.ts | 5 ++++ src/lib/httpClient.ts | 50 +++++++++++++++++++++++++++++++++++-- src/lib/schemaValidation.ts | 4 +++ src/lib/snapshotQueue.ts | 30 ++++++++++++++++------ src/tasks/finalizeBuild.ts | 13 +++++++--- src/types.ts | 1 + 6 files changed, 90 insertions(+), 13 deletions(-) diff --git a/src/lib/ctx.ts b/src/lib/ctx.ts index c180746..b500c62 100644 --- a/src/lib/ctx.ts +++ b/src/lib/ctx.ts @@ -24,6 +24,7 @@ export default (options: Record): Context => { let fetchResultsFileObj: string; let buildNameObj: string; let allowDuplicateSnapshotNames: boolean = false; + let useLambdaInternal: boolean = false; try { if (options.config) { config = JSON.parse(fs.readFileSync(options.config, 'utf-8')); @@ -96,6 +97,9 @@ export default (options: Record): Context => { if (config.allowDuplicateSnapshotNames) { allowDuplicateSnapshotNames = true; } + if (config.useLambdaInternal) { + useLambdaInternal = true; + } return { env: env, @@ -122,6 +126,7 @@ export default (options: Record): Context => { userAgent: config.userAgent || '', requestHeaders: config.requestHeaders || {}, allowDuplicateSnapshotNames: allowDuplicateSnapshotNames, + useLambdaInternal: useLambdaInternal, }, uploadFilePath: '', webStaticConfig: [], diff --git a/src/lib/httpClient.ts b/src/lib/httpClient.ts index 6251182..a2e3071 100644 --- a/src/lib/httpClient.ts +++ b/src/lib/httpClient.ts @@ -92,10 +92,10 @@ export default class httpClient { async request(config: AxiosRequestConfig, log: Logger): Promise> { log.debug(`http request: ${config.method} ${config.url}`); - if (config && config.data && !config.data.name && !config.data.snapshot) { + if (config && config.data && !config.data.skipLogging && !config.data.name && !config.data.snapshot) { log.debug(config.data); } - if (config && config.data && config.data.snapshotUuid) { + if (config && config.data && !config.data.skipLogging && config.data.snapshotUuid) { log.debug(config.data); } return this.axiosInstance.request(config) @@ -495,6 +495,38 @@ export default class httpClient { }, ctx.log) } + sendDomToLSRS(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string) { + return this.request({ + url: `/upload/dom`, + method: 'POST', + data: { + buildId: ctx.build.id, + snapshotName: snapshot.name, + snapshotUuid: snapshotUuid, + domContent: snapshot, + skipLogging: true + } + }, ctx.log); + } + + sendDomToLSRSForCaps(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string, capsBuildId: string, capsProjectToken: string) { + return this.request({ + url: `/upload/dom`, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + projectToken: capsProjectToken !== '' ? capsProjectToken : this.projectToken + }, + data: { + buildId: capsBuildId, + snapshotName: snapshot.name, + snapshotUuid: snapshotUuid, + domContent: snapshot, + skipLogging: true + } + }, ctx.log); + } + uploadLogs(ctx: Context, uploadURL: string) { const fileStream = fs.createReadStream(constants.LOG_FILE_PATH); const { size } = fs.statSync(constants.LOG_FILE_PATH); @@ -512,6 +544,20 @@ export default class httpClient { }, ctx.log) } + sendCliLogsToLSRS(ctx: Context) { + const logContent = fs.readFileSync(constants.LOG_FILE_PATH, 'utf-8'); + + return this.request({ + url: `/upload/logs`, + method: 'POST', + data: { + buildId: ctx.build.id, + logContent: logContent, + skipLogging: true + } + }, ctx.log); + } + uploadSnapshotToS3(ctx: Context, uploadURL: string, snapshot: Snapshot) { return this.request({ url: uploadURL, diff --git a/src/lib/schemaValidation.ts b/src/lib/schemaValidation.ts index 18710f7..d093179 100644 --- a/src/lib/schemaValidation.ts +++ b/src/lib/schemaValidation.ts @@ -262,6 +262,10 @@ const ConfigSchema = { allowDuplicateSnapshotNames: { type: "boolean", errorMessage: "Invalid config; allowDuplicateSnapshotNames must be true/false" + }, + useLambdaInternal: { + type: "boolean", + errorMessage: "Invalid config; useLambdaInternal must be true/false" } }, anyOf: [ diff --git a/src/lib/snapshotQueue.ts b/src/lib/snapshotQueue.ts index e6db854..709df79 100644 --- a/src/lib/snapshotQueue.ts +++ b/src/lib/snapshotQueue.ts @@ -5,6 +5,7 @@ import processSnapshot, {prepareSnapshot} from "./processSnapshot.js" import { v4 as uuidv4 } from 'uuid'; import { startPolling, stopTunnelHelper } from "./utils.js"; +const uploadDomToS3ViaEnv = process.env.USE_LAMBDA_INTERNAL || false; export default class Queue { private snapshots: Array = []; private processedSnapshots: Array> = []; @@ -333,10 +334,16 @@ export default class Queue { if (useCapsBuildId) { if (useKafkaFlowCaps) { const snapshotUuid = uuidv4(); - const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUploadCaps(this.ctx, processedSnapshot.name, snapshotUuid, capsBuildId, capsProjectToken); - const uploadUrl = presignedResponse.data.url; - - await this.ctx.client.uploadSnapshotToS3Caps(this.ctx, uploadUrl, processedSnapshot, capsProjectToken) + let uploadDomToS3 = this.ctx.config.useLambdaInternal || uploadDomToS3ViaEnv; + if (!uploadDomToS3) { + this.ctx.log.debug(`Uploading dom to S3 for snapshot using presigned URL for CAPS`); + const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUploadCaps(this.ctx, processedSnapshot.name, snapshotUuid, capsBuildId, capsProjectToken); + const uploadUrl = presignedResponse.data.url; + await this.ctx.client.uploadSnapshotToS3Caps(this.ctx, uploadUrl, processedSnapshot, capsProjectToken) + } else { + this.ctx.log.debug(`Uploading dom to S3 for snapshot using LSRS`); + await this.ctx.client.sendDomToLSRSForCaps(this.ctx, processedSnapshot, snapshotUuid, capsBuildId, capsProjectToken); + } await this.ctx.client.processSnapshotCaps(this.ctx, processedSnapshot, snapshotUuid, capsBuildId, capsProjectToken, discoveryErrors); } else { await this.ctx.client.uploadSnapshotForCaps(this.ctx, processedSnapshot, capsBuildId, capsProjectToken, discoveryErrors); @@ -372,10 +379,17 @@ export default class Queue { } if (this.ctx.build && this.ctx.build.useKafkaFlow) { const snapshotUuid = uuidv4(); - const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUpload(this.ctx, processedSnapshot.name, snapshotUuid); - const uploadUrl = presignedResponse.data.url; - - let snapshotUploadResponse = await this.ctx.client.uploadSnapshotToS3(this.ctx, uploadUrl, processedSnapshot); + let snapshotUploadResponse + let uploadDomToS3 = this.ctx.config.useLambdaInternal || uploadDomToS3ViaEnv; + if (!uploadDomToS3) { + this.ctx.log.debug(`Uploading dom to S3 for snapshot using presigned URL`); + const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUpload(this.ctx, processedSnapshot.name, snapshotUuid); + const uploadUrl = presignedResponse.data.url; + snapshotUploadResponse = await this.ctx.client.uploadSnapshotToS3(this.ctx, uploadUrl, processedSnapshot); + } else { + this.ctx.log.debug(`Uploading dom to S3 for snapshot using LSRS`); + snapshotUploadResponse = await this.ctx.client.sendDomToLSRS(this.ctx, processedSnapshot, snapshotUuid); + } if (!snapshotUploadResponse || Object.keys(snapshotUploadResponse).length === 0) { this.ctx.log.debug(`snapshot failed; Unable to upload dom to S3`); this.processedSnapshots.push({ name: snapshot?.name, error: `snapshot failed; Unable to upload dom to S3` }); diff --git a/src/tasks/finalizeBuild.ts b/src/tasks/finalizeBuild.ts index a5342e1..038d003 100644 --- a/src/tasks/finalizeBuild.ts +++ b/src/tasks/finalizeBuild.ts @@ -7,6 +7,7 @@ import { unlinkSync } from 'fs'; import constants from '../lib/constants.js'; import fs from 'fs'; +const uploadDomToS3ViaEnv = process.env.USE_LAMBDA_INTERNAL || false; export default (ctx: Context): ListrTask => { return { title: `Finalizing build`, @@ -76,9 +77,15 @@ export default (ctx: Context): ListrTask>; allowDuplicateSnapshotNames?: boolean; + useLambdaInternal?: boolean; }; uploadFilePath: string; webStaticConfig: WebStaticConfig; From b108c36a95794839e9fe7a4d1d71511b527a2d1a Mon Sep 17 00:00:00 2001 From: parthkirsan Date: Sat, 23 Aug 2025 08:38:18 +0530 Subject: [PATCH 2/2] 4.1.25 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 125f0e6..33d580d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lambdatest/smartui-cli", - "version": "4.1.24", + "version": "4.1.25", "description": "A command line interface (CLI) to run SmartUI tests on LambdaTest", "files": [ "dist/**/*"