Release PR version 4.1.26 Sync functionality in SDK

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lambdatest/smartui-cli",
-  "version": "4.1.25",
+  "version": "4.1.26",
   "description": "A command line interface (CLI) to run SmartUI tests on LambdaTest",
   "files": [
     "dist/**/*"

src/lib/ctx.ts

@@ -101,6 +101,11 @@ export default (options: Record<string, string>): Context => {
         useLambdaInternal = true;
     }
 
+    //if config.waitForPageRender has value and if its less than 30000 then make it to 30000 default
+    if (config.waitForPageRender && config.waitForPageRender < 30000) {
+        config.waitForPageRender = 30000;
+    }
+
     return {
         env: env,
         log: logger,

src/lib/httpClient.ts

@@ -321,7 +321,7 @@ export default class httpClient {
         }, ctx.log)
     }
 
-    processSnapshot(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string,  discoveryErrors: DiscoveryErrors) {
+    processSnapshot(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string,  discoveryErrors: DiscoveryErrors, variantCount: number, sync: boolean = false) {
         return this.request({
             url: `/build/${ctx.build.id}/snapshot`,
             method: 'POST',
@@ -330,12 +330,14 @@ export default class httpClient {
                 name: snapshot.name,
                 url: snapshot.url,
                 snapshotUuid: snapshotUuid,
+                variantCount: variantCount,
                 test: {
                     type: ctx.testType,
                     source: 'cli'
                 },
                 doRemoteDiscovery: snapshot.options.doRemoteDiscovery,
                 discoveryErrors: discoveryErrors,
+                sync: sync
             }
         }, ctx.log)
     }
@@ -623,4 +625,14 @@ export default class httpClient {
             data: requestData
         }, ctx.log)
     }
+
+    getSnapshotStatus(snapshotName: string, snapshotUuid: string, ctx: Context): Promise<Record<string, any>> {
+        return this.request({
+            url: `/snapshot/status?buildId=${ctx.build.id}&snapshotName=${snapshotName}&snapshotUUID=${snapshotUuid}`,
+            method: 'GET',
+            headers: {
+                'Content-Type': 'application/json',
+            }
+        }, ctx.log);
+    }
 }

src/lib/schemaValidation.ts

@@ -501,6 +501,18 @@ const SnapshotSchema: JSONSchemaType<Snapshot> = {
                 sessionId: {
                     type: "string",
                     errorMessage: "Invalid snapshot options; sessionId must be a string"
+                },
+                contextId: {
+                    type: "string",
+                    errorMessage: "Invalid snapshot options; contextId must be a string"
+                },
+                sync: {
+                    type: "boolean",
+                    errorMessage: "Invalid snapshot options; sync must be a boolean"
+                },
+                timeout: {
+                    type: "number",
+                    errorMessage: "Invalid snapshot options; timeout must be a number"
                 }
             },
             additionalProperties: false

src/lib/server.ts

@@ -7,6 +7,7 @@ import { validateSnapshot } from './schemaValidation.js'
 import { pingIntervalId } from './utils.js';
 import { startPolling } from './utils.js';
 
+const uploadDomToS3ViaEnv = process.env.USE_LAMBDA_INTERNAL || false;
 export default async (ctx: Context): Promise<FastifyInstance<Server, IncomingMessage, ServerResponse>> => {
 
 	const server: FastifyInstance<Server, IncomingMessage, ServerResponse> = fastify({
@@ -41,6 +42,7 @@ export default async (ctx: Context): Promise<FastifyInstance<Server, IncomingMes
 			// Fetch sessionId from snapshot options if present
 			const sessionId = snapshot?.options?.sessionId;
 			let capsBuildId = ''
+			const contextId = snapshot?.options?.contextId;
 
 			if (sessionId) {
 				// Check if sessionId exists in the map
@@ -71,7 +73,23 @@ export default async (ctx: Context): Promise<FastifyInstance<Server, IncomingMes
 			}
 
 			ctx.testType = testType;
-			ctx.snapshotQueue?.enqueue(snapshot);
+
+			if (contextId && !ctx.contextToSnapshotMap) {
+				ctx.contextToSnapshotMap = new Map();
+				ctx.log.debug(`Initialized empty context mapping map for contextId: ${contextId}`);
+			}
+
+			if (contextId && ctx.contextToSnapshotMap) {
+				ctx.contextToSnapshotMap.set(contextId, 0);
+				ctx.log.debug(`Marking contextId as captured and added to queue: ${contextId}`);
+			}
+
+			if(contextId){
+				ctx.snapshotQueue?.enqueueFront(snapshot);
+			}else{
+				ctx.snapshotQueue?.enqueue(snapshot);	
+			}
+
 			ctx.isSnapshotCaptured = true;
 			replyCode = 200;
 			replyBody = { data: { message: "success", warnings: [] }};
@@ -105,8 +123,16 @@ export default async (ctx: Context): Promise<FastifyInstance<Server, IncomingMes
 			if (ctx.server){
 				ctx.server.close();
 			}
-			let resp = await ctx.client.getS3PreSignedURL(ctx);
-            await ctx.client.uploadLogs(ctx, resp.data.url);
+
+			let uploadCLILogsToS3 = ctx?.config?.useLambdaInternal || uploadDomToS3ViaEnv;
+			if (!uploadCLILogsToS3) {
+				ctx.log.debug(`Log file to be uploaded`)
+				let resp = await ctx.client.getS3PreSignedURL(ctx);
+				await ctx.client.uploadLogs(ctx, resp.data.url);
+			} else {
+				ctx.log.debug(`Log file to be uploaded via LSRS`)
+				let resp = ctx.client.sendCliLogsToLSRS(ctx);
+			}
 
 			if (pingIntervalId !== null) {
 				clearInterval(pingIntervalId);
@@ -130,7 +156,113 @@ export default async (ctx: Context): Promise<FastifyInstance<Server, IncomingMes
 		reply.code(200).send({ status: 'Server is running', version: ctx.cliVersion });
 	});
 
-
+	// Get snapshot status
+	server.get('/snapshot/status', opts, async (request, reply) => {
+		let replyCode: number;
+		let replyBody: Record<string, any>;
+
+
+		try {
+			ctx.log.debug(`request.query : ${JSON.stringify(request.query)}`);
+			const { contextId, pollTimeout, snapshotName } = request.query as { contextId: string, pollTimeout: number, snapshotName: string };
+			if (!contextId || !snapshotName) {
+				throw new Error('contextId and snapshotName are required parameters');
+			}
+
+			const timeoutDuration = pollTimeout*1000 || 30000; 
+
+			// Check if we have stored snapshot status for this contextId
+			if (ctx.contextToSnapshotMap?.has(contextId)) {
+				let contextStatus = ctx.contextToSnapshotMap.get(contextId);
+
+				while (contextStatus==0) {
+					// Wait 5 seconds before next check
+					await new Promise(resolve => setTimeout(resolve, 5000));
+
+					contextStatus = ctx.contextToSnapshotMap.get(contextId);
+				}
+
+				if(contextStatus==2){
+					throw new Error("Snapshot Failed");
+				}
+
+				ctx.log.debug("Snapshot uploaded successfully");
+
+				// Poll external API until it returns 200 or timeout is reached
+				let lastExternalResponse: any = null; 
+				const startTime = Date.now(); 
+
+				while (true) {
+					try {
+						const externalResponse = await ctx.client.getSnapshotStatus(
+							snapshotName,
+							contextId,
+							ctx
+						);
+
+						lastExternalResponse = externalResponse;
+
+						if (externalResponse.statusCode === 200) {
+							replyCode = 200;
+							replyBody = externalResponse.data;
+							return reply.code(replyCode).send(replyBody);
+						} else if (externalResponse.statusCode === 202 ) {
+							replyBody= externalResponse.data;
+							ctx.log.debug(`External API attempt: Still processing, Pending Screenshots ${externalResponse.snapshotCount}`);
+							await new Promise(resolve => setTimeout(resolve, 5000));
+						}else if(externalResponse.statusCode===404){
+							ctx.log.debug(`Snapshot still processing, not uploaded`);
+							await new Promise(resolve => setTimeout(resolve, 5000));
+						}else {
+							ctx.log.debug(`Unexpected response from external API: ${JSON.stringify(externalResponse)}`);
+							replyCode = 500;
+							replyBody = { 
+								error: { 
+									message: `Unexpected response from external API: ${externalResponse.statusCode}`,
+									externalApiStatus: externalResponse.statusCode
+								}
+							};
+							return reply.code(replyCode).send(replyBody);
+						}
+
+						ctx.log.debug(`timeoutDuration: ${timeoutDuration}`);
+						ctx.log.debug(`Time passed: ${Date.now() - startTime}`);
+
+						if (Date.now() - startTime > timeoutDuration) {
+							replyCode = 202; 
+							replyBody = {
+								data: {
+									message: 'Request timed out-> Snapshot still processing'
+								} 
+							};
+							return reply.code(replyCode).send(replyBody);
+						}
+
+					} catch (externalApiError: any) {
+						ctx.log.debug(`External API call failed: ${externalApiError.message}`);
+						replyCode = 500;
+						replyBody = { 
+							error: { 
+								message: `External API call failed: ${externalApiError.message}`
+							}
+						};
+						return reply.code(replyCode).send(replyBody);
+					}
+				}
+			} else {
+				// No snapshot found for this contextId
+				replyCode = 404;
+				replyBody = { error: { message: `No snapshot found for contextId: ${contextId}` } };
+				return reply.code(replyCode).send(replyBody);
+			}
+		} catch (error: any) {
+			ctx.log.debug(`snapshot status failed; ${error}`);
+			replyCode = 500;
+			replyBody = { error: { message: error.message } };
+			return reply.code(replyCode).send(replyBody);
+		}
+	});
+
 
 	await server.listen({ port: ctx.options.port });
 	// store server's address for SDK

src/lib/snapshotQueue.ts

@@ -3,7 +3,7 @@ import { Snapshot, Context } from "../types.js";
 import constants from "./constants.js";
 import processSnapshot, {prepareSnapshot} from "./processSnapshot.js"
 import { v4 as uuidv4 } from 'uuid';
-import { startPolling, stopTunnelHelper } from "./utils.js";
+import { startPolling, stopTunnelHelper, calculateVariantCountFromSnapshot } from "./utils.js";
 
 const uploadDomToS3ViaEnv = process.env.USE_LAMBDA_INTERNAL || false;
 export default class Queue {
@@ -29,6 +29,16 @@ export default class Queue {
         }
     }
 
+    enqueueFront(item: Snapshot): void {
+        this.snapshots.unshift(item);
+        if (!this.ctx.config.delayedUpload) {
+            if (!this.processing) {
+                this.processing = true;
+                this.processNext();
+            }
+        }
+    }
+
     startProcessingfunc(): void {
         if (!this.processing) {
             this.processing = true;
@@ -129,6 +139,8 @@ export default class Queue {
         return drop;
     }
 
+
+
     private filterVariants(snapshot: Snapshot, config: any): boolean {
         let allVariantsDropped = true;
 
@@ -273,6 +285,7 @@ export default class Queue {
                 this.processingSnapshot = snapshot?.name;
                 let drop = false;
 
+
                 if (this.ctx.isStartExec && !this.ctx.config.tunnel) {
                     this.ctx.log.info(`Processing Snapshot: ${snapshot?.name}`);
                 }
@@ -332,6 +345,7 @@ export default class Queue {
 
 
                     if (useCapsBuildId) {
+                        this.ctx.log.info(`Using cached buildId: ${capsBuildId}`);
                         if (useKafkaFlowCaps) {
                             const snapshotUuid = uuidv4();
                             let uploadDomToS3 = this.ctx.config.useLambdaInternal || uploadDomToS3ViaEnv;
@@ -378,18 +392,23 @@ export default class Queue {
                             }
                         }
                         if (this.ctx.build && this.ctx.build.useKafkaFlow) {
-                            const snapshotUuid = uuidv4();
+                            let snapshotUuid = uuidv4();
                             let snapshotUploadResponse
-                            let uploadDomToS3 = this.ctx.config.useLambdaInternal || uploadDomToS3ViaEnv;
-                            if (!uploadDomToS3) {
+                            if (snapshot?.options?.contextId && this.ctx.contextToSnapshotMap?.has(snapshot.options.contextId)) {
+                                snapshotUuid = snapshot.options.contextId;
+                             }
+                             let uploadDomToS3 = this.ctx.config.useLambdaInternal || uploadDomToS3ViaEnv;
+                             if (!uploadDomToS3) {
                                 this.ctx.log.debug(`Uploading dom to S3 for snapshot using presigned URL`);
                                 const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUpload(this.ctx, processedSnapshot.name, snapshotUuid);
                                 const uploadUrl = presignedResponse.data.url;
                                 snapshotUploadResponse = await this.ctx.client.uploadSnapshotToS3(this.ctx, uploadUrl, processedSnapshot);
-                            } else {
+                             } else {
                                 this.ctx.log.debug(`Uploading dom to S3 for snapshot using LSRS`);
                                 snapshotUploadResponse = await this.ctx.client.sendDomToLSRS(this.ctx, processedSnapshot, snapshotUuid);
-                            }
+                             }
+
+
                             if (!snapshotUploadResponse || Object.keys(snapshotUploadResponse).length === 0) {
                                 this.ctx.log.debug(`snapshot failed; Unable to upload dom to S3`);
                                 this.processedSnapshots.push({ name: snapshot?.name, error: `snapshot failed; Unable to upload dom to S3` });
@@ -403,11 +422,19 @@ export default class Queue {
                                         this.ctx.log.debug(`Closed browser context for snapshot ${snapshot.name}`);
                                     }
                                 }
+                                if(snapshot?.options?.contextId){
+                                    this.ctx.contextToSnapshotMap?.set(snapshot?.options?.contextId,2);
+                                }
                                 this.processNext();
                             } else {
-                                await this.ctx.client.processSnapshot(this.ctx, processedSnapshot, snapshotUuid, discoveryErrors);
+                                await this.ctx.client.processSnapshot(this.ctx, processedSnapshot, snapshotUuid, discoveryErrors,calculateVariantCountFromSnapshot(processedSnapshot, this.ctx.config),snapshot?.options?.sync);
+                                if(snapshot?.options?.contextId && this.ctx.contextToSnapshotMap?.has(snapshot.options.contextId)){
+                                    this.ctx.contextToSnapshotMap.set(snapshot.options.contextId, 1);
+                                }
+                                this.ctx.log.debug(`ContextId: ${snapshot?.options?.contextId} status set to uploaded`);
                             }
                         } else {
+                            this.ctx.log.info(`Uploading snapshot to S3`);
                             await this.ctx.client.uploadSnapshot(this.ctx, processedSnapshot,  discoveryErrors);
                         }
                         this.ctx.totalSnapshots++;
@@ -418,6 +445,9 @@ export default class Queue {
             } catch (error: any) {
                 this.ctx.log.debug(`snapshot failed; ${error}`);
                 this.processedSnapshots.push({ name: snapshot?.name, error: error.message });
+                if (snapshot?.options?.contextId && this.ctx.contextToSnapshotMap) {
+                    this.ctx.contextToSnapshotMap.set(snapshot.options.contextId, 2);
+                }
             }
             // Close open browser contexts and pages
             if (this.ctx.browser) {