Large number of assets batch support

.github/workflows/release.yml

@@ -33,7 +33,6 @@ jobs:
           token: ${{ secrets.NPM_TOKEN }}
           package: ./package.json
           access: public
-          tag: beta
       - name: Create Release
         if: ${{ steps.publish-plugin.conclusion == 'success'  }}
         id: create_release

.talismanrc

@@ -1,4 +1,6 @@
 fileignoreconfig:
   - filename: .env-example
     checksum: 591f1e672d4df287107092b8fd37c27913e09225c6ced55293e1d459b1119d05
+  - filename: src/core/query-executor.ts
+    checksum: 53f90091c9877c79f43bf0e3dcc26281f0c6b2073d63380803cda1481712a5ae
 version: '1.0'

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@contentstack/cli-cm-export-query",
   "description": "Contentstack CLI plugin to export content from stack",
-  "version": "1.0.0-beta.2",
+  "version": "1.0.0-beta.3",
   "author": "Contentstack",
   "bugs": "https://github.com/contentstack/cli/issues",
   "dependencies": {

src/core/query-executor.ts

@@ -286,23 +286,132 @@ export class QueryExporter {
     log(this.exportQueryConfig, 'Starting export of referenced assets...', 'info');
 
     try {
+      const assetsDir = path.join(
+        sanitizePath(this.exportQueryConfig.exportDir),
+        sanitizePath(this.exportQueryConfig.branchName || ''),
+        'assets',
+      );
+
+      const metadataFilePath = path.join(assetsDir, 'metadata.json');
+      const assetFilePath = path.join(assetsDir, 'assets.json');
+
+      // Define temp file paths
+      const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json');
+      const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json');
+
       const assetHandler = new AssetReferenceHandler(this.exportQueryConfig);
 
       // Extract referenced asset UIDs from all entries
       const assetUIDs = assetHandler.extractReferencedAssets();
 
       if (assetUIDs.length > 0) {
-        log(this.exportQueryConfig, `Exporting ${assetUIDs.length} referenced assets...`, 'info');
+        log(this.exportQueryConfig, `Found ${assetUIDs.length} referenced assets to export`, 'info');
 
-        const query = {
-          modules: {
-            assets: {
-              uid: { $in: assetUIDs },
+        // Define batch size - can be configurable through exportQueryConfig
+        const batchSize = this.exportQueryConfig.assetBatchSize || 100;
+
+        if (assetUIDs.length <= batchSize) {
+          const query = {
+            modules: {
+              assets: {
+                uid: { $in: assetUIDs },
+              },
             },
-          },
-        };
+          };
+
+          await this.moduleExporter.exportModule('assets', { query });
+        }
+
+        // if asset size is bigger than batch size, then we need to export in batches
+        // Calculate number of batches
+        const totalBatches = Math.ceil(assetUIDs.length / batchSize);
+        log(this.exportQueryConfig, `Processing assets in ${totalBatches} batches of ${batchSize}`, 'info');
+
+        // Process assets in batches
+        for (let i = 0; i < totalBatches; i++) {
+          const start = i * batchSize;
+          const end = Math.min(start + batchSize, assetUIDs.length);
+          const batchAssetUIDs = assetUIDs.slice(start, end);
+
+          log(
+            this.exportQueryConfig,
+            `Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`,
+            'info',
+          );
+
+          const query = {
+            modules: {
+              assets: {
+                uid: { $in: batchAssetUIDs },
+              },
+            },
+          };
+
+          await this.moduleExporter.exportModule('assets', { query });
+
+          // Read the current batch's metadata.json and assets.json files
+          const currentMetadata: any = fsUtil.readFile(sanitizePath(metadataFilePath));
+          const currentAssets: any = fsUtil.readFile(sanitizePath(assetFilePath));
+
+          // Check if this is the first batch
+          if (i === 0) {
+            // For first batch, initialize temp files with current content
+            fsUtil.writeFile(sanitizePath(tempMetadataFilePath), currentMetadata);
+            fsUtil.writeFile(sanitizePath(tempAssetFilePath), currentAssets);
+            log(this.exportQueryConfig, `Initialized temporary files with first batch data`, 'info');
+          } else {
+            // For subsequent batches, append to temp files with incremented keys
+
+            // Handle metadata (which contains arrays of asset info)
+            const tempMetadata: any = fsUtil.readFile(sanitizePath(tempMetadataFilePath)) || {};
+
+            // Merge metadata by combining arrays
+            if (currentMetadata) {
+              Object.keys(currentMetadata).forEach((key: string) => {
+                if (!tempMetadata[key]) {
+                  tempMetadata[key] = currentMetadata[key];
+                }
+              });
+            }
+
+            // Write updated metadata back to temp file
+            fsUtil.writeFile(sanitizePath(tempMetadataFilePath), tempMetadata);
+
+            // Handle assets (which is an object with numeric keys)
+            const tempAssets: any = fsUtil.readFile(sanitizePath(tempAssetFilePath)) || {};
+            let nextIndex = Object.keys(tempAssets).length + 1;
+
+            // Add current assets with incremented keys
+            Object.values(currentAssets).forEach((value: any) => {
+              tempAssets[nextIndex.toString()] = value;
+              nextIndex++;
+            });
+
+            fsUtil.writeFile(sanitizePath(tempAssetFilePath), tempAssets);
+
+            log(this.exportQueryConfig, `Updated temporary files with batch ${i + 1} data`, 'info');
+          }
+
+          // Optional: Add delay between batches to avoid rate limiting
+          if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) {
+            await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs));
+          }
+        }
+
+        // After all batches are processed, copy temp files back to original files
+        const finalMetadata = fsUtil.readFile(sanitizePath(tempMetadataFilePath));
+        const finalAssets = fsUtil.readFile(sanitizePath(tempAssetFilePath));
+
+        fsUtil.writeFile(sanitizePath(metadataFilePath), finalMetadata);
+        fsUtil.writeFile(sanitizePath(assetFilePath), finalAssets);
+
+        log(this.exportQueryConfig, `Final data written back to original files`, 'info');
+
+        // Clean up temp files
+        fsUtil.removeFile(sanitizePath(tempMetadataFilePath));
+        fsUtil.removeFile(sanitizePath(tempAssetFilePath));
 
-        await this.moduleExporter.exportModule('assets', { query });
+        log(this.exportQueryConfig, `Temporary files cleaned up`, 'info');
         log(this.exportQueryConfig, 'Referenced assets exported successfully', 'success');
       } else {
         log(this.exportQueryConfig, 'No referenced assets found in entries', 'info');

src/types/index.ts

@@ -185,6 +185,9 @@ export interface QueryExportConfig extends DefaultConfig {
   logsPath: string;
   dataPath: string;
   exportDelayMs?: number;
+  batchDelayMs?: number;
+  assetBatchSize?: number;
+  assetBatchDelayMs?: number;
 }
 
 export interface QueryMetadata {

src/utils/config-handler.ts

@@ -24,6 +24,11 @@ export async function setupQueryExportConfig(flags: any): Promise<QueryExportCon
     externalConfigPath: path.join(__dirname, '../config/export-config.json'),
   };
 
+  // override the external config path if the user provides a config file
+  if (flags.config) {
+    exportQueryConfig.externalConfigPath = sanitizePath(flags['config']);
+  }
+
   // Handle authentication
   if (flags.alias) {
     const { token, apiKey } = configHandler.get(`tokens.${flags.alias}`) || {};