diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8adfdf9..23d07f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,6 @@ jobs: token: ${{ secrets.NPM_TOKEN }} package: ./package.json access: public - tag: beta - name: Create Release if: ${{ steps.publish-plugin.conclusion == 'success' }} id: create_release diff --git a/.talismanrc b/.talismanrc index f268b29..f93d4c7 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,6 @@ fileignoreconfig: - filename: .env-example checksum: 591f1e672d4df287107092b8fd37c27913e09225c6ced55293e1d459b1119d05 + - filename: src/core/query-executor.ts + checksum: 53f90091c9877c79f43bf0e3dcc26281f0c6b2073d63380803cda1481712a5ae version: '1.0' diff --git a/package.json b/package.json index 31b4030..d09b80c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@contentstack/cli-cm-export-query", "description": "Contentstack CLI plugin to export content from stack", - "version": "1.0.0-beta.2", + "version": "1.0.0-beta.3", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { diff --git a/snyk_output.log b/snyk_output.log deleted file mode 100644 index e69de29..0000000 diff --git a/src/core/query-executor.ts b/src/core/query-executor.ts index c4de8b1..c14036e 100644 --- a/src/core/query-executor.ts +++ b/src/core/query-executor.ts @@ -286,23 +286,132 @@ export class QueryExporter { log(this.exportQueryConfig, 'Starting export of referenced assets...', 'info'); try { + const assetsDir = path.join( + sanitizePath(this.exportQueryConfig.exportDir), + sanitizePath(this.exportQueryConfig.branchName || ''), + 'assets', + ); + + const metadataFilePath = path.join(assetsDir, 'metadata.json'); + const assetFilePath = path.join(assetsDir, 'assets.json'); + + // Define temp file paths + const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json'); + const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json'); + const assetHandler = new AssetReferenceHandler(this.exportQueryConfig); // Extract referenced asset UIDs from all entries const assetUIDs = assetHandler.extractReferencedAssets(); if (assetUIDs.length > 0) { - log(this.exportQueryConfig, `Exporting ${assetUIDs.length} referenced assets...`, 'info'); + log(this.exportQueryConfig, `Found ${assetUIDs.length} referenced assets to export`, 'info'); - const query = { - modules: { - assets: { - uid: { $in: assetUIDs }, + // Define batch size - can be configurable through exportQueryConfig + const batchSize = this.exportQueryConfig.assetBatchSize || 100; + + if (assetUIDs.length <= batchSize) { + const query = { + modules: { + assets: { + uid: { $in: assetUIDs }, + }, }, - }, - }; + }; + + await this.moduleExporter.exportModule('assets', { query }); + } + + // if asset size is bigger than batch size, then we need to export in batches + // Calculate number of batches + const totalBatches = Math.ceil(assetUIDs.length / batchSize); + log(this.exportQueryConfig, `Processing assets in ${totalBatches} batches of ${batchSize}`, 'info'); + + // Process assets in batches + for (let i = 0; i < totalBatches; i++) { + const start = i * batchSize; + const end = Math.min(start + batchSize, assetUIDs.length); + const batchAssetUIDs = assetUIDs.slice(start, end); + + log( + this.exportQueryConfig, + `Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`, + 'info', + ); + + const query = { + modules: { + assets: { + uid: { $in: batchAssetUIDs }, + }, + }, + }; + + await this.moduleExporter.exportModule('assets', { query }); + + // Read the current batch's metadata.json and assets.json files + const currentMetadata: any = fsUtil.readFile(sanitizePath(metadataFilePath)); + const currentAssets: any = fsUtil.readFile(sanitizePath(assetFilePath)); + + // Check if this is the first batch + if (i === 0) { + // For first batch, initialize temp files with current content + fsUtil.writeFile(sanitizePath(tempMetadataFilePath), currentMetadata); + fsUtil.writeFile(sanitizePath(tempAssetFilePath), currentAssets); + log(this.exportQueryConfig, `Initialized temporary files with first batch data`, 'info'); + } else { + // For subsequent batches, append to temp files with incremented keys + + // Handle metadata (which contains arrays of asset info) + const tempMetadata: any = fsUtil.readFile(sanitizePath(tempMetadataFilePath)) || {}; + + // Merge metadata by combining arrays + if (currentMetadata) { + Object.keys(currentMetadata).forEach((key: string) => { + if (!tempMetadata[key]) { + tempMetadata[key] = currentMetadata[key]; + } + }); + } + + // Write updated metadata back to temp file + fsUtil.writeFile(sanitizePath(tempMetadataFilePath), tempMetadata); + + // Handle assets (which is an object with numeric keys) + const tempAssets: any = fsUtil.readFile(sanitizePath(tempAssetFilePath)) || {}; + let nextIndex = Object.keys(tempAssets).length + 1; + + // Add current assets with incremented keys + Object.values(currentAssets).forEach((value: any) => { + tempAssets[nextIndex.toString()] = value; + nextIndex++; + }); + + fsUtil.writeFile(sanitizePath(tempAssetFilePath), tempAssets); + + log(this.exportQueryConfig, `Updated temporary files with batch ${i + 1} data`, 'info'); + } + + // Optional: Add delay between batches to avoid rate limiting + if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) { + await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs)); + } + } + + // After all batches are processed, copy temp files back to original files + const finalMetadata = fsUtil.readFile(sanitizePath(tempMetadataFilePath)); + const finalAssets = fsUtil.readFile(sanitizePath(tempAssetFilePath)); + + fsUtil.writeFile(sanitizePath(metadataFilePath), finalMetadata); + fsUtil.writeFile(sanitizePath(assetFilePath), finalAssets); + + log(this.exportQueryConfig, `Final data written back to original files`, 'info'); + + // Clean up temp files + fsUtil.removeFile(sanitizePath(tempMetadataFilePath)); + fsUtil.removeFile(sanitizePath(tempAssetFilePath)); - await this.moduleExporter.exportModule('assets', { query }); + log(this.exportQueryConfig, `Temporary files cleaned up`, 'info'); log(this.exportQueryConfig, 'Referenced assets exported successfully', 'success'); } else { log(this.exportQueryConfig, 'No referenced assets found in entries', 'info'); diff --git a/src/types/index.ts b/src/types/index.ts index bbd19c6..758bc04 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -185,6 +185,9 @@ export interface QueryExportConfig extends DefaultConfig { logsPath: string; dataPath: string; exportDelayMs?: number; + batchDelayMs?: number; + assetBatchSize?: number; + assetBatchDelayMs?: number; } export interface QueryMetadata { diff --git a/src/utils/config-handler.ts b/src/utils/config-handler.ts index 49013c4..edad66c 100644 --- a/src/utils/config-handler.ts +++ b/src/utils/config-handler.ts @@ -24,6 +24,11 @@ export async function setupQueryExportConfig(flags: any): Promise