diff --git a/examples/@default/sandbox/@models/mymodel/metadata.json b/examples/@default/sandbox/@models/mymodel/metadata.json index 567dc65..257d494 100644 --- a/examples/@default/sandbox/@models/mymodel/metadata.json +++ b/examples/@default/sandbox/@models/mymodel/metadata.json @@ -38,55 +38,61 @@ "minRelativeProgress": 0.01, "warmStart": false, "earlyStop": true, + "inputLabelColumns": [ + "label" + ], "dataSplitMethod": "AUTO_SPLIT", "learnRateStrategy": "LINE_SEARCH", "initialLearnRate": 0.15, "optimizationStrategy": "BATCH_GRADIENT_DESCENT", - "calculatePValues": false + "calculatePValues": false, + "enableGlobalExplain": false, + "fitIntercept": true, + "standardizeFeatures": true }, - "trainingStartTime": "1679395011891", + "trainingStartTime": "1690084242490", "results": [ { "index": 0, - "durationMs": "1931", - "trainingLoss": 1491.0539770637374, + "durationMs": "1818", + "trainingLoss": 1491.0539770637376, "learnRate": 0.3 }, { "index": 1, - "durationMs": "2353", - "trainingLoss": 241.92978201881218, + "durationMs": "1967", + "trainingLoss": 241.92978201881212, "learnRate": 0.6 }, { "index": 2, - "durationMs": "2808", - "trainingLoss": 41.69727113362378, + "durationMs": "1798", + "trainingLoss": 41.697271133623744, "learnRate": 0.6 }, { "index": 3, - "durationMs": "1995", - "trainingLoss": 7.920357012251938, + "durationMs": "1850", + "trainingLoss": 7.920357012251903, "learnRate": 0.6 }, { "index": 4, - "durationMs": "2232", - "trainingLoss": 2.603613218455023, + "durationMs": "1670", + "trainingLoss": 2.603613218455012, "learnRate": 0.6 } ], "evaluationMetrics": { "regressionMetrics": { - "meanAbsoluteError": 1.3283107802514735, - "meanSquaredError": 2.603613218455023, - "meanSquaredLogError": 0.0012176719511574083, - "medianAbsoluteError": 1.0159802433960223, + "meanAbsoluteError": 1.3283107802514682, + "meanSquaredError": 2.603613218455012, + "meanSquaredLogError": 0.0012176719511573831, + "medianAbsoluteError": 1.0159802433960081, "rSquared": 0.9968441051897515 } }, - "startTime": "2023-03-21T10:36:51.891Z" + "startTime": "2023-07-23T03:50:42.490Z" } ] } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 44069b8..8aa8e1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@google-cloud/bigquery": "^6.0.3", "@types/node": "^20.1.1", - "commander": "^10.0.0", + "commander": "^11.0.0", "log-update": "^5.0.1", "p-throttle": "^5.0.0", "picocolors": "^1.0.0", @@ -23,7 +23,7 @@ }, "devDependencies": { "@tsconfig/strictest": "^2.0.1", - "@vitest/coverage-istanbul": "^0.29.1", + "@vitest/coverage-istanbul": "^0.31.4", "ts-node": "^10.7.0", "typescript": "^5.0.4", "vitest": "^0.31.0", @@ -99,9 +99,9 @@ } }, "node_modules/@babel/core/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" @@ -154,9 +154,9 @@ } }, "node_modules/@babel/helper-compilation-targets/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" @@ -994,14 +994,14 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.1.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.1.tgz", - "integrity": "sha512-uKBEevTNb+l6/aCQaKVnUModfEMjAl98lw2Si9P5y4hLu9tm6AlX2ZIoXZX6Wh9lJueYPrGPKk5WMCNHg/u6/A==" + "version": "20.4.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.1.tgz", + "integrity": "sha512-JIzsAvJeA/5iY6Y/OxZbv1lUcc8dNSE77lb2gnBH+/PJ3lFR1Ccvgwl5JWnHAkNHcRsT0TbpVOsiMKZ1F/yyJg==" }, "node_modules/@vitest/coverage-istanbul": { - "version": "0.29.5", - "resolved": "https://registry.npmjs.org/@vitest/coverage-istanbul/-/coverage-istanbul-0.29.5.tgz", - "integrity": "sha512-VTubpTdkOuT3cbo8qAFqCTZ8NXeBOVVHz7lrqueXMJK4WyDIHxnhAdWLrr8HKz5s36acOm6vi6TiFSUm3WHtwQ==", + "version": "0.31.4", + "resolved": "https://registry.npmjs.org/@vitest/coverage-istanbul/-/coverage-istanbul-0.31.4.tgz", + "integrity": "sha512-nQsWQzatd4NXyM7uiIimInOiCopn7gwhFaA8kbBY7u5eYbKyHkMYVgAyqr1gvyPWe4wINdKonqdyQIc110RJkw==", "dev": true, "dependencies": { "istanbul-lib-coverage": "^3.2.0", @@ -1012,10 +1012,10 @@ "test-exclude": "^6.0.0" }, "funding": { - "url": "https://github.com/sponsors/antfu" + "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "vitest": ">=0.28.0 <1" + "vitest": ">=0.30.0 <1" } }, "node_modules/@vitest/expect": { @@ -1328,9 +1328,9 @@ } }, "node_modules/acorn-import-assertions": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz", - "integrity": "sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz", + "integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==", "dev": true, "peerDependencies": { "acorn": "^8" @@ -1800,11 +1800,11 @@ "dev": true }, "node_modules/commander": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.0.tgz", - "integrity": "sha512-zS5PnTI22FIRM6ylNW8G4Ap0IEOyk62fhLSD0+uHRT9McRCLGpkVNvao4bjimpK/GShynyQkFFxHhwMcETmduA==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.0.0.tgz", + "integrity": "sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ==", "engines": { - "node": ">=14" + "node": ">=16" } }, "node_modules/concat-map": { @@ -1833,9 +1833,9 @@ } }, "node_modules/concordance/node_modules/semver": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz", - "integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dev": true, "dependencies": { "lru-cache": "^6.0.0" @@ -2024,9 +2024,9 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.13.0", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.13.0.tgz", - "integrity": "sha512-eyV8f0y1+bzyfh8xAwW/WTSZpLbjhqc4ne9eGSH4Zo2ejdyiNG9pU6mf9DG8a7+Auk6MFTlNOT4Y2y/9k8GKVg==", + "version": "5.15.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz", + "integrity": "sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==", "dev": true, "dependencies": { "graceful-fs": "^4.2.4", @@ -2689,9 +2689,9 @@ } }, "node_modules/istanbul-lib-instrument/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" @@ -2995,9 +2995,9 @@ } }, "node_modules/make-dir/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" @@ -3665,9 +3665,9 @@ ] }, "node_modules/schema-utils": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.2.tgz", - "integrity": "sha512-pvjEHOgWc9OWA/f/DE3ohBWTD6EleVLf7iFUkoSwAxttdBhB9QUebQgxER2kWueOvRJXPHNnyrvvh9eZINB8Eg==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", "dev": true, "dependencies": { "@types/json-schema": "^7.0.8", @@ -3683,9 +3683,9 @@ } }, "node_modules/semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", "bin": { "semver": "bin/semver" } @@ -4459,9 +4459,9 @@ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "node_modules/webpack": { - "version": "5.80.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.80.0.tgz", - "integrity": "sha512-OIMiq37XK1rWO8mH9ssfFKZsXg4n6klTEDL7S8/HqbAOBBaiy8ABvXvz0dDCXeEF9gqwxSvVk611zFPjS8hJxA==", + "version": "5.88.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.88.1.tgz", + "integrity": "sha512-FROX3TxQnC/ox4N+3xQoWZzvGXSuscxR32rbzjpXgEzWudJFEJBpdlkkob2ylrv5yzzufD1zph1OoFsLtm6stQ==", "dev": true, "dependencies": { "@types/eslint-scope": "^3.7.3", @@ -4470,10 +4470,10 @@ "@webassemblyjs/wasm-edit": "^1.11.5", "@webassemblyjs/wasm-parser": "^1.11.5", "acorn": "^8.7.1", - "acorn-import-assertions": "^1.7.6", + "acorn-import-assertions": "^1.9.0", "browserslist": "^4.14.5", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.13.0", + "enhanced-resolve": "^5.15.0", "es-module-lexer": "^1.2.1", "eslint-scope": "5.1.1", "events": "^3.2.0", @@ -4483,7 +4483,7 @@ "loader-runner": "^4.2.0", "mime-types": "^2.1.27", "neo-async": "^2.6.2", - "schema-utils": "^3.1.2", + "schema-utils": "^3.2.0", "tapable": "^2.1.1", "terser-webpack-plugin": "^5.3.7", "watchpack": "^2.4.0", diff --git a/package.json b/package.json index cfb9149..1b07332 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dependencies": { "@google-cloud/bigquery": "^6.0.3", "@types/node": "^20.1.1", - "commander": "^10.0.0", + "commander": "^11.0.0", "log-update": "^5.0.1", "p-throttle": "^5.0.0", "picocolors": "^1.0.0", @@ -37,7 +37,7 @@ }, "devDependencies": { "@tsconfig/strictest": "^2.0.1", - "@vitest/coverage-istanbul": "^0.29.1", + "@vitest/coverage-istanbul": "^0.31.4", "ts-node": "^10.7.0", "typescript": "^5.0.4", "vitest": "^0.31.0", diff --git a/src/commands/cli.ts b/src/commands/cli.ts index d750a7a..e8c0c38 100644 --- a/src/commands/cli.ts +++ b/src/commands/cli.ts @@ -1,7 +1,7 @@ import * as fs from 'node:fs'; import { formatLocalfiles } from '../../src/commands/fix.js'; -import { pushLocalFilesToBigQuery } from '../../src/commands/push.js'; +import { PushContext, pushLocalFilesToBigQuery } from '../../src/commands/push.js'; import { createBundleSQL } from '../../src/commands/bundle.js'; import { pullBigQueryResources } from '../../src/commands/pull.js'; @@ -36,9 +36,9 @@ export function createCLI() { ) .argument('[projects...]') .option( - '--force', - 'Force to apply changes such as deletion without confirmation', - false, + '--sweeping-mode ', + 'Strategy for undefined/orphans BigQuery resources. option: confirm(default), ignore, rename', + 'confirm', ) .option( '--label ', @@ -147,6 +147,22 @@ export function createCLI() { 500, ); + const sweepMode: PushContext['SweepStrategy']['mode'] = (() => { + if(!cmdOptions.sweepingMode) { + return 'confirm' + } + switch (cmdOptions.sweepingMode) { + case 'ignore': + case 'rename_and_7d_expire': + case 'confirm': + case 'force': + return cmdOptions.sweepingMode + default: + // Error for invalid option + throw Error(`Invalid --sweeping-mode option: ${cmdOptions.sweepingMode}`); + } + })(); + for (const project of projects) { const ctx = { BigQuery: { @@ -155,10 +171,13 @@ export function createCLI() { }, rootPath: rootDir, dryRun: cmdOptions.dryRun ?? false, - force: cmdOptions.force ?? false, + SweepStrategy: { + mode: sweepMode, + ignorePrefix: 'zorphan__', + }, reporter: cmdOptions.format ?? 'console', enableDataLineage: cmdOptions.enableDatalineage ?? false, - }; + } as PushContext; const failed = await pushLocalFilesToBigQuery(ctx, jobOption); if (failed > 0) { diff --git a/src/commands/push.ts b/src/commands/push.ts index 98013b9..1a522c2 100644 --- a/src/commands/push.ts +++ b/src/commands/push.ts @@ -28,17 +28,17 @@ import { normalizeShardingTableId, path2bq, } from '../../src/bigquery.js'; -import { createCleanupTasks } from '../../src/tasks/cleanup.js'; +import { SweepStrategy, createCleanupTasks } from '../../src/tasks/cleanup.js'; type PushContext = { dryRun: boolean; - force: boolean; rootPath: string; enableDataLineage: boolean; BigQuery: { projectId: string; client: BigQuery; }; + SweepStrategy: SweepStrategy; reporter: BuiltInReporters; }; @@ -542,4 +542,4 @@ async function pushLocalFilesToBigQuery( return failedTasks; } -export { buildDAG, pushLocalFilesToBigQuery }; +export { buildDAG, pushLocalFilesToBigQuery, PushContext}; diff --git a/src/tasks/cleanup.ts b/src/tasks/cleanup.ts index 2fff1d2..6d13bb0 100644 --- a/src/tasks/cleanup.ts +++ b/src/tasks/cleanup.ts @@ -10,25 +10,36 @@ import { prompt } from '../../src/prompt.js'; type PushContext = { dryRun: boolean; - force: boolean; rootPath: string; BigQuery: { projectId: string; client: BigQuery; }; + SweepStrategy: SweepStrategy; // reporter: BuiltInReporters; }; +type SweepStrategy = { + mode: 'confirm' | 'force' | 'ignore' | 'rename_and_7d_expire'; + ignorePrefix: string; +}; + const cleanupBigQueryDataset = async ( bqClient: BigQuery, rootDir: string, projectId: string, datasetId: string, + mode: SweepStrategy['mode'], options?: { dryRun?: boolean; withoutConrimation: boolean; + ignorePrefix: string; }, ): Promise => { + if(mode === 'ignore') { + return []; + } + const defaultProjectId = await bqClient.getProjectId(); const nsProject = projectId != '@default' ? projectId : defaultProjectId; @@ -113,7 +124,7 @@ const cleanupBigQueryDataset = async ( if (!isForce && !isDryRun) { const ans = await prompt( [ - `Found BigQuery reousrces with no local files. Do you delete these resources? (y/n)`, + `Found BigQuery reousrces with no definition by local files. Do you delete these resources? (y/n)`, `[${resourceType}s]`, ` ${[...kind.keys()].join('\n ')} `, 'Ans>', @@ -125,26 +136,83 @@ const cleanupBigQueryDataset = async ( } for (const [bqId, resource] of kind) { - const task = new BigQueryJobTask( - [ - nsProject, - datasetId, - '(DELETE)', - resourceType.toUpperCase(), - bqId.split('.').pop(), - ] - .join('/'), - async () => { - try { - console.error(`${isDryRun ? '(DRYRUN) ' : ''}Deleting ${bqId}`); - if (!isDryRun) { - await resource.delete(); - } - } catch (e) { + // Skip process prefixed by ignorePrefix + const task: BigQueryJobTask = (() => { + switch(mode) { + case 'rename_and_7d_expire': + return new BigQueryJobTask( + [ + nsProject, + datasetId, + '(RENAME)', + resourceType.toUpperCase(), + bqId.split('.').pop(), + ] + .join('/'), + async () => { + if (options?.ignorePrefix && (bqId.split('.').pop() ?? 'undefined').startsWith(options.ignorePrefix)) { + return {isDryRun}; + } + + if(resourceType === 'Table') { + const dataset = (resource?.parent as Dataset); + const client: BigQuery = (dataset.parent as BigQuery); + const newTableName = `${(options?.ignorePrefix ?? 'zz_')}${resource.id}` + const tableRenameTo = `${dataset.projectId}.${dataset.id}.${newTableName}`; + const query = ` + -- BigQuery Porter: Rename table + ALTER TABLE \`${bqId}\` RENAME TO \`${newTableName}\`; + ALTER TABLE \`${tableRenameTo}\` SET OPTIONS( + expiration_timestamp=TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 7 DAY) + ); + ` + try { + console.error(`${isDryRun ? '(DRYRUN) ' : ''} Renaming ${bqId} to ${tableRenameTo}`); + const [job] = await client.createQueryJob({ + query, + jobPrefix: `bqport-metadata_rename-`, + }) + if (!job) { + throw Error('Renamign Exception: Exception'); + } + return { isDryRun } + } catch (e) { + console.error(e); + } + + } + + return { isDryRun }; + }, + ); + default: + return new BigQueryJobTask( + [ + nsProject, + datasetId, + '(DELETE)', + resourceType.toUpperCase(), + bqId.split('.').pop(), + ] + .join('/'), + async () => { + try { + console.error(`${isDryRun ? '(DRYRUN) ' : '🗑️'} Deleting ${bqId}`); + if (!isDryRun) { + // switch (mode) { + // case 'rename': + // break; + // default: + await resource.delete(); + } + } catch (e) { + } + return { isDryRun }; + }, + ); } - return { isDryRun }; - }, - ); + })(); + tasks.push(task); } } @@ -166,9 +234,11 @@ const createCleanupTasks = async ( ctx.rootPath, ctx.BigQuery.projectId, path.basename(dataset), + ctx.SweepStrategy.mode, { dryRun: ctx.dryRun, - withoutConrimation: ctx.force, + withoutConrimation: ctx.SweepStrategy.mode !== 'confirm', + ignorePrefix: ctx.SweepStrategy.ignorePrefix, }, ).catch((e) => { console.error(e); @@ -180,4 +250,4 @@ const createCleanupTasks = async ( return tasks; }; -export { cleanupBigQueryDataset, createCleanupTasks }; +export { cleanupBigQueryDataset, createCleanupTasks, SweepStrategy}; diff --git a/tests/scenario/__snapshots__/pull.spec.ts.snap b/tests/scenario/__snapshots__/pull.spec.ts.snap index ddf53aa..1c43011 100644 --- a/tests/scenario/__snapshots__/pull.spec.ts.snap +++ b/tests/scenario/__snapshots__/pull.spec.ts.snap @@ -24,20 +24,6 @@ Map { ], \\"location\\": \\"US\\" }", - "@default/examples/sample_table/metadata.json" => "{ - \\"type\\": \\"TABLE\\", - \\"clustering\\": { - \\"fields\\": [ - \\"a\\" - ] - } -}", - "@default/examples/sample_table/schema.json" => "[ - { - \\"name\\": \\"a\\", - \\"type\\": \\"INTEGER\\" - } -]", } `; @@ -75,28 +61,6 @@ OPTIONS( ], \\"location\\": \\"US\\" }", - "@default/examples/sample_table/ddl.sql" => "CREATE TABLE IF NOT EXISTS \`bigquery-porter.examples.sample_table\` -( - a INT64 -) -CLUSTER BY a -OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T12:18:34.213Z\\" -);", - "@default/examples/sample_table/metadata.json" => "{ - \\"type\\": \\"TABLE\\", - \\"clustering\\": { - \\"fields\\": [ - \\"a\\" - ] - } -}", - "@default/examples/sample_table/schema.json" => "[ - { - \\"name\\": \\"a\\", - \\"type\\": \\"INTEGER\\" - } -]", "@default/examples_eu/ddl.sql" => "CREATE SCHEMA IF NOT EXISTS \`bigquery-porter.examples_eu\` OPTIONS( default_table_expiration_days=60.0, @@ -125,22 +89,6 @@ OPTIONS( ], \\"location\\": \\"EU\\" }", - "@default/examples_eu/sample_table/ddl.sql" => "CREATE TABLE IF NOT EXISTS \`bigquery-porter.examples_eu.sample_table\` -( - a INT64 -) -OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-12T23:44:41.925Z\\" -);", - "@default/examples_eu/sample_table/metadata.json" => "{ - \\"type\\": \\"TABLE\\" -}", - "@default/examples_eu/sample_table/schema.json" => "[ - { - \\"name\\": \\"a\\", - \\"type\\": \\"INTEGER\\" - } -]", "@default/sample_linked_dataset/README.md" => "ソース リスティング: [JP Holidays preprocessed by BQ FUN](//\\\\(bigquery//analyticshub:projects/120299025068/locations/asia-northeast1/dataExchanges/jpdata_18253a34a30/listings/jp_holidays_preprocessed_by_bq_fun_18253c4e9dc/create-linked-dataset\\\\)\\\\(analyticshub:projects/120299025068/locations/asia-northeast1/dataExchanges/jpdata_18253a34a30/listings/jp_holidays_preprocessed_by_bq_fun_18253c4e9dc\\\\))", "@default/sample_linked_dataset/ddl.sql" => "CREATE SCHEMA IF NOT EXISTS \`bigquery-porter.sample_linked_dataset\` OPTIONS( @@ -233,55 +181,61 @@ OPTIONS( \\"minRelativeProgress\\": 0.01, \\"warmStart\\": false, \\"earlyStop\\": true, + \\"inputLabelColumns\\": [ + \\"label\\" + ], \\"dataSplitMethod\\": \\"AUTO_SPLIT\\", \\"learnRateStrategy\\": \\"LINE_SEARCH\\", \\"initialLearnRate\\": 0.15, \\"optimizationStrategy\\": \\"BATCH_GRADIENT_DESCENT\\", - \\"calculatePValues\\": false + \\"calculatePValues\\": false, + \\"enableGlobalExplain\\": false, + \\"fitIntercept\\": true, + \\"standardizeFeatures\\": true }, - \\"trainingStartTime\\": \\"1679395011891\\", + \\"trainingStartTime\\": \\"1690084242490\\", \\"results\\": [ { \\"index\\": 0, - \\"durationMs\\": \\"1931\\", - \\"trainingLoss\\": 1491.0539770637374, + \\"durationMs\\": \\"1818\\", + \\"trainingLoss\\": 1491.0539770637376, \\"learnRate\\": 0.3 }, { \\"index\\": 1, - \\"durationMs\\": \\"2353\\", - \\"trainingLoss\\": 241.92978201881218, + \\"durationMs\\": \\"1967\\", + \\"trainingLoss\\": 241.92978201881212, \\"learnRate\\": 0.6 }, { \\"index\\": 2, - \\"durationMs\\": \\"2808\\", - \\"trainingLoss\\": 41.69727113362378, + \\"durationMs\\": \\"1798\\", + \\"trainingLoss\\": 41.697271133623744, \\"learnRate\\": 0.6 }, { \\"index\\": 3, - \\"durationMs\\": \\"1995\\", - \\"trainingLoss\\": 7.920357012251938, + \\"durationMs\\": \\"1850\\", + \\"trainingLoss\\": 7.920357012251903, \\"learnRate\\": 0.6 }, { \\"index\\": 4, - \\"durationMs\\": \\"2232\\", - \\"trainingLoss\\": 2.603613218455023, + \\"durationMs\\": \\"1670\\", + \\"trainingLoss\\": 2.603613218455012, \\"learnRate\\": 0.6 } ], \\"evaluationMetrics\\": { \\"regressionMetrics\\": { - \\"meanAbsoluteError\\": 1.3283107802514735, - \\"meanSquaredError\\": 2.603613218455023, - \\"meanSquaredLogError\\": 0.0012176719511574083, - \\"medianAbsoluteError\\": 1.0159802433960223, + \\"meanAbsoluteError\\": 1.3283107802514682, + \\"meanSquaredError\\": 2.603613218455012, + \\"meanSquaredLogError\\": 0.0012176719511573831, + \\"medianAbsoluteError\\": 1.0159802433960081, \\"rSquared\\": 0.9968441051897515 } }, - \\"startTime\\": \\"2023-03-21T10:36:51.891Z\\" + \\"startTime\\": \\"2023-07-23T03:50:42.490Z\\" } ] }", @@ -370,7 +324,7 @@ OPTIONS( ) CLUSTER BY a OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T10:39:16.121Z\\" + expiration_timestamp=TIMESTAMP \\"2023-09-21T03:50:49.196Z\\" );", "@default/sandbox/sample_clone_table/metadata.json" => "{ \\"type\\": \\"TABLE\\", @@ -388,7 +342,7 @@ OPTIONS( ]", "@default/sandbox/sample_materialized_view/ddl.sql" => "CREATE MATERIALIZED VIEW IF NOT EXISTS \`bigquery-porter.sandbox.sample_materialized_view\` OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T10:36:56.363Z\\" + expiration_timestamp=TIMESTAMP \\"2023-09-21T03:50:46.175Z\\" ) AS select 1 as a from \`sandbox.sample_table\`;", "@default/sandbox/sample_materialized_view/metadata.json" => "{ @@ -427,7 +381,7 @@ OPTIONS( date_jst DATE ) OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T10:36:53.787Z\\" + expiration_timestamp=TIMESTAMP \\"2023-09-21T03:50:44.134Z\\" );", "@default/sandbox/sample_sharding_@/metadata.json" => "{ \\"type\\": \\"TABLE\\" @@ -439,11 +393,11 @@ OPTIONS( } ]", "@default/sandbox/sample_snapshot_table/ddl.sql" => "CREATE SNAPSHOT TABLE \`bigquery-porter.sandbox.sample_snapshot_table\` -CLONE \`takegue.sandbox.sample_table\` +CLONE \`bigquery-porter.sandbox.sample_table\` OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T10:36:52.966Z\\" + expiration_timestamp=TIMESTAMP \\"2023-07-28T19:01:36.543Z\\" ) -FOR SYSTEM_TIME AS OF TIMESTAMP \\"2023-03-18T10:36:52.311Z\\";", +FOR SYSTEM_TIME AS OF TIMESTAMP \\"2023-05-29T18:01:35.384Z\\";", "@default/sandbox/sample_snapshot_table/metadata.json" => "{ \\"type\\": \\"SNAPSHOT\\", \\"clustering\\": { @@ -453,19 +407,19 @@ FOR SYSTEM_TIME AS OF TIMESTAMP \\"2023-03-18T10:36:52.311Z\\";", }, \\"snapshotDefinition\\": { \\"baseTableReference\\": { - \\"projectId\\": \\"takegue\\", + \\"projectId\\": \\"bigquery-porter\\", \\"datasetId\\": \\"sandbox\\", \\"tableId\\": \\"sample_table\\" }, - \\"snapshotTime\\": \\"2023-03-18T10:36:52.311Z\\" + \\"snapshotTime\\": \\"2023-05-29T18:01:35.384Z\\" }, \\"cloneDefinition\\": { \\"baseTableReference\\": { - \\"projectId\\": \\"takegue\\", + \\"projectId\\": \\"bigquery-porter\\", \\"datasetId\\": \\"sandbox\\", \\"tableId\\": \\"sample_table\\" }, - \\"snapshotTime\\": \\"2023-03-18T10:36:52.311Z\\" + \\"snapshotTime\\": \\"2023-05-29T18:01:35.384Z\\" } }", "@default/sandbox/sample_snapshot_table/schema.json" => "[ @@ -482,7 +436,7 @@ FOR SYSTEM_TIME AS OF TIMESTAMP \\"2023-03-18T10:36:52.311Z\\";", ) CLUSTER BY a OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T10:36:54.290Z\\" + expiration_timestamp=TIMESTAMP \\"2023-09-21T03:50:44.115Z\\" );", "@default/sandbox/sample_table/metadata.json" => "{ \\"type\\": \\"TABLE\\", diff --git a/tests/scenario/__snapshots__/push.spec.ts.snap b/tests/scenario/__snapshots__/push.spec.ts.snap index 47dc65d..bff2ac1 100644 --- a/tests/scenario/__snapshots__/push.spec.ts.snap +++ b/tests/scenario/__snapshots__/push.spec.ts.snap @@ -80,8 +80,8 @@ exports[`CLI: push > push --help 1`] = ` Deploy your local BigQuery Resources in topological-sorted order Options: - --force Force to apply changes such as deletion without confirmation (default: - false) + --sweeping-mode Strategy for undefined/orphans BigQuery resources. option: + confirm(default), ignore, rename (default: \\"confirm\\") --label A label to set on a query job. The format is \\"key:value\\"; repeat this option to specify a list of values -p, --parameter Either a file containing a JSON list of query parameters, or a query diff --git a/tests/task_cleanup.spec.ts b/tests/task_cleanup.spec.ts index b383cda..abf4b69 100644 --- a/tests/task_cleanup.spec.ts +++ b/tests/task_cleanup.spec.ts @@ -35,9 +35,11 @@ describe('Task: Cleanup tasks', () => { _root, _project, 'missing_dataset', + 'confirm', { dryRun: true, withoutConrimation: true, + ignorePrefix: '', }, ); @@ -51,9 +53,11 @@ describe('Task: Cleanup tasks', () => { _root, _project, 'missing_dataset', + 'confirm', { dryRun: true, withoutConrimation: true, + ignorePrefix: '', }, ); @@ -97,7 +101,7 @@ describe('Task: Cleanup tasks', () => { } }); - it.concurrent('Should remove table missing in local without prompt', async (ctx) => { + it.concurrent("Shouldn't remove table missing in local without prompt with ignore mode", async (ctx) => { const { settings: { _root, _project } } = ctx as any; const tasks = await createCleanupTasks( { @@ -106,8 +110,56 @@ describe('Task: Cleanup tasks', () => { client: bqClient, }, dryRun: false, - force: true, rootPath: _root, + SweepStrategy: { + mode: 'ignore', + ignorePrefix: '', + } + }, + ); + + expect(tasks.length).toBe(0); + }); + + it.concurrent('Should remove table missing in local with prompt', async (ctx) => { + const { settings: { _root, _project } } = ctx as any; + + const tasks = await createCleanupTasks( + { + BigQuery: { + projectId: _project, + client: bqClient, + }, + dryRun: false, + rootPath: _root, + SweepStrategy: { + mode: 'force', + ignorePrefix: '', + } + }, + ); + + expect(tasks.length).toBe(1); + expect(tasks[0]?.name).toBe( + 'bigquery-public-data/austin_bikeshare/(DELETE)/TABLE/bikeshare_stations', + ); + }); + + it.concurrent('Dry run', async (ctx) => { + const { settings: { _root, _project } } = ctx as any; + + const tasks = await createCleanupTasks( + { + BigQuery: { + projectId: _project, + client: bqClient, + }, + dryRun: true, + rootPath: _root, + SweepStrategy: { + mode: 'confirm', + ignorePrefix: '', + } }, ); @@ -127,14 +179,17 @@ describe('Task: Cleanup tasks', () => { client: bqClient, }, dryRun: false, - force: false, rootPath: _root, + SweepStrategy: { + mode: 'rename_and_7d_expire', + ignorePrefix: '', + } }, ); expect(tasks.length).toBe(1); expect(tasks[0]?.name).toBe( - 'bigquery-public-data/austin_bikeshare/(DELETE)/TABLE/bikeshare_stations', + 'bigquery-public-data/austin_bikeshare/(RENAME)/TABLE/bikeshare_stations', ); }); }); @@ -180,8 +235,11 @@ describe('Task: Cleanup tasks', () => { client: bqClient, }, dryRun: false, - force: true, rootPath: _root, + SweepStrategy: { + mode: 'confirm', + ignorePrefix: '', + } }, );