From 9fd27c6c9d83e653573fa2afc5e7de3092e1becc Mon Sep 17 00:00:00 2001 From: takegue Date: Thu, 13 Jul 2023 08:39:18 +0900 Subject: [PATCH 1/9] Maintainace: update test fixtures --- .../sandbox/@models/mymodel/metadata.json | 27 ++--- .../scenario/__snapshots__/pull.spec.ts.snap | 101 +++++------------- 2 files changed, 41 insertions(+), 87 deletions(-) diff --git a/examples/@default/sandbox/@models/mymodel/metadata.json b/examples/@default/sandbox/@models/mymodel/metadata.json index 567dc65..5dae41e 100644 --- a/examples/@default/sandbox/@models/mymodel/metadata.json +++ b/examples/@default/sandbox/@models/mymodel/metadata.json @@ -38,55 +38,58 @@ "minRelativeProgress": 0.01, "warmStart": false, "earlyStop": true, + "inputLabelColumns": [ + "label" + ], "dataSplitMethod": "AUTO_SPLIT", "learnRateStrategy": "LINE_SEARCH", "initialLearnRate": 0.15, "optimizationStrategy": "BATCH_GRADIENT_DESCENT", "calculatePValues": false }, - "trainingStartTime": "1679395011891", + "trainingStartTime": "1684782126499", "results": [ { "index": 0, - "durationMs": "1931", - "trainingLoss": 1491.0539770637374, + "durationMs": "1896", + "trainingLoss": 1491.0539770637376, "learnRate": 0.3 }, { "index": 1, - "durationMs": "2353", - "trainingLoss": 241.92978201881218, + "durationMs": "2198", + "trainingLoss": 241.92978201881212, "learnRate": 0.6 }, { "index": 2, - "durationMs": "2808", + "durationMs": "2054", "trainingLoss": 41.69727113362378, "learnRate": 0.6 }, { "index": 3, - "durationMs": "1995", - "trainingLoss": 7.920357012251938, + "durationMs": "2052", + "trainingLoss": 7.920357012251929, "learnRate": 0.6 }, { "index": 4, - "durationMs": "2232", + "durationMs": "2336", "trainingLoss": 2.603613218455023, "learnRate": 0.6 } ], "evaluationMetrics": { "regressionMetrics": { - "meanAbsoluteError": 1.3283107802514735, + "meanAbsoluteError": 1.3283107802514729, "meanSquaredError": 2.603613218455023, - "meanSquaredLogError": 0.0012176719511574083, + "meanSquaredLogError": 0.001217671951157408, "medianAbsoluteError": 1.0159802433960223, "rSquared": 0.9968441051897515 } }, - "startTime": "2023-03-21T10:36:51.891Z" + "startTime": "2023-05-22T19:02:06.499Z" } ] } \ No newline at end of file diff --git a/tests/scenario/__snapshots__/pull.spec.ts.snap b/tests/scenario/__snapshots__/pull.spec.ts.snap index ddf53aa..eed777a 100644 --- a/tests/scenario/__snapshots__/pull.spec.ts.snap +++ b/tests/scenario/__snapshots__/pull.spec.ts.snap @@ -24,20 +24,6 @@ Map { ], \\"location\\": \\"US\\" }", - "@default/examples/sample_table/metadata.json" => "{ - \\"type\\": \\"TABLE\\", - \\"clustering\\": { - \\"fields\\": [ - \\"a\\" - ] - } -}", - "@default/examples/sample_table/schema.json" => "[ - { - \\"name\\": \\"a\\", - \\"type\\": \\"INTEGER\\" - } -]", } `; @@ -75,28 +61,6 @@ OPTIONS( ], \\"location\\": \\"US\\" }", - "@default/examples/sample_table/ddl.sql" => "CREATE TABLE IF NOT EXISTS \`bigquery-porter.examples.sample_table\` -( - a INT64 -) -CLUSTER BY a -OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T12:18:34.213Z\\" -);", - "@default/examples/sample_table/metadata.json" => "{ - \\"type\\": \\"TABLE\\", - \\"clustering\\": { - \\"fields\\": [ - \\"a\\" - ] - } -}", - "@default/examples/sample_table/schema.json" => "[ - { - \\"name\\": \\"a\\", - \\"type\\": \\"INTEGER\\" - } -]", "@default/examples_eu/ddl.sql" => "CREATE SCHEMA IF NOT EXISTS \`bigquery-porter.examples_eu\` OPTIONS( default_table_expiration_days=60.0, @@ -125,22 +89,6 @@ OPTIONS( ], \\"location\\": \\"EU\\" }", - "@default/examples_eu/sample_table/ddl.sql" => "CREATE TABLE IF NOT EXISTS \`bigquery-porter.examples_eu.sample_table\` -( - a INT64 -) -OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-12T23:44:41.925Z\\" -);", - "@default/examples_eu/sample_table/metadata.json" => "{ - \\"type\\": \\"TABLE\\" -}", - "@default/examples_eu/sample_table/schema.json" => "[ - { - \\"name\\": \\"a\\", - \\"type\\": \\"INTEGER\\" - } -]", "@default/sample_linked_dataset/README.md" => "ソース リスティング: [JP Holidays preprocessed by BQ FUN](//\\\\(bigquery//analyticshub:projects/120299025068/locations/asia-northeast1/dataExchanges/jpdata_18253a34a30/listings/jp_holidays_preprocessed_by_bq_fun_18253c4e9dc/create-linked-dataset\\\\)\\\\(analyticshub:projects/120299025068/locations/asia-northeast1/dataExchanges/jpdata_18253a34a30/listings/jp_holidays_preprocessed_by_bq_fun_18253c4e9dc\\\\))", "@default/sample_linked_dataset/ddl.sql" => "CREATE SCHEMA IF NOT EXISTS \`bigquery-porter.sample_linked_dataset\` OPTIONS( @@ -233,55 +181,58 @@ OPTIONS( \\"minRelativeProgress\\": 0.01, \\"warmStart\\": false, \\"earlyStop\\": true, + \\"inputLabelColumns\\": [ + \\"label\\" + ], \\"dataSplitMethod\\": \\"AUTO_SPLIT\\", \\"learnRateStrategy\\": \\"LINE_SEARCH\\", \\"initialLearnRate\\": 0.15, \\"optimizationStrategy\\": \\"BATCH_GRADIENT_DESCENT\\", \\"calculatePValues\\": false }, - \\"trainingStartTime\\": \\"1679395011891\\", + \\"trainingStartTime\\": \\"1684782126499\\", \\"results\\": [ { \\"index\\": 0, - \\"durationMs\\": \\"1931\\", - \\"trainingLoss\\": 1491.0539770637374, + \\"durationMs\\": \\"1896\\", + \\"trainingLoss\\": 1491.0539770637376, \\"learnRate\\": 0.3 }, { \\"index\\": 1, - \\"durationMs\\": \\"2353\\", - \\"trainingLoss\\": 241.92978201881218, + \\"durationMs\\": \\"2198\\", + \\"trainingLoss\\": 241.92978201881212, \\"learnRate\\": 0.6 }, { \\"index\\": 2, - \\"durationMs\\": \\"2808\\", + \\"durationMs\\": \\"2054\\", \\"trainingLoss\\": 41.69727113362378, \\"learnRate\\": 0.6 }, { \\"index\\": 3, - \\"durationMs\\": \\"1995\\", - \\"trainingLoss\\": 7.920357012251938, + \\"durationMs\\": \\"2052\\", + \\"trainingLoss\\": 7.920357012251929, \\"learnRate\\": 0.6 }, { \\"index\\": 4, - \\"durationMs\\": \\"2232\\", + \\"durationMs\\": \\"2336\\", \\"trainingLoss\\": 2.603613218455023, \\"learnRate\\": 0.6 } ], \\"evaluationMetrics\\": { \\"regressionMetrics\\": { - \\"meanAbsoluteError\\": 1.3283107802514735, + \\"meanAbsoluteError\\": 1.3283107802514729, \\"meanSquaredError\\": 2.603613218455023, - \\"meanSquaredLogError\\": 0.0012176719511574083, + \\"meanSquaredLogError\\": 0.001217671951157408, \\"medianAbsoluteError\\": 1.0159802433960223, \\"rSquared\\": 0.9968441051897515 } }, - \\"startTime\\": \\"2023-03-21T10:36:51.891Z\\" + \\"startTime\\": \\"2023-05-22T19:02:06.499Z\\" } ] }", @@ -370,7 +321,7 @@ OPTIONS( ) CLUSTER BY a OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T10:39:16.121Z\\" + expiration_timestamp=TIMESTAMP \\"2023-07-21T19:02:15.707Z\\" );", "@default/sandbox/sample_clone_table/metadata.json" => "{ \\"type\\": \\"TABLE\\", @@ -388,7 +339,7 @@ OPTIONS( ]", "@default/sandbox/sample_materialized_view/ddl.sql" => "CREATE MATERIALIZED VIEW IF NOT EXISTS \`bigquery-porter.sandbox.sample_materialized_view\` OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T10:36:56.363Z\\" + expiration_timestamp=TIMESTAMP \\"2023-07-21T19:02:10.020Z\\" ) AS select 1 as a from \`sandbox.sample_table\`;", "@default/sandbox/sample_materialized_view/metadata.json" => "{ @@ -427,7 +378,7 @@ OPTIONS( date_jst DATE ) OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T10:36:53.787Z\\" + expiration_timestamp=TIMESTAMP \\"2023-07-21T19:02:08.362Z\\" );", "@default/sandbox/sample_sharding_@/metadata.json" => "{ \\"type\\": \\"TABLE\\" @@ -439,11 +390,11 @@ OPTIONS( } ]", "@default/sandbox/sample_snapshot_table/ddl.sql" => "CREATE SNAPSHOT TABLE \`bigquery-porter.sandbox.sample_snapshot_table\` -CLONE \`takegue.sandbox.sample_table\` +CLONE \`bigquery-porter.sandbox.sample_table\` OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T10:36:52.966Z\\" + expiration_timestamp=TIMESTAMP \\"2023-07-28T19:01:36.543Z\\" ) -FOR SYSTEM_TIME AS OF TIMESTAMP \\"2023-03-18T10:36:52.311Z\\";", +FOR SYSTEM_TIME AS OF TIMESTAMP \\"2023-05-29T18:01:35.384Z\\";", "@default/sandbox/sample_snapshot_table/metadata.json" => "{ \\"type\\": \\"SNAPSHOT\\", \\"clustering\\": { @@ -453,19 +404,19 @@ FOR SYSTEM_TIME AS OF TIMESTAMP \\"2023-03-18T10:36:52.311Z\\";", }, \\"snapshotDefinition\\": { \\"baseTableReference\\": { - \\"projectId\\": \\"takegue\\", + \\"projectId\\": \\"bigquery-porter\\", \\"datasetId\\": \\"sandbox\\", \\"tableId\\": \\"sample_table\\" }, - \\"snapshotTime\\": \\"2023-03-18T10:36:52.311Z\\" + \\"snapshotTime\\": \\"2023-05-29T18:01:35.384Z\\" }, \\"cloneDefinition\\": { \\"baseTableReference\\": { - \\"projectId\\": \\"takegue\\", + \\"projectId\\": \\"bigquery-porter\\", \\"datasetId\\": \\"sandbox\\", \\"tableId\\": \\"sample_table\\" }, - \\"snapshotTime\\": \\"2023-03-18T10:36:52.311Z\\" + \\"snapshotTime\\": \\"2023-05-29T18:01:35.384Z\\" } }", "@default/sandbox/sample_snapshot_table/schema.json" => "[ @@ -482,7 +433,7 @@ FOR SYSTEM_TIME AS OF TIMESTAMP \\"2023-03-18T10:36:52.311Z\\";", ) CLUSTER BY a OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-05-20T10:36:54.290Z\\" + expiration_timestamp=TIMESTAMP \\"2023-07-21T19:02:08.519Z\\" );", "@default/sandbox/sample_table/metadata.json" => "{ \\"type\\": \\"TABLE\\", From 89042fdfa8f54a864092a289546728b139fbd7a6 Mon Sep 17 00:00:00 2001 From: takegue Date: Fri, 14 Jul 2023 08:48:54 +0900 Subject: [PATCH 2/9] Build smart sweeping bigquery resources options on push --- .../sandbox/@models/mymodel/metadata.json | 4 +- src/commands/cli.ts | 31 ++++- src/commands/push.ts | 6 +- src/tasks/cleanup.ts | 116 ++++++++++++++---- .../scenario/__snapshots__/pull.spec.ts.snap | 4 +- .../scenario/__snapshots__/push.spec.ts.snap | 4 +- tests/task_cleanup.spec.ts | 68 +++++++++- 7 files changed, 192 insertions(+), 41 deletions(-) diff --git a/examples/@default/sandbox/@models/mymodel/metadata.json b/examples/@default/sandbox/@models/mymodel/metadata.json index 5dae41e..31b2ad7 100644 --- a/examples/@default/sandbox/@models/mymodel/metadata.json +++ b/examples/@default/sandbox/@models/mymodel/metadata.json @@ -45,7 +45,9 @@ "learnRateStrategy": "LINE_SEARCH", "initialLearnRate": 0.15, "optimizationStrategy": "BATCH_GRADIENT_DESCENT", - "calculatePValues": false + "calculatePValues": false, + "fitIntercept": true, + "standardizeFeatures": true }, "trainingStartTime": "1684782126499", "results": [ diff --git a/src/commands/cli.ts b/src/commands/cli.ts index d750a7a..e8c0c38 100644 --- a/src/commands/cli.ts +++ b/src/commands/cli.ts @@ -1,7 +1,7 @@ import * as fs from 'node:fs'; import { formatLocalfiles } from '../../src/commands/fix.js'; -import { pushLocalFilesToBigQuery } from '../../src/commands/push.js'; +import { PushContext, pushLocalFilesToBigQuery } from '../../src/commands/push.js'; import { createBundleSQL } from '../../src/commands/bundle.js'; import { pullBigQueryResources } from '../../src/commands/pull.js'; @@ -36,9 +36,9 @@ export function createCLI() { ) .argument('[projects...]') .option( - '--force', - 'Force to apply changes such as deletion without confirmation', - false, + '--sweeping-mode ', + 'Strategy for undefined/orphans BigQuery resources. option: confirm(default), ignore, rename', + 'confirm', ) .option( '--label ', @@ -147,6 +147,22 @@ export function createCLI() { 500, ); + const sweepMode: PushContext['SweepStrategy']['mode'] = (() => { + if(!cmdOptions.sweepingMode) { + return 'confirm' + } + switch (cmdOptions.sweepingMode) { + case 'ignore': + case 'rename_and_7d_expire': + case 'confirm': + case 'force': + return cmdOptions.sweepingMode + default: + // Error for invalid option + throw Error(`Invalid --sweeping-mode option: ${cmdOptions.sweepingMode}`); + } + })(); + for (const project of projects) { const ctx = { BigQuery: { @@ -155,10 +171,13 @@ export function createCLI() { }, rootPath: rootDir, dryRun: cmdOptions.dryRun ?? false, - force: cmdOptions.force ?? false, + SweepStrategy: { + mode: sweepMode, + ignorePrefix: 'zorphan__', + }, reporter: cmdOptions.format ?? 'console', enableDataLineage: cmdOptions.enableDatalineage ?? false, - }; + } as PushContext; const failed = await pushLocalFilesToBigQuery(ctx, jobOption); if (failed > 0) { diff --git a/src/commands/push.ts b/src/commands/push.ts index 98013b9..1a522c2 100644 --- a/src/commands/push.ts +++ b/src/commands/push.ts @@ -28,17 +28,17 @@ import { normalizeShardingTableId, path2bq, } from '../../src/bigquery.js'; -import { createCleanupTasks } from '../../src/tasks/cleanup.js'; +import { SweepStrategy, createCleanupTasks } from '../../src/tasks/cleanup.js'; type PushContext = { dryRun: boolean; - force: boolean; rootPath: string; enableDataLineage: boolean; BigQuery: { projectId: string; client: BigQuery; }; + SweepStrategy: SweepStrategy; reporter: BuiltInReporters; }; @@ -542,4 +542,4 @@ async function pushLocalFilesToBigQuery( return failedTasks; } -export { buildDAG, pushLocalFilesToBigQuery }; +export { buildDAG, pushLocalFilesToBigQuery, PushContext}; diff --git a/src/tasks/cleanup.ts b/src/tasks/cleanup.ts index 2fff1d2..6d13bb0 100644 --- a/src/tasks/cleanup.ts +++ b/src/tasks/cleanup.ts @@ -10,25 +10,36 @@ import { prompt } from '../../src/prompt.js'; type PushContext = { dryRun: boolean; - force: boolean; rootPath: string; BigQuery: { projectId: string; client: BigQuery; }; + SweepStrategy: SweepStrategy; // reporter: BuiltInReporters; }; +type SweepStrategy = { + mode: 'confirm' | 'force' | 'ignore' | 'rename_and_7d_expire'; + ignorePrefix: string; +}; + const cleanupBigQueryDataset = async ( bqClient: BigQuery, rootDir: string, projectId: string, datasetId: string, + mode: SweepStrategy['mode'], options?: { dryRun?: boolean; withoutConrimation: boolean; + ignorePrefix: string; }, ): Promise => { + if(mode === 'ignore') { + return []; + } + const defaultProjectId = await bqClient.getProjectId(); const nsProject = projectId != '@default' ? projectId : defaultProjectId; @@ -113,7 +124,7 @@ const cleanupBigQueryDataset = async ( if (!isForce && !isDryRun) { const ans = await prompt( [ - `Found BigQuery reousrces with no local files. Do you delete these resources? (y/n)`, + `Found BigQuery reousrces with no definition by local files. Do you delete these resources? (y/n)`, `[${resourceType}s]`, ` ${[...kind.keys()].join('\n ')} `, 'Ans>', @@ -125,26 +136,83 @@ const cleanupBigQueryDataset = async ( } for (const [bqId, resource] of kind) { - const task = new BigQueryJobTask( - [ - nsProject, - datasetId, - '(DELETE)', - resourceType.toUpperCase(), - bqId.split('.').pop(), - ] - .join('/'), - async () => { - try { - console.error(`${isDryRun ? '(DRYRUN) ' : ''}Deleting ${bqId}`); - if (!isDryRun) { - await resource.delete(); - } - } catch (e) { + // Skip process prefixed by ignorePrefix + const task: BigQueryJobTask = (() => { + switch(mode) { + case 'rename_and_7d_expire': + return new BigQueryJobTask( + [ + nsProject, + datasetId, + '(RENAME)', + resourceType.toUpperCase(), + bqId.split('.').pop(), + ] + .join('/'), + async () => { + if (options?.ignorePrefix && (bqId.split('.').pop() ?? 'undefined').startsWith(options.ignorePrefix)) { + return {isDryRun}; + } + + if(resourceType === 'Table') { + const dataset = (resource?.parent as Dataset); + const client: BigQuery = (dataset.parent as BigQuery); + const newTableName = `${(options?.ignorePrefix ?? 'zz_')}${resource.id}` + const tableRenameTo = `${dataset.projectId}.${dataset.id}.${newTableName}`; + const query = ` + -- BigQuery Porter: Rename table + ALTER TABLE \`${bqId}\` RENAME TO \`${newTableName}\`; + ALTER TABLE \`${tableRenameTo}\` SET OPTIONS( + expiration_timestamp=TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 7 DAY) + ); + ` + try { + console.error(`${isDryRun ? '(DRYRUN) ' : ''} Renaming ${bqId} to ${tableRenameTo}`); + const [job] = await client.createQueryJob({ + query, + jobPrefix: `bqport-metadata_rename-`, + }) + if (!job) { + throw Error('Renamign Exception: Exception'); + } + return { isDryRun } + } catch (e) { + console.error(e); + } + + } + + return { isDryRun }; + }, + ); + default: + return new BigQueryJobTask( + [ + nsProject, + datasetId, + '(DELETE)', + resourceType.toUpperCase(), + bqId.split('.').pop(), + ] + .join('/'), + async () => { + try { + console.error(`${isDryRun ? '(DRYRUN) ' : '🗑️'} Deleting ${bqId}`); + if (!isDryRun) { + // switch (mode) { + // case 'rename': + // break; + // default: + await resource.delete(); + } + } catch (e) { + } + return { isDryRun }; + }, + ); } - return { isDryRun }; - }, - ); + })(); + tasks.push(task); } } @@ -166,9 +234,11 @@ const createCleanupTasks = async ( ctx.rootPath, ctx.BigQuery.projectId, path.basename(dataset), + ctx.SweepStrategy.mode, { dryRun: ctx.dryRun, - withoutConrimation: ctx.force, + withoutConrimation: ctx.SweepStrategy.mode !== 'confirm', + ignorePrefix: ctx.SweepStrategy.ignorePrefix, }, ).catch((e) => { console.error(e); @@ -180,4 +250,4 @@ const createCleanupTasks = async ( return tasks; }; -export { cleanupBigQueryDataset, createCleanupTasks }; +export { cleanupBigQueryDataset, createCleanupTasks, SweepStrategy}; diff --git a/tests/scenario/__snapshots__/pull.spec.ts.snap b/tests/scenario/__snapshots__/pull.spec.ts.snap index eed777a..021474a 100644 --- a/tests/scenario/__snapshots__/pull.spec.ts.snap +++ b/tests/scenario/__snapshots__/pull.spec.ts.snap @@ -188,7 +188,9 @@ OPTIONS( \\"learnRateStrategy\\": \\"LINE_SEARCH\\", \\"initialLearnRate\\": 0.15, \\"optimizationStrategy\\": \\"BATCH_GRADIENT_DESCENT\\", - \\"calculatePValues\\": false + \\"calculatePValues\\": false, + \\"fitIntercept\\": true, + \\"standardizeFeatures\\": true }, \\"trainingStartTime\\": \\"1684782126499\\", \\"results\\": [ diff --git a/tests/scenario/__snapshots__/push.spec.ts.snap b/tests/scenario/__snapshots__/push.spec.ts.snap index 47dc65d..1495347 100644 --- a/tests/scenario/__snapshots__/push.spec.ts.snap +++ b/tests/scenario/__snapshots__/push.spec.ts.snap @@ -80,8 +80,8 @@ exports[`CLI: push > push --help 1`] = ` Deploy your local BigQuery Resources in topological-sorted order Options: - --force Force to apply changes such as deletion without confirmation (default: - false) + --sweeping-mode Strategy for undefined/orphans BigQuery resources. option: + confirm(default), ignore, backup (default: \\"confirm\\") --label A label to set on a query job. The format is \\"key:value\\"; repeat this option to specify a list of values -p, --parameter Either a file containing a JSON list of query parameters, or a query diff --git a/tests/task_cleanup.spec.ts b/tests/task_cleanup.spec.ts index b383cda..abf4b69 100644 --- a/tests/task_cleanup.spec.ts +++ b/tests/task_cleanup.spec.ts @@ -35,9 +35,11 @@ describe('Task: Cleanup tasks', () => { _root, _project, 'missing_dataset', + 'confirm', { dryRun: true, withoutConrimation: true, + ignorePrefix: '', }, ); @@ -51,9 +53,11 @@ describe('Task: Cleanup tasks', () => { _root, _project, 'missing_dataset', + 'confirm', { dryRun: true, withoutConrimation: true, + ignorePrefix: '', }, ); @@ -97,7 +101,7 @@ describe('Task: Cleanup tasks', () => { } }); - it.concurrent('Should remove table missing in local without prompt', async (ctx) => { + it.concurrent("Shouldn't remove table missing in local without prompt with ignore mode", async (ctx) => { const { settings: { _root, _project } } = ctx as any; const tasks = await createCleanupTasks( { @@ -106,8 +110,56 @@ describe('Task: Cleanup tasks', () => { client: bqClient, }, dryRun: false, - force: true, rootPath: _root, + SweepStrategy: { + mode: 'ignore', + ignorePrefix: '', + } + }, + ); + + expect(tasks.length).toBe(0); + }); + + it.concurrent('Should remove table missing in local with prompt', async (ctx) => { + const { settings: { _root, _project } } = ctx as any; + + const tasks = await createCleanupTasks( + { + BigQuery: { + projectId: _project, + client: bqClient, + }, + dryRun: false, + rootPath: _root, + SweepStrategy: { + mode: 'force', + ignorePrefix: '', + } + }, + ); + + expect(tasks.length).toBe(1); + expect(tasks[0]?.name).toBe( + 'bigquery-public-data/austin_bikeshare/(DELETE)/TABLE/bikeshare_stations', + ); + }); + + it.concurrent('Dry run', async (ctx) => { + const { settings: { _root, _project } } = ctx as any; + + const tasks = await createCleanupTasks( + { + BigQuery: { + projectId: _project, + client: bqClient, + }, + dryRun: true, + rootPath: _root, + SweepStrategy: { + mode: 'confirm', + ignorePrefix: '', + } }, ); @@ -127,14 +179,17 @@ describe('Task: Cleanup tasks', () => { client: bqClient, }, dryRun: false, - force: false, rootPath: _root, + SweepStrategy: { + mode: 'rename_and_7d_expire', + ignorePrefix: '', + } }, ); expect(tasks.length).toBe(1); expect(tasks[0]?.name).toBe( - 'bigquery-public-data/austin_bikeshare/(DELETE)/TABLE/bikeshare_stations', + 'bigquery-public-data/austin_bikeshare/(RENAME)/TABLE/bikeshare_stations', ); }); }); @@ -180,8 +235,11 @@ describe('Task: Cleanup tasks', () => { client: bqClient, }, dryRun: false, - force: true, rootPath: _root, + SweepStrategy: { + mode: 'confirm', + ignorePrefix: '', + } }, ); From d9d60a64a6a8b59e75f185a57ba06633adc6be95 Mon Sep 17 00:00:00 2001 From: takegue Date: Sun, 16 Jul 2023 15:41:33 +0900 Subject: [PATCH 3/9] Update fixture --- tests/scenario/__snapshots__/push.spec.ts.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scenario/__snapshots__/push.spec.ts.snap b/tests/scenario/__snapshots__/push.spec.ts.snap index 1495347..bff2ac1 100644 --- a/tests/scenario/__snapshots__/push.spec.ts.snap +++ b/tests/scenario/__snapshots__/push.spec.ts.snap @@ -81,7 +81,7 @@ Deploy your local BigQuery Resources in topological-sorted order Options: --sweeping-mode Strategy for undefined/orphans BigQuery resources. option: - confirm(default), ignore, backup (default: \\"confirm\\") + confirm(default), ignore, rename (default: \\"confirm\\") --label A label to set on a query job. The format is \\"key:value\\"; repeat this option to specify a list of values -p, --parameter Either a file containing a JSON list of query parameters, or a query From 11a45ce3f95c2a1b85a3ffe01323846543225af2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 17:47:59 +0900 Subject: [PATCH 4/9] chore(deps): Bump semver from 5.7.1 to 5.7.2 (#166) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [semver](https://github.com/npm/node-semver) from 5.7.1 to 5.7.2.
Release notes

Sourced from semver's releases.

v5.7.2

5.7.2 (2023-07-10)

Bug Fixes

Changelog

Sourced from semver's changelog.

5.7.2 (2023-07-10)

Bug Fixes

5.7

  • Add minVersion method

5.6

  • Move boolean loose param to an options object, with backwards-compatibility protection.
  • Add ability to opt out of special prerelease version handling with the includePrerelease option flag.

5.5

  • Add version coercion capabilities

5.4

  • Add intersection checking

5.3

  • Add minSatisfying method

5.2

  • Add prerelease(v) that returns prerelease components

5.1

  • Add Backus-Naur for ranges
  • Remove excessively cute inspection methods

5.0

  • Remove AMD/Browserified build artifacts
  • Fix ltr and gtr when using the * range
  • Fix for range * with a prerelease identifier
Commits
Maintainer changes

This version was pushed to npm by lukekarrys, a new releaser for semver since your current version.


[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=semver&package-manager=npm_and_yarn&previous-version=5.7.1&new-version=5.7.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/takegue/bigquery-porter/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index 250cac7..337b10b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -99,9 +99,9 @@ } }, "node_modules/@babel/core/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" @@ -154,9 +154,9 @@ } }, "node_modules/@babel/helper-compilation-targets/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" @@ -1833,9 +1833,9 @@ } }, "node_modules/concordance/node_modules/semver": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz", - "integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dev": true, "dependencies": { "lru-cache": "^6.0.0" @@ -2689,9 +2689,9 @@ } }, "node_modules/istanbul-lib-instrument/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" @@ -2995,9 +2995,9 @@ } }, "node_modules/make-dir/node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" @@ -3683,9 +3683,9 @@ } }, "node_modules/semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", "bin": { "semver": "bin/semver" } From 0d9a5f29e3931bb36e6781155567be3048b5a60c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 17:51:06 +0900 Subject: [PATCH 5/9] chore(deps): Bump @types/node from 20.1.1 to 20.4.1 (#165) Bumps [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) from 20.1.1 to 20.4.1.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=@types/node&package-manager=npm_and_yarn&previous-version=20.1.1&new-version=20.4.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: takegue --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 337b10b..83bd03b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -994,9 +994,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.1.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.1.tgz", - "integrity": "sha512-uKBEevTNb+l6/aCQaKVnUModfEMjAl98lw2Si9P5y4hLu9tm6AlX2ZIoXZX6Wh9lJueYPrGPKk5WMCNHg/u6/A==" + "version": "20.4.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.1.tgz", + "integrity": "sha512-JIzsAvJeA/5iY6Y/OxZbv1lUcc8dNSE77lb2gnBH+/PJ3lFR1Ccvgwl5JWnHAkNHcRsT0TbpVOsiMKZ1F/yyJg==" }, "node_modules/@vitest/coverage-istanbul": { "version": "0.29.5", From b088edd4894569d323ac302112a3ddff8cbbf5e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 09:21:17 +0000 Subject: [PATCH 6/9] chore(deps-dev): Bump webpack from 5.80.0 to 5.88.1 (#163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [webpack](https://github.com/webpack/webpack) from 5.80.0 to 5.88.1.
Release notes

Sourced from webpack's releases.

v5.88.1

Developer Experience

Full Changelog: https://github.com/webpack/webpack/compare/v5.88.0...v5.88.1

v5.88.0

New Features

Bug Fixes

Developer Experience

Dependencies & Maintenance

New Contributors

Full Changelog: https://github.com/webpack/webpack/compare/v5.87.0...v5.88.0

v5.87.0

New Features

Bug Fixes

Developer Experience

Dependencies & Maintenance

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=webpack&package-manager=npm_and_yarn&previous-version=5.80.0&new-version=5.88.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: takegue --- package-lock.json | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index 83bd03b..9c269c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1328,9 +1328,9 @@ } }, "node_modules/acorn-import-assertions": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz", - "integrity": "sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz", + "integrity": "sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==", "dev": true, "peerDependencies": { "acorn": "^8" @@ -2024,9 +2024,9 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.13.0", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.13.0.tgz", - "integrity": "sha512-eyV8f0y1+bzyfh8xAwW/WTSZpLbjhqc4ne9eGSH4Zo2ejdyiNG9pU6mf9DG8a7+Auk6MFTlNOT4Y2y/9k8GKVg==", + "version": "5.15.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.15.0.tgz", + "integrity": "sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==", "dev": true, "dependencies": { "graceful-fs": "^4.2.4", @@ -3665,9 +3665,9 @@ ] }, "node_modules/schema-utils": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.2.tgz", - "integrity": "sha512-pvjEHOgWc9OWA/f/DE3ohBWTD6EleVLf7iFUkoSwAxttdBhB9QUebQgxER2kWueOvRJXPHNnyrvvh9eZINB8Eg==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", "dev": true, "dependencies": { "@types/json-schema": "^7.0.8", @@ -4459,9 +4459,9 @@ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "node_modules/webpack": { - "version": "5.80.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.80.0.tgz", - "integrity": "sha512-OIMiq37XK1rWO8mH9ssfFKZsXg4n6klTEDL7S8/HqbAOBBaiy8ABvXvz0dDCXeEF9gqwxSvVk611zFPjS8hJxA==", + "version": "5.88.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.88.1.tgz", + "integrity": "sha512-FROX3TxQnC/ox4N+3xQoWZzvGXSuscxR32rbzjpXgEzWudJFEJBpdlkkob2ylrv5yzzufD1zph1OoFsLtm6stQ==", "dev": true, "dependencies": { "@types/eslint-scope": "^3.7.3", @@ -4470,10 +4470,10 @@ "@webassemblyjs/wasm-edit": "^1.11.5", "@webassemblyjs/wasm-parser": "^1.11.5", "acorn": "^8.7.1", - "acorn-import-assertions": "^1.7.6", + "acorn-import-assertions": "^1.9.0", "browserslist": "^4.14.5", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.13.0", + "enhanced-resolve": "^5.15.0", "es-module-lexer": "^1.2.1", "eslint-scope": "5.1.1", "events": "^3.2.0", @@ -4483,7 +4483,7 @@ "loader-runner": "^4.2.0", "mime-types": "^2.1.27", "neo-async": "^2.6.2", - "schema-utils": "^3.1.2", + "schema-utils": "^3.2.0", "tapable": "^2.1.1", "terser-webpack-plugin": "^5.3.7", "watchpack": "^2.4.0", From 373c2b078f6a4b0bdef197d35f96078c0f00d014 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 18:21:41 +0900 Subject: [PATCH 7/9] chore(deps): Bump commander from 10.0.0 to 11.0.0 (#161) Bumps [commander](https://github.com/tj/commander.js) from 10.0.0 to 11.0.0.
Release notes

Sourced from commander's releases.

v11.0.0

Fixed

  • help command works when help option is disabled (#1864)

Changed

  • leading and trailing spaces are now ignored by the .arguments() method (#1874)
  • refine "types" exports for ESM to follow TypeScript guidelines (#1886)
  • Breaking: Commander 11 requires Node.js v16 or higher

v10.0.1

Added

Fixed

  • remove unused Option.optionFlags property from TypeScript definition (#1844)

Changed

  • assume boolean option intended if caller passes string instead of hash to .implies() (#1854)
Changelog

Sourced from commander's changelog.

[11.0.0] (2023-06-16)

Fixed

  • help command works when help option is disabled (#1864)

Changed

  • leading and trailing spaces are now ignored by the .arguments() method (#1874)
  • refine "types" exports for ESM to follow TypeScript guidelines (#1886)
  • Breaking: Commander 11 requires Node.js v16 or higher

[10.0.1] (2023-04-15)

Added

Fixed

  • remove unused Option.optionFlags property from TypeScript definition (#1844)

Changed

  • assume boolean option intended if caller passes string instead of hash to .implies() (#1854)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=commander&package-manager=npm_and_yarn&previous-version=10.0.0&new-version=11.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: takegue --- package-lock.json | 10 +++++----- package.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9c269c1..3367e58 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@google-cloud/bigquery": "^6.0.3", "@types/node": "^20.1.1", - "commander": "^10.0.0", + "commander": "^11.0.0", "log-update": "^5.0.1", "p-throttle": "^5.0.0", "picocolors": "^1.0.0", @@ -1800,11 +1800,11 @@ "dev": true }, "node_modules/commander": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.0.tgz", - "integrity": "sha512-zS5PnTI22FIRM6ylNW8G4Ap0IEOyk62fhLSD0+uHRT9McRCLGpkVNvao4bjimpK/GShynyQkFFxHhwMcETmduA==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.0.0.tgz", + "integrity": "sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ==", "engines": { - "node": ">=14" + "node": ">=16" } }, "node_modules/concat-map": { diff --git a/package.json b/package.json index cfb9149..fe7f7e6 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dependencies": { "@google-cloud/bigquery": "^6.0.3", "@types/node": "^20.1.1", - "commander": "^10.0.0", + "commander": "^11.0.0", "log-update": "^5.0.1", "p-throttle": "^5.0.0", "picocolors": "^1.0.0", From fbffb2aa78981dfee18322a6d332fab60e5afd97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 23 Jul 2023 12:49:12 +0900 Subject: [PATCH 8/9] chore(deps-dev): Bump @vitest/coverage-istanbul from 0.29.5 to 0.31.4 (#167) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [@vitest/coverage-istanbul](https://github.com/vitest-dev/vitest/tree/HEAD/packages/coverage-istanbul) from 0.29.5 to 0.31.4.
Release notes

Sourced from @​vitest/coverage-istanbul's releases.

v0.31.4

   🚀 Features

   🐞 Bug Fixes

    View changes on GitHub

v0.31.3

   🚀 Features

   🐞 Bug Fixes

    View changes on GitHub

v0.31.2

   🚀 Features

   🐞 Bug Fixes

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=@vitest/coverage-istanbul&package-manager=npm_and_yarn&previous-version=0.29.5&new-version=0.31.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 12 ++++++------ package.json | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3367e58..0346999 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ }, "devDependencies": { "@tsconfig/strictest": "^2.0.1", - "@vitest/coverage-istanbul": "^0.29.1", + "@vitest/coverage-istanbul": "^0.31.4", "ts-node": "^10.7.0", "typescript": "^5.0.4", "vitest": "^0.31.0", @@ -999,9 +999,9 @@ "integrity": "sha512-JIzsAvJeA/5iY6Y/OxZbv1lUcc8dNSE77lb2gnBH+/PJ3lFR1Ccvgwl5JWnHAkNHcRsT0TbpVOsiMKZ1F/yyJg==" }, "node_modules/@vitest/coverage-istanbul": { - "version": "0.29.5", - "resolved": "https://registry.npmjs.org/@vitest/coverage-istanbul/-/coverage-istanbul-0.29.5.tgz", - "integrity": "sha512-VTubpTdkOuT3cbo8qAFqCTZ8NXeBOVVHz7lrqueXMJK4WyDIHxnhAdWLrr8HKz5s36acOm6vi6TiFSUm3WHtwQ==", + "version": "0.31.4", + "resolved": "https://registry.npmjs.org/@vitest/coverage-istanbul/-/coverage-istanbul-0.31.4.tgz", + "integrity": "sha512-nQsWQzatd4NXyM7uiIimInOiCopn7gwhFaA8kbBY7u5eYbKyHkMYVgAyqr1gvyPWe4wINdKonqdyQIc110RJkw==", "dev": true, "dependencies": { "istanbul-lib-coverage": "^3.2.0", @@ -1012,10 +1012,10 @@ "test-exclude": "^6.0.0" }, "funding": { - "url": "https://github.com/sponsors/antfu" + "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "vitest": ">=0.28.0 <1" + "vitest": ">=0.30.0 <1" } }, "node_modules/@vitest/expect": { diff --git a/package.json b/package.json index fe7f7e6..1b07332 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ }, "devDependencies": { "@tsconfig/strictest": "^2.0.1", - "@vitest/coverage-istanbul": "^0.29.1", + "@vitest/coverage-istanbul": "^0.31.4", "ts-node": "^10.7.0", "typescript": "^5.0.4", "vitest": "^0.31.0", From 834843498c05c238c32954cb4256f7f9c3b40cfe Mon Sep 17 00:00:00 2001 From: takegue Date: Sun, 23 Jul 2023 22:17:22 +0900 Subject: [PATCH 9/9] Regular fixture update due to bigquery default expiration --- .../sandbox/@models/mymodel/metadata.json | 29 ++++++++------- .../scenario/__snapshots__/pull.spec.ts.snap | 37 ++++++++++--------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/examples/@default/sandbox/@models/mymodel/metadata.json b/examples/@default/sandbox/@models/mymodel/metadata.json index 31b2ad7..257d494 100644 --- a/examples/@default/sandbox/@models/mymodel/metadata.json +++ b/examples/@default/sandbox/@models/mymodel/metadata.json @@ -46,52 +46,53 @@ "initialLearnRate": 0.15, "optimizationStrategy": "BATCH_GRADIENT_DESCENT", "calculatePValues": false, + "enableGlobalExplain": false, "fitIntercept": true, "standardizeFeatures": true }, - "trainingStartTime": "1684782126499", + "trainingStartTime": "1690084242490", "results": [ { "index": 0, - "durationMs": "1896", + "durationMs": "1818", "trainingLoss": 1491.0539770637376, "learnRate": 0.3 }, { "index": 1, - "durationMs": "2198", + "durationMs": "1967", "trainingLoss": 241.92978201881212, "learnRate": 0.6 }, { "index": 2, - "durationMs": "2054", - "trainingLoss": 41.69727113362378, + "durationMs": "1798", + "trainingLoss": 41.697271133623744, "learnRate": 0.6 }, { "index": 3, - "durationMs": "2052", - "trainingLoss": 7.920357012251929, + "durationMs": "1850", + "trainingLoss": 7.920357012251903, "learnRate": 0.6 }, { "index": 4, - "durationMs": "2336", - "trainingLoss": 2.603613218455023, + "durationMs": "1670", + "trainingLoss": 2.603613218455012, "learnRate": 0.6 } ], "evaluationMetrics": { "regressionMetrics": { - "meanAbsoluteError": 1.3283107802514729, - "meanSquaredError": 2.603613218455023, - "meanSquaredLogError": 0.001217671951157408, - "medianAbsoluteError": 1.0159802433960223, + "meanAbsoluteError": 1.3283107802514682, + "meanSquaredError": 2.603613218455012, + "meanSquaredLogError": 0.0012176719511573831, + "medianAbsoluteError": 1.0159802433960081, "rSquared": 0.9968441051897515 } }, - "startTime": "2023-05-22T19:02:06.499Z" + "startTime": "2023-07-23T03:50:42.490Z" } ] } \ No newline at end of file diff --git a/tests/scenario/__snapshots__/pull.spec.ts.snap b/tests/scenario/__snapshots__/pull.spec.ts.snap index 021474a..1c43011 100644 --- a/tests/scenario/__snapshots__/pull.spec.ts.snap +++ b/tests/scenario/__snapshots__/pull.spec.ts.snap @@ -189,52 +189,53 @@ OPTIONS( \\"initialLearnRate\\": 0.15, \\"optimizationStrategy\\": \\"BATCH_GRADIENT_DESCENT\\", \\"calculatePValues\\": false, + \\"enableGlobalExplain\\": false, \\"fitIntercept\\": true, \\"standardizeFeatures\\": true }, - \\"trainingStartTime\\": \\"1684782126499\\", + \\"trainingStartTime\\": \\"1690084242490\\", \\"results\\": [ { \\"index\\": 0, - \\"durationMs\\": \\"1896\\", + \\"durationMs\\": \\"1818\\", \\"trainingLoss\\": 1491.0539770637376, \\"learnRate\\": 0.3 }, { \\"index\\": 1, - \\"durationMs\\": \\"2198\\", + \\"durationMs\\": \\"1967\\", \\"trainingLoss\\": 241.92978201881212, \\"learnRate\\": 0.6 }, { \\"index\\": 2, - \\"durationMs\\": \\"2054\\", - \\"trainingLoss\\": 41.69727113362378, + \\"durationMs\\": \\"1798\\", + \\"trainingLoss\\": 41.697271133623744, \\"learnRate\\": 0.6 }, { \\"index\\": 3, - \\"durationMs\\": \\"2052\\", - \\"trainingLoss\\": 7.920357012251929, + \\"durationMs\\": \\"1850\\", + \\"trainingLoss\\": 7.920357012251903, \\"learnRate\\": 0.6 }, { \\"index\\": 4, - \\"durationMs\\": \\"2336\\", - \\"trainingLoss\\": 2.603613218455023, + \\"durationMs\\": \\"1670\\", + \\"trainingLoss\\": 2.603613218455012, \\"learnRate\\": 0.6 } ], \\"evaluationMetrics\\": { \\"regressionMetrics\\": { - \\"meanAbsoluteError\\": 1.3283107802514729, - \\"meanSquaredError\\": 2.603613218455023, - \\"meanSquaredLogError\\": 0.001217671951157408, - \\"medianAbsoluteError\\": 1.0159802433960223, + \\"meanAbsoluteError\\": 1.3283107802514682, + \\"meanSquaredError\\": 2.603613218455012, + \\"meanSquaredLogError\\": 0.0012176719511573831, + \\"medianAbsoluteError\\": 1.0159802433960081, \\"rSquared\\": 0.9968441051897515 } }, - \\"startTime\\": \\"2023-05-22T19:02:06.499Z\\" + \\"startTime\\": \\"2023-07-23T03:50:42.490Z\\" } ] }", @@ -323,7 +324,7 @@ OPTIONS( ) CLUSTER BY a OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-07-21T19:02:15.707Z\\" + expiration_timestamp=TIMESTAMP \\"2023-09-21T03:50:49.196Z\\" );", "@default/sandbox/sample_clone_table/metadata.json" => "{ \\"type\\": \\"TABLE\\", @@ -341,7 +342,7 @@ OPTIONS( ]", "@default/sandbox/sample_materialized_view/ddl.sql" => "CREATE MATERIALIZED VIEW IF NOT EXISTS \`bigquery-porter.sandbox.sample_materialized_view\` OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-07-21T19:02:10.020Z\\" + expiration_timestamp=TIMESTAMP \\"2023-09-21T03:50:46.175Z\\" ) AS select 1 as a from \`sandbox.sample_table\`;", "@default/sandbox/sample_materialized_view/metadata.json" => "{ @@ -380,7 +381,7 @@ OPTIONS( date_jst DATE ) OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-07-21T19:02:08.362Z\\" + expiration_timestamp=TIMESTAMP \\"2023-09-21T03:50:44.134Z\\" );", "@default/sandbox/sample_sharding_@/metadata.json" => "{ \\"type\\": \\"TABLE\\" @@ -435,7 +436,7 @@ FOR SYSTEM_TIME AS OF TIMESTAMP \\"2023-05-29T18:01:35.384Z\\";", ) CLUSTER BY a OPTIONS( - expiration_timestamp=TIMESTAMP \\"2023-07-21T19:02:08.519Z\\" + expiration_timestamp=TIMESTAMP \\"2023-09-21T03:50:44.115Z\\" );", "@default/sandbox/sample_table/metadata.json" => "{ \\"type\\": \\"TABLE\\",