Remove deprecated run cache and gcloud project ID fields (#1584)

* Remove deprecated run cache and gcloud project ID fields * Remove run cache test * Tidying
dataform-co · Nov 22, 2023 · d544851 · d544851
1 parent 2a8f584
commit d544851
Show file tree

Hide file tree

Showing 9 changed files with 7 additions and 95 deletions.
diff --git a/cli/api/commands/build.ts b/cli/api/commands/build.ts
@@ -12,26 +12,12 @@ export async function build(
   runConfig: dataform.IRunConfig,
   dbadapter: dbadapters.IDbAdapter
 ) {
-  runConfig = {
-    ...runConfig,
-    useRunCache: false
-  };
-
   const prunedGraph = prune(compiledGraph, runConfig);
 
   const allInvolvedTargets = new StringifiedSet<dataform.ITarget>(
     targetStringifier,
     prunedGraph.tables.map(table => table.target)
   );
-  if (runConfig.useRunCache) {
-    for (const includedAction of [
-      ...prunedGraph.tables,
-      ...prunedGraph.operations,
-      ...prunedGraph.assertions
-    ]) {
-      allInvolvedTargets.add(includedAction.target);
-    }
-  }
 
   return new Builder(
     prunedGraph,

diff --git a/cli/api/commands/run.ts b/cli/api/commands/run.ts
@@ -383,21 +383,7 @@ export class Runner {
       }
     }
 
-    let newMetadata: dataform.ITableMetadata;
-    if (this.graph.projectConfig.useRunCache) {
-      try {
-        newMetadata = await this.dbadapter.table(action.target);
-      } catch (e) {
-        // Ignore Errors thrown when trying to get new table metadata; just allow the relevant
-        // warehouseStateAfterRunByTarget entry to be cleared out (below).
-      }
-    }
-    if (newMetadata) {
-      this.warehouseStateByTarget.set(action.target, newMetadata);
-      this.notifyListeners();
-    } else {
-      this.warehouseStateByTarget.delete(action.target);
-    }
+    this.warehouseStateByTarget.delete(action.target);
 
     if (actionResult.status === dataform.ActionResult.ExecutionStatus.RUNNING) {
       actionResult.status = dataform.ActionResult.ExecutionStatus.SUCCESSFUL;

diff --git a/core/main.ts b/core/main.ts
@@ -28,12 +28,6 @@ export function main(coreExecutionRequest: Uint8Array | string): Uint8Array | st
 
   const projectConfigOverride = compileRequest.compileConfig.projectConfigOverride ?? {};
 
-  // Stop using the deprecated 'gcloudProjectId' field.
-  if (!originalProjectConfig.defaultDatabase) {
-    originalProjectConfig.defaultDatabase = originalProjectConfig.gcloudProjectId;
-  }
-  delete originalProjectConfig.gcloudProjectId;
-
   let projectConfig = { ...originalProjectConfig };
 
   // Merge in general project config overrides.

diff --git a/core/session.ts b/core/session.ts
@@ -401,16 +401,6 @@ export class Session {
       [].concat(compiledGraph.tables, compiledGraph.assertions, compiledGraph.operations)
     );
 
-    if (this.config.useRunCache) {
-      this.checkRunCachingCorrectness(
-        [].concat(
-          compiledGraph.tables,
-          compiledGraph.assertions,
-          compiledGraph.operations.filter(operation => operation.hasOutput)
-        )
-      );
-    }
-
     utils.throwIfInvalid(compiledGraph, dataform.CompiledGraph.verify);
     return compiledGraph;
   }
@@ -675,28 +665,6 @@ export class Session {
     });
   }
 
-  private checkRunCachingCorrectness(actionsWithOutput: IActionProto[]) {
-    actionsWithOutput.forEach(action => {
-      if (action.dependencyTargets?.length > 0) {
-        return;
-      }
-      if (
-        [dataform.ActionHermeticity.HERMETIC, dataform.ActionHermeticity.NON_HERMETIC].includes(
-          action.hermeticity
-        )
-      ) {
-        return;
-      }
-      this.compileError(
-        new Error(
-          "Zero-dependency actions which create datasets are required to explicitly declare 'hermetic: (true|false)' when run caching is turned on."
-        ),
-        action.fileName,
-        action.target
-      );
-    });
-  }
-
   private removeNonUniqueActionsFromCompiledGraph(compiledGraph: dataform.CompiledGraph) {
     function getNonUniqueTargets(targets: dataform.ITarget[]): StringifiedSet<dataform.ITarget> {
       const allTargets = new StringifiedSet<dataform.ITarget>(targetStringifier);

diff --git a/protos/core.proto b/protos/core.proto
@@ -19,28 +19,15 @@ message ProjectConfig {
 
   map<string, string> vars = 14;
 
-  int32 concurrent_query_limit = 13;
-  int32 idempotent_action_retries = 8;
-
   string database_suffix = 15;
   string schema_suffix = 7;
   string table_prefix = 11;
 
-  // Deprecated.
-  // Please use 'default_database' instead.
-  string gcloud_project_id = 6;
-  // This setting no longer has any effect.
-  bool use_run_cache = 10;
-
-  reserved 3, 4, 12;
-}
-
-message GenerateIndexConfig {
-  CompileConfig compile_config = 1;
-  repeated string include_paths = 2;
-  repeated string definition_paths = 3;
+  // TODO(ekrekr): Move these to be CLI flags instead.
+  int32 concurrent_query_limit = 13;
+  int32 idempotent_action_retries = 8;
 
-  reserved 4;
+  reserved 3, 4, 6, 10, 12;
 }
 
 message CompileConfig {

diff --git a/tests/api/projects.spec.ts b/tests/api/projects.spec.ts
@@ -82,11 +82,6 @@ suite("examples", () => {
             {
               fileName: "definitions/has_compile_errors/table_with_materialized.sqlx",
               message: "The 'materialized' option is only valid for BigQuery views"
-            },
-            {
-              fileName: "definitions/has_compile_errors/view_without_hermetic.sqlx",
-              message:
-                "Zero-dependency actions which create datasets are required to explicitly declare 'hermetic: (true|false)' when run caching is turned on."
             }
           ]);
 

diff --git a/tests/api/projects/common_v2/dataform.json b/tests/api/projects/common_v2/dataform.json
@@ -3,6 +3,5 @@
   "defaultDatabase": "tada-analytics",
   "defaultSchema": "df_integration_test",
   "assertionSchema": "df_integration_test_assertions",
-  "useRunCache": true,
   "defaultLocation": "US"
 }
diff --git a/tests/cli/cli.spec.ts b/tests/cli/cli.spec.ts
@@ -160,8 +160,7 @@ select 1 as \${dataform.projectConfig.vars.testVar2}
         }
       },
       runConfig: {
-        fullRefresh: false,
-        useRunCache: false
+        fullRefresh: false
       },
       warehouseState: {}
     });

diff --git a/tests/integration/bigquery.spec.ts b/tests/integration/bigquery.spec.ts
@@ -257,9 +257,7 @@ suite("@dataform/integration/bigquery", { parallel: true }, ({ before, after })
   suite("evaluate", async () => {
     test("evaluate from valid compiled graph as valid", async () => {
       // Create and run the project.
-      const compiledGraph = await compile("tests/integration/bigquery_project", "evaluate", {
-        useRunCache: false
-      });
+      const compiledGraph = await compile("tests/integration/bigquery_project", "evaluate");
       const executionGraph = await dfapi.build(compiledGraph, {}, dbadapter);
       await dfapi.run(dbadapter, executionGraph).result();