Remove JS context from SQL files (#1672)

* Remove JS context from SQL files * Tidy * Update example projects * Tidy * Simpler sql escaping and example
dataform-co · Feb 8, 2024 · 800e36d · 800e36d
1 parent 994203b
commit 800e36d
Show file tree

Hide file tree

Showing 15 changed files with 43 additions and 48 deletions.
diff --git a/core/actions/assertion.ts b/core/actions/assertion.ts
@@ -109,7 +109,7 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
       description: config.description
     });
 
-    this.query(nativeRequire(config.filename).queryAsContextable);
+    this.query(nativeRequire(config.filename).query);
   }
 
   public config(config: IAssertionConfig) {

diff --git a/core/actions/operation.ts b/core/actions/operation.ts
@@ -108,7 +108,7 @@ export class Operation extends ActionBuilder<dataform.Operation> {
       description: config.description
     });
 
-    this.queries(nativeRequire(config.filename).queryAsContextable);
+    this.queries(nativeRequire(config.filename).query);
   }
 
   public config(config: IOperationConfig) {

diff --git a/core/actions/table.ts b/core/actions/table.ts
@@ -425,7 +425,7 @@ export class Table extends ActionBuilder<dataform.Table> {
         bigquery: bigqueryOptions
       });
     }
-    this.query(nativeRequire(tableTypeConfig.filename).queryAsContextable);
+    this.query(nativeRequire(tableTypeConfig.filename).query);
     if (tableTypeConfig.preOperations) {
       this.preOps(tableTypeConfig.preOperations);
     }

diff --git a/core/compilers.ts b/core/compilers.ts
@@ -42,11 +42,11 @@ export function compile(code: string, path: string): string {
     return `exports.asJson = ${notebookAsJson}`;
   }
   if (Path.fileExtension(path) === "sql") {
-    const { sql } = extractSqlxParts(SyntaxTreeNode.create(code));
-    return `exports.queryAsContextable = (ctx) => {
-      ${CONTEXT_FUNCTIONS}
-      return \`${sql.join("")}\`;
-    }`;
+    const escapedCode = code
+      .replace(/\\/g, "\\\\")
+      .replace(/`/g, "\\`")
+      .replace(/\${/g, "\\${");
+    return `exports.query = \`${escapedCode}\`;`;
   }
   return code;
 }

diff --git a/core/main.ts b/core/main.ts
@@ -70,6 +70,7 @@ export function main(coreExecutionRequest: Uint8Array | string): Uint8Array | st
   globalAny.operate = session.operate.bind(session);
   globalAny.assert = session.assert.bind(session);
   globalAny.declare = session.declare.bind(session);
+  globalAny.notebook = session.notebook.bind(session);
   globalAny.test = session.test.bind(session);
 
   loadActionConfigs(session, compileRequest.compileConfig.filePaths);

diff --git a/core/main_test.ts b/core/main_test.ts
@@ -765,10 +765,7 @@ actions:
 - operation:
     filename: action.sql`
       );
-      fs.writeFileSync(
-        path.join(projectDir, "definitions/action.sql"),
-        "SELECT ${database()} AS proofThatContextIsRead"
-      );
+      fs.writeFileSync(path.join(projectDir, "definitions/action.sql"), "SELECT 1");
 
       const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
 
@@ -780,7 +777,7 @@ actions:
               name: "action"
             },
             fileName: "definitions/action.sql",
-            queries: ["SELECT dataform AS proofThatContextIsRead"],
+            queries: ["SELECT 1"],
             target: {
               database: "dataform",
               name: "action"
@@ -878,10 +875,7 @@ actions:
 - table:
     filename: action.sql`
       );
-      fs.writeFileSync(
-        path.join(projectDir, "definitions/action.sql"),
-        "SELECT ${database()} AS proofThatContextIsRead"
-      );
+      fs.writeFileSync(path.join(projectDir, "definitions/action.sql"), "SELECT 1");
 
       const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
 
@@ -893,7 +887,7 @@ actions:
               name: "action"
             },
             fileName: "definitions/action.sql",
-            query: "SELECT dataform AS proofThatContextIsRead",
+            query: "SELECT 1",
             target: {
               database: "dataform",
               name: "action"
@@ -924,11 +918,7 @@ actions:
     -  someKey1
     -  someKey2`
       );
-      fs.writeFileSync(
-        path.join(projectDir, "definitions/action.sql"),
-        "SELECT ${database()} AS ${when(incremental(), `proofThatIncrementalContextIsRead`, " +
-          "`proofThatContextIsRead`) }"
-      );
+      fs.writeFileSync(path.join(projectDir, "definitions/action.sql"), "SELECT 1");
 
       const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
 
@@ -940,8 +930,8 @@ actions:
               name: "action"
             },
             fileName: "definitions/action.sql",
-            query: "SELECT dataform AS proofThatContextIsRead",
-            incrementalQuery: "SELECT dataform AS proofThatIncrementalContextIsRead",
+            query: "SELECT 1",
+            incrementalQuery: "SELECT 1",
             target: {
               database: "dataform",
               name: "action"
@@ -970,10 +960,7 @@ actions:
 - view:
     filename: action.sql`
       );
-      fs.writeFileSync(
-        path.join(projectDir, "definitions/action.sql"),
-        "SELECT ${database()} AS proofThatContextIsRead"
-      );
+      fs.writeFileSync(path.join(projectDir, "definitions/action.sql"), "SELECT 1");
 
       const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
 
@@ -985,7 +972,7 @@ actions:
               name: "action"
             },
             fileName: "definitions/action.sql",
-            query: "SELECT dataform AS proofThatContextIsRead",
+            query: "SELECT 1",
             target: {
               database: "dataform",
               name: "action"
@@ -1012,10 +999,7 @@ actions:
 - assertion:
     filename: action.sql`
       );
-      fs.writeFileSync(
-        path.join(projectDir, "definitions/action.sql"),
-        "SELECT ${database()} AS proofThatContextIsRead"
-      );
+      fs.writeFileSync(path.join(projectDir, "definitions/action.sql"), "SELECT 1");
 
       const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
 
@@ -1027,7 +1011,7 @@ actions:
               name: "action"
             },
             fileName: "definitions/action.sql",
-            query: "SELECT dataform AS proofThatContextIsRead",
+            query: "SELECT 1",
             target: {
               database: "dataform",
               name: "action"
@@ -1094,7 +1078,7 @@ actions:
       );
       fs.writeFileSync(
         path.join(projectDir, "definitions/utf8characters:私🙂 and some spaces.sql"),
-        "SELECT ${database()} AS proofThatContextIsRead"
+        "SELECT 1"
       );
 
       const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
@@ -1107,7 +1091,7 @@ actions:
               name: "utf8characters:私🙂 and some spaces"
             },
             fileName: "definitions/utf8characters:私🙂 and some spaces.sql",
-            queries: ["SELECT dataform AS proofThatContextIsRead"],
+            queries: ["SELECT 1"],
             target: {
               database: "dataform",
               name: "utf8characters:私🙂 and some spaces"

diff --git a/examples/examples_test.ts b/examples/examples_test.ts
@@ -10,15 +10,15 @@ suite("examples", () => {
 
     const graph = await compile({ projectDir: "examples/stackoverflow_reporter" });
 
-    expect(graph.graphErrors.compilationErrors.length).equals(0);
+    expect(graph.graphErrors.compilationErrors).deep.equals([]);
   });
 
   test("extreme_weather_programming compiles", async () => {
     fs.copySync("examples/node_modules", "examples/extreme_weather_programming/node_modules");
 
     const graph = await compile({ projectDir: "examples/extreme_weather_programming" });
 
-    expect(graph.graphErrors.compilationErrors.length).equals(0);
+    expect(graph.graphErrors.compilationErrors).deep.equals([]);
     expect(graph.tables.length).equals(3);
     expect(graph.notebooks.length).equals(1);
   });

diff --git a/examples/extreme_weather_programming/definitions/actions.yaml b/examples/extreme_weather_programming/definitions/actions.yaml
@@ -1,9 +1,14 @@
 actions:
   - view:
       filename: repositories_created_during_extreme_weather.sql
+      dependencyTargets:
+        - name: was_there_extreme_weather
+        - name: repositories_that_mention_extreme_weather
   - view:
       filename: repositories_that_mention_extreme_weather.sql
   - view:
       filename: was_there_extreme_weather.sql
   - notebook:
       filename: snowy_repository_creation.ipynb
+      dependencyTargets:
+        - name: repositories_created_during_extreme_weather
diff --git a/...s/extreme_weather_programming/definitions/repositories_created_during_extreme_weather.sql b/...s/extreme_weather_programming/definitions/repositories_created_during_extreme_weather.sql
@@ -1,7 +1,7 @@
 SELECT
     *
 FROM
-    ${ ref("was_there_extreme_weather") }
-    LEFT OUTER JOIN ${ ref("repositories_that_mention_extreme_weather") } USING (date)
+    `dataform-demos.was_there_extreme_weather`
+    LEFT OUTER JOIN `dataform-demos.repositories_that_mention_extreme_weather` USING (date)
 ORDER BY
     date
diff --git a/...les/extreme_weather_programming/definitions/repositories_that_mention_extreme_weather.sql b/...les/extreme_weather_programming/definitions/repositories_that_mention_extreme_weather.sql
@@ -6,7 +6,7 @@ FROM
 WHERE
     REGEXP_CONTAINS(
         repository_description,
-        "${constants.WEATHER.join(" | ")}"
+        "snow | hail | thunder | tornado"
     )
 GROUP BY
     date

diff --git a/examples/extreme_weather_programming/definitions/was_there_extreme_weather.sql b/examples/extreme_weather_programming/definitions/was_there_extreme_weather.sql
@@ -5,7 +5,10 @@ FROM
     `bigquery-public-data.samples.gsod`
 WHERE
     (
-        ${ constants.WEATHER.map(weather => `${weather} = TRUE`).join(" OR\n   ") }
+        snow = TRUE
+        OR hail = TRUE
+        OR thunder = TRUE
+        OR tornado = TRUE
     ) -- The GitHub data only ranges from 2007 to 2012.
     -- The weather data is only available up to 2010.
     AND year >= 2007

diff --git a/examples/extreme_weather_programming/includes/constants.js b/examples/extreme_weather_programming/includes/constants.js
diff --git a/examples/extreme_weather_programming/workflow_settings.yaml b/examples/extreme_weather_programming/workflow_settings.yaml
@@ -1,2 +1,3 @@
-defaultDataset: "dataform-demos"
+defaultProject: "dataform-demos"
 defaultLocation: "us"
+defaultDataset: "dataform"
diff --git a/protos/configs.proto b/protos/configs.proto
@@ -243,6 +243,10 @@ message ActionConfig {
 
     // is specified, if a row arrives whose key matches an existing row's key,
     // then the existing row is overwritten with the new data.
+
+    // If set, unique key represents a set of names of columns that will act as
+    // a the unique key. To enforce this, when updating the incremental
+    // table, Dataform merges rows with `uniqueKey` instead of appending them.
     repeated string unique_key = 11;
 
     // Description of the incremental table.

diff --git a/version.bzl b/version.bzl
@@ -1,3 +1,3 @@
 # NOTE: If you change the format of this line, you must change the bash command
 # in /scripts/publish to extract the version string correctly.
-DF_VERSION = "3.0.0-alpha.4"
+DF_VERSION = "3.0.0-alpha.5"