Validate local schema and connectors together before deploy (#952)

* Starting on build

* types

* Progress!

* Prettify errors

* Line numbers, and save deployment metadata

* Extremely defensive

src/commands/dataconnect-sdk-generate.ts

@@ -4,7 +4,7 @@ import { Command } from "../command";
 import { Options } from "../options";
 import { DataConnectEmulator, DataConnectEmulatorArgs } from "../emulator/dataconnectEmulator";
 import { needProjectId } from "../projectUtils";
-import { load } from "../dataconnect/source";
+import { load } from "../dataconnect/load";
 import { readFirebaseJson } from "../dataconnect/fileUtils";
 import { logger } from "../logger";
 

src/dataconnect/build.ts

@@ -0,0 +1,24 @@
+import { DataConnectEmulator, DataConnectEmulatorArgs } from "../emulator/dataconnectEmulator";
+import { Options } from "../options";
+import { FirebaseError } from "../error";
+import { prettify } from "./graphqlError";
+import { DeploymentMetadata } from "./types";
+
+export async function build(options: Options, configDir: string): Promise<DeploymentMetadata> {
+  // We can build even if there is no project declared.
+  const projectId = options.project ?? "demo-test";
+  const args: DataConnectEmulatorArgs = {
+    projectId,
+    configDir,
+    auto_download: true,
+    rc: options.rc,
+  };
+  const dataconnectEmulator = new DataConnectEmulator(args);
+  const buildResult = await dataconnectEmulator.build();
+  if (buildResult?.errors?.length) {
+    throw new FirebaseError(
+      `There are errors in your schema and connector files:\n${buildResult.errors.map(prettify).join("\n")}`,
+    );
+  }
+  return buildResult?.deploymentMetadata ?? {};
+}

src/dataconnect/fileUtils.ts

@@ -6,7 +6,7 @@ import { ConnectorYaml, DataConnectYaml, File, ServiceInfo } from "./types";
 import { readFileFromDirectory, wrappedSafeLoad } from "../utils";
 import { Config } from "../config";
 import { DataConnectMultiple } from "../firebaseConfig";
-import { load } from "./source";
+import { load } from "./load";
 
 export function readFirebaseJson(config: Config): DataConnectMultiple {
   if (!config.has("dataconnect")) {

src/dataconnect/graphqlError.ts

@@ -0,0 +1,10 @@
+import { GraphqlError } from "./types";
+
+export function prettify(err: GraphqlError): string {
+  const message = err.message;
+  let header = err.extensions.file ?? "";
+  for (const loc of err.locations) {
+    header += `(${loc.line}, ${loc.column})`;
+  }
+  return header.length ? `${header}: ${message}` : message;
+}

src/dataconnect/source.ts → src/dataconnect/load.ts

@@ -33,6 +33,7 @@ export async function load(
 
   return {
     serviceName,
+    sourceDirectory,
     schema: {
       name: `${serviceName}/schemas/${SCHEMA_ID}`,
       primaryDatasource: toDatasource(projectId, locationId, dataConnectYaml.schema.datasource),

src/dataconnect/types.ts

@@ -65,6 +65,30 @@ export interface Diff {
   destructive: boolean;
 }
 
+export interface GraphqlError {
+  message: string;
+  locations: {
+    line: number;
+    column: number;
+  }[];
+  extensions: {
+    file?: string;
+    [key: string]: any;
+  };
+}
+export interface BuildResult {
+  errors?: GraphqlError[];
+  deploymentMetadata?: DeploymentMetadata;
+}
+
+export interface DeploymentMetadata {
+  primaryDataSource?: {
+    postgres?: {
+      requiredExtensions?: string[];
+    };
+  };
+}
+
 // YAML types
 export interface DataConnectYaml {
   specVersion?: string;
@@ -113,12 +137,14 @@ export interface KotlinSDK {
 // Helper types && converters
 export interface ServiceInfo {
   serviceName: string;
+  sourceDirectory: string;
   schema: Schema;
   connectorInfo: {
     connector: Connector;
     connectorYaml: ConnectorYaml;
   }[];
   dataConnectYaml: DataConnectYaml;
+  deploymentMetadata?: DeploymentMetadata;
 }
 
 export function toDatasource(

src/deploy/dataconnect/prepare.ts

@@ -1,14 +1,15 @@
 import * as path from "path";
 
 import { Options } from "../../options";
-import { load } from "../../dataconnect/source";
+import { load } from "../../dataconnect/load";
 import { readFirebaseJson } from "../../dataconnect/fileUtils";
 import { logger } from "../../logger";
 import * as utils from "../../utils";
 import { ensure } from "../../ensureApiEnabled";
 import { needProjectId } from "../../projectUtils";
 import { dataconnectOrigin } from "../../api";
 import { getResourceFilters } from "../../dataconnect/filters";
+import { build } from "../../dataconnect/build";
 
 /**
  * Prepares for a Firebase DataConnect deployment by loading schemas and connectors from file.
@@ -21,15 +22,18 @@ export default async function (context: any, options: Options): Promise<void> {
   const serviceCfgs = readFirebaseJson(options.config);
   utils.logLabeledBullet("dataconnect", `Preparing to deploy`);
   const filters = getResourceFilters(options);
-  context.dataconnect = {
-    serviceInfos: await Promise.all(
-      serviceCfgs.map((c) =>
-        load(projectId, c.location, path.join(options.cwd || process.cwd(), c.source)),
-      ),
+  const serviceInfos = await Promise.all(
+    serviceCfgs.map((c) =>
+      load(projectId, c.location, path.join(options.cwd || process.cwd(), c.source)),
     ),
+  );
+  for (const si of serviceInfos) {
+    si.deploymentMetadata = await build(options, si.sourceDirectory);
+  }
+  context.dataconnect = {
+    serviceInfos,
     filters,
   };
-
   utils.logLabeledBullet("dataconnect", `Successfully prepared schema and connectors`);
   logger.debug(JSON.stringify(context.dataconnect, null, 2));
   return;

src/emulator/dataconnectEmulator.ts

@@ -7,6 +7,7 @@ import { EmulatorInfo, EmulatorInstance, Emulators } from "./types";
 import { FirebaseError } from "../error";
 import { EmulatorLogger } from "./emulatorLogger";
 import { RC } from "../rc";
+import { BuildResult } from "../dataconnect/types";
 
 export interface DataConnectEmulatorArgs {
   projectId?: string;
@@ -71,6 +72,22 @@ export class DataConnectEmulator implements EmulatorInstance {
     return res.stdout;
   }
 
+  async build(): Promise<BuildResult> {
+    const commandInfo = await downloadIfNecessary(Emulators.DATACONNECT);
+    const cmd = ["build", `--config_dir=${this.args.configDir}`];
+
+    const res = childProcess.spawnSync(commandInfo.binary, cmd, { encoding: "utf-8" });
+    if (res.error) {
+      throw new FirebaseError(`Error starting up Data Connect emulator: ${res.error}`);
+    }
+    try {
+      return JSON.parse(res.stdout) as BuildResult;
+    } catch (err) {
+      // JSON parse errors are unreadable.
+      throw new FirebaseError("Unable to parse `fdc build` output");
+    }
+  }
+
   private getLocalConectionString() {
     if (dataConnectLocalConnString()) {
       return dataConnectLocalConnString();