Support webhooks in Data Connect behind an experiment flag.

firebase-vscode/src/data-connect/config.ts

@@ -1,6 +1,6 @@
 import { isPathInside } from "./file-utils";
 import { DeepReadOnly } from "../metaprogramming";
-import { ConnectorYaml, DataConnectYaml } from "../dataconnect/types";
+import { ConnectorYaml, DataConnectYaml, mainSchemaYaml } from "../../../src/dataconnect/types";
 import { Result, ResultValue } from "../result";
 import { computed, effect, signal } from "@preact/signals-core";
 import {
@@ -265,7 +265,7 @@ export class ResolvedDataConnectConfig {
   }
 
   get schemaDir(): string {
-    return this.value.schema.source;
+    return mainSchemaYaml(this.value).source;
   }
 
   get relativePath(): string {

src/commands/dataconnect-services-list.ts

@@ -8,6 +8,7 @@
 import { ensureApis } from "../dataconnect/ensureApis";
 import * as Table from "cli-table3";
 
+// TODO: Update this command to also list secondary schema information.
 export const command = new Command("dataconnect:services:list")
   .description("list all deployed Data Connect services")
   .before(requirePermissions, [
@@ -19,7 +20,7 @@
     const projectId = needProjectId(options);
     await ensureApis(projectId);
     const services = await client.listAllServices(projectId);
     const table: Record<string, any>[] = new Table({
       head: [
         "Service ID",
         "Location",
@@ -30,7 +31,7 @@
       ],
       style: { head: ["yellow"] },
     });
     const jsonOutput: { services: Record<string, any>[] } = { services: [] };
     for (const service of services) {
       const schema = (await client.getSchema(service.name)) ?? {
         name: "",
@@ -43,7 +44,7 @@
       const instanceName = postgresDatasource?.postgresql?.cloudSql?.instance ?? "";
       const instanceId = instanceName.split("/").pop();
       const dbId = postgresDatasource?.postgresql?.database ?? "";
       const dbName = `CloudSQL Instance: ${instanceId}\nDatabase: ${dbId}`;
       table.push([
         serviceName.serviceId,
         serviceName.location,

src/commands/dataconnect-sql-diff.ts

@@ -6,6 +6,7 @@ import { requirePermissions } from "../requirePermissions";
 import { pickService } from "../dataconnect/load";
 import { diffSchema } from "../dataconnect/schemaMigration";
 import { requireAuth } from "../requireAuth";
+import { mainSchema, mainSchemaYaml } from "../dataconnect/types";
 
 export const command = new Command("dataconnect:sql:diff [serviceId]")
   .description(
@@ -24,8 +25,8 @@ export const command = new Command("dataconnect:sql:diff [serviceId]")
 
     const diffs = await diffSchema(
       options,
-      serviceInfo.schema,
-      serviceInfo.dataConnectYaml.schema.datasource.postgresql?.schemaValidation,
+      mainSchema(serviceInfo.schemas),
+      mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.schemaValidation,
     );
     return { projectId, serviceId, diffs };
   });

src/commands/dataconnect-sql-grant.ts

@@ -9,6 +9,7 @@ import { requireAuth } from "../requireAuth";
 import { FirebaseError } from "../error";
 import { fdcSqlRoleMap } from "../gcp/cloudsql/permissionsSetup";
 import { iamUserIsCSQLAdmin } from "../gcp/cloudsql/cloudsqladmin";
+import { mainSchema } from "../dataconnect/types";
 
 const allowedRoles = Object.keys(fdcSqlRoleMap);
 
@@ -51,6 +52,6 @@ export const command = new Command("dataconnect:sql:grant [serviceId]")
     await ensureApis(projectId);
     const serviceInfo = await pickService(projectId, options.config, serviceId);
 
-    await grantRoleToUserInSchema(options, serviceInfo.schema);
+    await grantRoleToUserInSchema(options, mainSchema(serviceInfo.schemas));
     return { projectId, serviceId };
   });

src/commands/dataconnect-sql-migrate.ts

@@ -8,6 +8,7 @@ import { requireAuth } from "../requireAuth";
 import { requirePermissions } from "../requirePermissions";
 import { ensureApis } from "../dataconnect/ensureApis";
 import { logLabeledSuccess } from "../utils";
+import { mainSchema, mainSchemaYaml } from "../dataconnect/types";
 
 export const command = new Command("dataconnect:sql:migrate [serviceId]")
   .description("migrate your CloudSQL database's schema to match your local Data Connect schema")
@@ -23,18 +24,19 @@ export const command = new Command("dataconnect:sql:migrate [serviceId]")
     const projectId = needProjectId(options);
     await ensureApis(projectId);
     const serviceInfo = await pickService(projectId, options.config, serviceId);
-    const instanceId =
-      serviceInfo.dataConnectYaml.schema.datasource.postgresql?.cloudSql.instanceId;
+    const instanceId = mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.cloudSql
+      .instanceId;
     if (!instanceId) {
       throw new FirebaseError(
         "dataconnect.yaml is missing field schema.datasource.postgresql.cloudsql.instanceId",
       );
     }
     const diffs = await migrateSchema({
       options,
-      schema: serviceInfo.schema,
+      schema: mainSchema(serviceInfo.schemas),
       validateOnly: true,
-      schemaValidation: serviceInfo.dataConnectYaml.schema.datasource.postgresql?.schemaValidation,
+      schemaValidation: mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql
+        ?.schemaValidation,
     });
     if (diffs.length) {
       logLabeledSuccess(

src/commands/dataconnect-sql-setup.ts

@@ -10,6 +10,7 @@ import { setupSQLPermissions, getSchemaMetadata } from "../gcp/cloudsql/permissi
 import { DEFAULT_SCHEMA } from "../gcp/cloudsql/permissions";
 import { getIdentifiers, ensureServiceIsConnectedToCloudSql } from "../dataconnect/schemaMigration";
 import { setupIAMUsers } from "../gcp/cloudsql/connect";
+import { mainSchema, mainSchemaYaml } from "../dataconnect/types";
 
 export const command = new Command("dataconnect:sql:setup [serviceId]")
   .description("set up your CloudSQL database")
@@ -24,15 +25,17 @@ export const command = new Command("dataconnect:sql:setup [serviceId]")
     const projectId = needProjectId(options);
     await ensureApis(projectId);
     const serviceInfo = await pickService(projectId, options.config, serviceId);
-    const instanceId =
-      serviceInfo.dataConnectYaml.schema.datasource.postgresql?.cloudSql.instanceId;
+    const instanceId = mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.cloudSql
+      .instanceId;
     if (!instanceId) {
       throw new FirebaseError(
         "dataconnect.yaml is missing field schema.datasource.postgresql.cloudsql.instanceId",
       );
     }
 
-    const { serviceName, instanceName, databaseId } = getIdentifiers(serviceInfo.schema);
+    const { serviceName, instanceName, databaseId } = getIdentifiers(
+      mainSchema(serviceInfo.schemas),
+    );
     await ensureServiceIsConnectedToCloudSql(
       serviceName,
       instanceName,

src/commands/dataconnect-sql-shell.ts

@@ -17,6 +17,7 @@
 import { FirebaseError } from "../error";
 import { FBToolsAuthClient } from "../gcp/cloudsql/fbToolsAuthClient";
 import { confirmDangerousQuery, interactiveExecuteQuery } from "../gcp/cloudsql/interactive";
+import { mainSchema } from "../dataconnect/types";
 
 // Not a comprehensive list, used for keyword coloring.
 const sqlKeywords = [
@@ -62,8 +63,8 @@
   return query;
 }
 
 async function mainShellLoop(conn: pg.PoolClient) {
   while (true) {
     const query = await promptForQuery();
     if (query.toLowerCase() === ".exit") {
       break;
@@ -91,7 +92,7 @@
     const projectId = needProjectId(options);
     await ensureApis(projectId);
     const serviceInfo = await pickService(projectId, options.config, serviceId);
-    const { instanceId, databaseId } = getIdentifiers(serviceInfo.schema);
+    const { instanceId, databaseId } = getIdentifiers(mainSchema(serviceInfo.schemas));
     const { user: username } = await getIAMUser(options);
     const instance = await cloudSqlAdminClient.getInstance(projectId, instanceId);
 
@@ -99,7 +100,7 @@
     const connectionName = instance.connectionName;
     if (!connectionName) {
       throw new FirebaseError(
         `Could not get instance connection string for ${options.instanceId}:${options.databaseId}`,
       );
     }
     const connector: Connector = new Connector({

src/dataconnect/client.spec.ts

@@ -7,7 +7,7 @@
 import { FirebaseError } from "../error";
 import * as types from "./types";
 
 chai.use(require("chai-as-promised"));
 
 describe("client", () => {
   let sandbox: sinon.SinonSandbox;
@@ -124,6 +124,13 @@
       expect(getStub).to.be.calledWith("projects/p/locations/l/services/s/schemas/main");
     });
 
+    it("getSchema with schemaId", async () => {
+      getStub.resolves({ body: { name: "schema" } });
+      const schema = await client.getSchema("projects/p/locations/l/services/s", "schemaId");
+      expect(schema).to.deep.equal({ name: "schema" });
+      expect(getStub).to.be.calledWith("projects/p/locations/l/services/s/schemas/schemaId");
+    });
+
     it("getSchema returns undefined if not found", async () => {
       getStub.rejects(new FirebaseError("err", { status: 404 }));
       const schema = await client.getSchema("projects/p/locations/l/services/s");

src/dataconnect/client.ts

@@ -6,7 +6,7 @@
 const DATACONNECT_API_VERSION = "v1";
 const PAGE_SIZE_MAX = 100;
 
 const dataconnectClient = () =>
   new Client({
     urlPrefix: dataconnectOrigin(),
     apiVersion: DATACONNECT_API_VERSION,
@@ -83,11 +83,12 @@
 
 /** Schema methods */
 
-export async function getSchema(serviceName: string): Promise<types.Schema | undefined> {
+export async function getSchema(
+  serviceName: string,
+  schemaId: string = types.MAIN_SCHEMA_ID,
+): Promise<types.Schema | undefined> {
   try {
-    const res = await dataconnectClient().get<types.Schema>(
-      `${serviceName}/schemas/${types.SCHEMA_ID}`,
-    );
+    const res = await dataconnectClient().get<types.Schema>(`${serviceName}/schemas/${schemaId}`);
     return res.body;
   } catch (err: any) {
     if (err.status !== 404) {
@@ -146,7 +147,7 @@
 
 export async function deleteSchema(serviceName: string): Promise<void> {
   const op = await dataconnectClient().delete<types.Schema>(
-    `${serviceName}/schemas/${types.SCHEMA_ID}`,
+    `${serviceName}/schemas/${types.MAIN_SCHEMA_ID}`,
   );
   await operationPoller.pollOperation<void>({
     apiOrigin: dataconnectOrigin(),

src/dataconnect/load.ts

@@ -7,7 +7,7 @@ import { Config } from "../config";
 import { FirebaseError } from "../error";
 import {
   toDatasource,
-  SCHEMA_ID,
+  MAIN_SCHEMA_ID,
   ConnectorYaml,
   DataConnectYaml,
   File,
@@ -16,6 +16,7 @@ import {
 } from "./types";
 import { readFileFromDirectory, wrappedSafeLoad } from "../utils";
 import { DataConnectMultiple } from "../firebaseConfig";
+import * as experiments from "../experiments";
 
 // pickService reads firebase.json and returns all services with a given serviceId.
 // If serviceID is not provided and there is a single service, return that.
@@ -77,8 +78,20 @@ export async function load(
   const resolvedDir = config.path(sourceDirectory);
   const dataConnectYaml = await readDataConnectYaml(resolvedDir);
   const serviceName = `projects/${projectId}/locations/${dataConnectYaml.location}/services/${dataConnectYaml.serviceId}`;
-  const schemaDir = path.join(resolvedDir, dataConnectYaml.schema.source);
-  const schemaGQLs = await readGQLFiles(schemaDir);
+  const schemaYamls = dataConnectYaml.schema ? [dataConnectYaml.schema] : dataConnectYaml.schemas;
+  const schemas = await Promise.all(
+    schemaYamls!.map(async (yaml) => {
+      const schemaDir = path.join(resolvedDir, yaml.source);
+      const schemaGQLs = await readGQLFiles(schemaDir);
+      return {
+        name: `${serviceName}/schemas/${yaml.id || MAIN_SCHEMA_ID}`,
+        datasources: [toDatasource(projectId, dataConnectYaml.location, yaml.datasource)],
+        source: {
+          files: schemaGQLs,
+        },
+      };
+    }),
+  );
   const connectorInfo = await Promise.all(
     dataConnectYaml.connectorDirs.map(async (dir) => {
       const connectorDir = path.join(resolvedDir, dir);
@@ -100,15 +113,7 @@ export async function load(
   return {
     serviceName,
     sourceDirectory: resolvedDir,
-    schema: {
-      name: `${serviceName}/schemas/${SCHEMA_ID}`,
-      datasources: [
-        toDatasource(projectId, dataConnectYaml.location, dataConnectYaml.schema.datasource),
-      ],
-      source: {
-        files: schemaGQLs,
-      },
-    },
+    schemas: schemas,
     dataConnectYaml,
     connectorInfo,
   };
@@ -149,6 +154,12 @@ function validateDataConnectYaml(unvalidated: any): DataConnectYaml {
   if (!unvalidated["location"]) {
     throw new FirebaseError("Missing required field 'location' in dataconnect.yaml");
   }
+  if (!experiments.isEnabled("fdcwebhooks") && unvalidated["schemas"]) {
+    throw new FirebaseError("Unsupported field 'schemas' in dataconnect.yaml");
+  }
+  if (!unvalidated["schema"] && !unvalidated["schemas"]) {
+    throw new FirebaseError("Either 'schema' or 'schemas' is required in dataconnect.yaml");
+  }
   return unvalidated as DataConnectYaml;
 }
 

src/dataconnect/schemaMigration.ts

@@ -1,7 +1,7 @@
 import * as clc from "colorette";
 import { format } from "sql-formatter";
 
-import { IncompatibleSqlSchemaError, Diff, SCHEMA_ID, SchemaValidation } from "./types";
+import { IncompatibleSqlSchemaError, Diff, MAIN_SCHEMA_ID, SchemaValidation } from "./types";
 import { getSchema, upsertSchema, deleteConnector } from "./client";
 import {
   getIAMUser,
@@ -178,7 +178,7 @@ export async function migrateSchema(args: {
     const postgresql = schema.datasources.find((d) => d.postgresql)?.postgresql;
     if (!postgresql) {
       throw new FirebaseError(
-        `Cannot find Postgres datasource in the schema to deploy: ${serviceName}/schemas/${SCHEMA_ID}.\nIts datasources: ${JSON.stringify(schema.datasources)}`,
+        `Cannot find Postgres datasource in the schema to deploy: ${serviceName}/schemas/${MAIN_SCHEMA_ID}.\nIts datasources: ${JSON.stringify(schema.datasources)}`,
       );
     }
     postgresql.schemaValidation = "NONE";
@@ -374,7 +374,7 @@ export function getIdentifiers(schema: Schema): {
     );
   }
   const instanceId = instanceName.split("/").pop()!;
-  const serviceName = schema.name.replace(`/schemas/${SCHEMA_ID}`, "");
+  const serviceName = schema.name.replace(`/schemas/${MAIN_SCHEMA_ID}`, "");
   return {
     databaseId,
     instanceId,
@@ -626,7 +626,7 @@ export async function ensureServiceIsConnectedToCloudSql(
     logLabeledBullet("dataconnect", `Linking the Cloud SQL instance...`);
     // If no schema has been deployed yet, deploy an empty one to get connectivity.
     currentSchema = {
-      name: `${serviceName}/schemas/${SCHEMA_ID}`,
+      name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`,
       source: {
         files: [],
       },

src/dataconnect/types.ts

@@ -1,5 +1,5 @@
-// Schema is a singleton, so we always call it 'main'
-export const SCHEMA_ID = "main";
+// The database schema ID is always 'main'
+export const MAIN_SCHEMA_ID = "main";
 
 // API Types
 interface BaseResource {
@@ -26,7 +26,9 @@ export interface Connector extends BaseResource {
 }
 
 export interface Datasource {
+  // One of postgresql or httpGraphql must be set.
   postgresql?: PostgreSql;
+  httpGraphql?: HttpGraphql;
 }
 
 export type SchemaValidation = "STRICT" | "COMPATIBLE";
@@ -39,6 +41,11 @@ export interface PostgreSql {
   schemaMigration?: "MIGRATE_COMPATIBLE";
 }
 
+export interface HttpGraphql {
+  uri: string;
+  timeout?: string;
+}
+
 export interface CloudSqlInstance {
   instance: string;
 }
@@ -116,13 +123,16 @@ export function requiresVector(dm?: DeploymentMetadata): boolean {
 export interface DataConnectYaml {
   specVersion?: string;
   serviceId: string;
-  schema: SchemaYaml;
+  // One of `schema` or `schemas` is required.
+  schema?: SchemaYaml;
+  schemas?: SchemaYaml[];
   location: string;
   connectorDirs: string[];
 }
 
 export interface SchemaYaml {
   source: string;
+  id?: string;
   datasource: DatasourceYaml;
 }
 
@@ -134,6 +144,10 @@ export interface DatasourceYaml {
     };
     schemaValidation?: SchemaValidation;
   };
+  httpGraphql?: {
+    uri: string;
+    timeout?: string;
+  };
 }
 
 export interface ConnectorYaml {
@@ -183,7 +197,7 @@ export interface DartSDK {
 export interface ServiceInfo {
   serviceName: string;
   sourceDirectory: string;
-  schema: Schema;
+  schemas: Schema[];
   connectorInfo: ConnectorInfo[];
   dataConnectYaml: DataConnectYaml;
   deploymentMetadata?: DeploymentMetadata;
@@ -211,9 +225,43 @@ export function toDatasource(
       },
     };
   }
+  if (ds?.httpGraphql) {
+    return {
+      httpGraphql: {
+        uri: ds.httpGraphql.uri,
+        timeout: ds.httpGraphql.timeout,
+      },
+    };
+  }
   return {};
 }
 
+/** Returns the main schema YAML for a Data Connect YAML */
+export function mainSchemaYaml(dataconnectYaml: DataConnectYaml): SchemaYaml {
+  if (dataconnectYaml.schema) {
+    return dataconnectYaml.schema;
+  }
+  const mainSch = dataconnectYaml.schemas?.find((s) => s.id === MAIN_SCHEMA_ID || !s.id);
+  if (!mainSch) {
+    throw new Error(`Service ${dataconnectYaml.serviceId} has no main schema defined`);
+  }
+  return mainSch;
+}
+
+/** Returns the main schema from a list of schemas */
+export function mainSchema(schemas: Schema[]): Schema {
+  const mainSch = schemas.find((s) => isMainSchema(s));
+  if (!mainSch) {
+    throw new Error(`No main schema is defined`);
+  }
+  return mainSch;
+}
+
+/** Returns true if the schema is the main schema */
+export function isMainSchema(schema: Schema): boolean {
+  return schema.name.endsWith(`/schemas/${MAIN_SCHEMA_ID}`);
+}
+
 /** Start Dataplane Client Types */
 export interface ExecuteGraphqlRequest {
   query: string;