From dc1a6524ab540ca82914ce91ecd37d7bb930fb1a Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Fri, 7 Nov 2025 13:31:03 -0800 Subject: [PATCH 01/15] Support webhooks in Data Connect behind an experiment flag. --- src/experiments.ts | 5 +++++ src/init/features/dataconnect/index.ts | 8 +++++++- .../init/dataconnect/dataconnect-fdcwebhooks.yaml | 13 +++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 templates/init/dataconnect/dataconnect-fdcwebhooks.yaml diff --git a/src/experiments.ts b/src/experiments.ts index a18e4ca8eb7..7ec9aa6975c 100644 --- a/src/experiments.ts +++ b/src/experiments.ts @@ -148,6 +148,11 @@ export const ALL_EXPERIMENTS = experiments({ shortDescription: "Adds experimental App Testing feature", public: true, }, + fdcwebhooks: { + shortDescription: "Enable Firebase Data Connect webhooks feature.", + default: false, + public: false, + }, }); export type ExperimentName = keyof typeof ALL_EXPERIMENTS; diff --git a/src/init/features/dataconnect/index.ts b/src/init/features/dataconnect/index.ts index 11f460ba1ef..8b2188b3df9 100644 --- a/src/init/features/dataconnect/index.ts +++ b/src/init/features/dataconnect/index.ts @@ -9,6 +9,7 @@ import { setupCloudSql } from "../../../dataconnect/provisionCloudSql"; import { checkFreeTrialInstanceUsed, upgradeInstructions } from "../../../dataconnect/freeTrial"; import * as cloudsql from "../../../gcp/cloudsql/cloudsqladmin"; import { ensureApis, ensureGIFApiTos } from "../../../dataconnect/ensureApis"; +import * as experiments from "../../../experiments"; import { listLocations, listAllServices, @@ -44,6 +45,9 @@ import { trackGA4 } from "../../../track"; export const FDC_DEFAULT_REGION = "us-east4"; const DATACONNECT_YAML_TEMPLATE = readTemplateSync("init/dataconnect/dataconnect.yaml"); +const DATACONNECT_YAML_WEBHOOKS_EXPERIMENT_TEMPLATE = readTemplateSync( + "init/dataconnect/dataconnect-fdcwebhooks.yaml", +); const CONNECTOR_YAML_TEMPLATE = readTemplateSync("init/dataconnect/connector.yaml"); const SCHEMA_TEMPLATE = readTemplateSync("init/dataconnect/schema.gql"); const QUERIES_TEMPLATE = readTemplateSync("init/dataconnect/queries.gql"); @@ -478,7 +482,9 @@ function subDataconnectYamlValues(replacementValues: { cloudSqlInstanceId: "__cloudSqlInstanceId__", connectorDirs: "__connectorDirs__", }; - let replaced = DATACONNECT_YAML_TEMPLATE; + let replaced = experiments.isEnabled("fdcwebhooks") + ? DATACONNECT_YAML_WEBHOOKS_EXPERIMENT_TEMPLATE + : DATACONNECT_YAML_TEMPLATE; for (const [k, v] of Object.entries(replacementValues)) { replaced = replaced.replace(replacements[k], JSON.stringify(v)); } diff --git a/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml b/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml new file mode 100644 index 00000000000..91c262c1831 --- /dev/null +++ b/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml @@ -0,0 +1,13 @@ +specVersion: "v1" +serviceId: __serviceId__ +location: __location__ +schemas: + - source: "./schemas/main" + datasource: + postgresql: + database: __cloudSqlDatabase__ + cloudSql: + instanceId: __cloudSqlInstanceId__ + # schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly. + # schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect. +connectorDirs: __connectorDirs__ From 5623de9861bd3a8fb63533df4513ae8b6c588973 Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Fri, 7 Nov 2025 15:57:02 -0800 Subject: [PATCH 02/15] Update existing logic to specifically refer to the main schema. --- src/dataconnect/client.ts | 4 ++-- src/dataconnect/load.ts | 4 ++-- src/dataconnect/schemaMigration.ts | 8 ++++---- src/dataconnect/types.ts | 27 +++++++++++++++++++++++--- src/deploy/dataconnect/deploy.ts | 4 ++-- src/deploy/dataconnect/prepare.ts | 6 +++--- src/deploy/dataconnect/release.ts | 4 ++-- src/init/features/dataconnect/index.ts | 8 ++++---- 8 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/dataconnect/client.ts b/src/dataconnect/client.ts index b0135de8939..2cca3813e54 100644 --- a/src/dataconnect/client.ts +++ b/src/dataconnect/client.ts @@ -86,7 +86,7 @@ export async function deleteService(serviceName: string): Promise export async function getSchema(serviceName: string): Promise { try { const res = await dataconnectClient().get( - `${serviceName}/schemas/${types.SCHEMA_ID}`, + `${serviceName}/schemas/${types.MAIN_SCHEMA_ID}`, ); return res.body; } catch (err: any) { @@ -146,7 +146,7 @@ export async function upsertSchema( export async function deleteSchema(serviceName: string): Promise { const op = await dataconnectClient().delete( - `${serviceName}/schemas/${types.SCHEMA_ID}`, + `${serviceName}/schemas/${types.MAIN_SCHEMA_ID}`, ); await operationPoller.pollOperation({ apiOrigin: dataconnectOrigin(), diff --git a/src/dataconnect/load.ts b/src/dataconnect/load.ts index 829dddd05a6..f53480a27c1 100644 --- a/src/dataconnect/load.ts +++ b/src/dataconnect/load.ts @@ -7,7 +7,7 @@ import { Config } from "../config"; import { FirebaseError } from "../error"; import { toDatasource, - SCHEMA_ID, + MAIN_SCHEMA_ID, ConnectorYaml, DataConnectYaml, File, @@ -101,7 +101,7 @@ export async function load( serviceName, sourceDirectory: resolvedDir, schema: { - name: `${serviceName}/schemas/${SCHEMA_ID}`, + name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`, datasources: [ toDatasource(projectId, dataConnectYaml.location, dataConnectYaml.schema.datasource), ], diff --git a/src/dataconnect/schemaMigration.ts b/src/dataconnect/schemaMigration.ts index 7de85a86d7f..fb9e5c24e8d 100644 --- a/src/dataconnect/schemaMigration.ts +++ b/src/dataconnect/schemaMigration.ts @@ -1,7 +1,7 @@ import * as clc from "colorette"; import { format } from "sql-formatter"; -import { IncompatibleSqlSchemaError, Diff, SCHEMA_ID, SchemaValidation } from "./types"; +import { IncompatibleSqlSchemaError, Diff, MAIN_SCHEMA_ID, SchemaValidation } from "./types"; import { getSchema, upsertSchema, deleteConnector } from "./client"; import { getIAMUser, @@ -178,7 +178,7 @@ export async function migrateSchema(args: { const postgresql = schema.datasources.find((d) => d.postgresql)?.postgresql; if (!postgresql) { throw new FirebaseError( - `Cannot find Postgres datasource in the schema to deploy: ${serviceName}/schemas/${SCHEMA_ID}.\nIts datasources: ${JSON.stringify(schema.datasources)}`, + `Cannot find Postgres datasource in the schema to deploy: ${serviceName}/schemas/${MAIN_SCHEMA_ID}.\nIts datasources: ${JSON.stringify(schema.datasources)}`, ); } postgresql.schemaValidation = "NONE"; @@ -374,7 +374,7 @@ export function getIdentifiers(schema: Schema): { ); } const instanceId = instanceName.split("/").pop()!; - const serviceName = schema.name.replace(`/schemas/${SCHEMA_ID}`, ""); + const serviceName = schema.name.replace(`/schemas/${MAIN_SCHEMA_ID}`, ""); return { databaseId, instanceId, @@ -626,7 +626,7 @@ export async function ensureServiceIsConnectedToCloudSql( logLabeledBullet("dataconnect", `Linking the Cloud SQL instance...`); // If no schema has been deployed yet, deploy an empty one to get connectivity. currentSchema = { - name: `${serviceName}/schemas/${SCHEMA_ID}`, + name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`, source: { files: [], }, diff --git a/src/dataconnect/types.ts b/src/dataconnect/types.ts index c9b7dfeaa2d..6274f2f5e93 100644 --- a/src/dataconnect/types.ts +++ b/src/dataconnect/types.ts @@ -1,5 +1,5 @@ -// Schema is a singleton, so we always call it 'main' -export const SCHEMA_ID = "main"; +// The database schema ID is always 'main' +export const MAIN_SCHEMA_ID = "main"; // API Types interface BaseResource { @@ -15,6 +15,7 @@ export interface Service extends BaseResource { export interface Schema extends BaseResource { name: string; + id?: string; datasources: Datasource[]; source: Source; @@ -26,7 +27,9 @@ export interface Connector extends BaseResource { } export interface Datasource { + // One of postgresql or httpGraphql must be set. postgresql?: PostgreSql; + httpGraphql?: HttpGraphql; } export type SchemaValidation = "STRICT" | "COMPATIBLE"; @@ -39,6 +42,11 @@ export interface PostgreSql { schemaMigration?: "MIGRATE_COMPATIBLE"; } +export interface HttpGraphql { + uri: string; + timeout?: string; +} + export interface CloudSqlInstance { instance: string; } @@ -183,7 +191,9 @@ export interface DartSDK { export interface ServiceInfo { serviceName: string; sourceDirectory: string; - schema: Schema; + // One of `schema` or `schemas` is required. + schema?: Schema; + schemas?: Schema[]; connectorInfo: ConnectorInfo[]; dataConnectYaml: DataConnectYaml; deploymentMetadata?: DeploymentMetadata; @@ -214,6 +224,17 @@ export function toDatasource( return {}; } +/** Returns the main schema for a service info */ +export function mainSchema(serviceInfo: ServiceInfo): Schema { + if (serviceInfo.schema) { + return serviceInfo.schema; + } + if (serviceInfo.schemas && serviceInfo.schemas.length > 0) { + return serviceInfo.schemas.find((s) => s.name.endsWith(`/schemas/${MAIN_SCHEMA_ID}`))!; + } + throw new Error(`Service ${serviceInfo.serviceName} has no schema defined`); +} + /** Start Dataplane Client Types */ export interface ExecuteGraphqlRequest { query: string; diff --git a/src/deploy/dataconnect/deploy.ts b/src/deploy/dataconnect/deploy.ts index e2620f99515..6822e4b1357 100644 --- a/src/deploy/dataconnect/deploy.ts +++ b/src/deploy/dataconnect/deploy.ts @@ -1,7 +1,7 @@ import { Options } from "../../options"; import * as client from "../../dataconnect/client"; import * as utils from "../../utils"; -import { Service, ServiceInfo, requiresVector } from "../../dataconnect/types"; +import { Service, ServiceInfo, mainSchema, requiresVector } from "../../dataconnect/types"; import { needProjectId } from "../../projectUtils"; import { setupCloudSql } from "../../dataconnect/provisionCloudSql"; import { parseServiceName } from "../../dataconnect/names"; @@ -88,7 +88,7 @@ export default async function (context: Context, options: Options): Promise { - const postgresDatasource = s.schema.datasources.find((d) => d.postgresql); + const postgresDatasource = mainSchema(s).datasources.find((d) => d.postgresql); if (postgresDatasource) { const instanceId = postgresDatasource.postgresql?.cloudSql?.instance.split("/").pop(); const databaseId = postgresDatasource.postgresql?.database; diff --git a/src/deploy/dataconnect/prepare.ts b/src/deploy/dataconnect/prepare.ts index 6624ab685c0..94f02733c83 100644 --- a/src/deploy/dataconnect/prepare.ts +++ b/src/deploy/dataconnect/prepare.ts @@ -14,7 +14,7 @@ import { setupCloudSql } from "../../dataconnect/provisionCloudSql"; import { checkBillingEnabled } from "../../gcp/cloudbilling"; import { parseServiceName } from "../../dataconnect/names"; import { FirebaseError } from "../../error"; -import { requiresVector } from "../../dataconnect/types"; +import { mainSchema, requiresVector } from "../../dataconnect/types"; import { diffSchema } from "../../dataconnect/schemaMigration"; import { upgradeInstructions } from "../../dataconnect/freeTrial"; import { Context, initDeployStats } from "./context"; @@ -65,7 +65,7 @@ export default async function (context: Context, options: DeployOptions): Promis for (const si of serviceInfos) { await diffSchema( options, - si.schema, + mainSchema(si), si.dataConnectYaml.schema?.datasource?.postgresql?.schemaValidation, ); } @@ -76,7 +76,7 @@ export default async function (context: Context, options: DeployOptions): Promis return !filters || filters?.some((f) => si.dataConnectYaml.serviceId === f.serviceId); }) .map(async (s) => { - const postgresDatasource = s.schema.datasources.find((d) => d.postgresql); + const postgresDatasource = mainSchema(s).datasources.find((d) => d.postgresql); if (postgresDatasource) { const instanceId = postgresDatasource.postgresql?.cloudSql?.instance.split("/").pop(); const databaseId = postgresDatasource.postgresql?.database; diff --git a/src/deploy/dataconnect/release.ts b/src/deploy/dataconnect/release.ts index 044ed039b3f..314f078b82f 100644 --- a/src/deploy/dataconnect/release.ts +++ b/src/deploy/dataconnect/release.ts @@ -1,5 +1,5 @@ import * as utils from "../../utils"; -import { Connector, ServiceInfo } from "../../dataconnect/types"; +import { Connector, mainSchema, ServiceInfo } from "../../dataconnect/types"; import { listConnectors, upsertConnector } from "../../dataconnect/client"; import { promptDeleteConnector } from "../../dataconnect/prompts"; import { Options } from "../../options"; @@ -35,7 +35,7 @@ export default async function (context: Context, options: Options): Promise ({ - schema: s.schema, + schema: mainSchema(s), validationMode: s.dataConnectYaml?.schema?.datasource?.postgresql?.schemaValidation, })); const wantConnectors = serviceInfos.flatMap((si) => diff --git a/src/init/features/dataconnect/index.ts b/src/init/features/dataconnect/index.ts index 8b2188b3df9..c8903c5be71 100644 --- a/src/init/features/dataconnect/index.ts +++ b/src/init/features/dataconnect/index.ts @@ -18,7 +18,7 @@ import { createService, upsertSchema, } from "../../../dataconnect/client"; -import { Schema, Service, File, SCHEMA_ID } from "../../../dataconnect/types"; +import { Schema, Service, File, MAIN_SCHEMA_ID } from "../../../dataconnect/types"; import { parseCloudSQLInstanceName, parseServiceName } from "../../../dataconnect/names"; import { logger } from "../../../logger"; import { readTemplateSync } from "../../../templates"; @@ -350,7 +350,7 @@ function schemasDeploySequence( // No Cloud SQL is being provisioned, just deploy the schema sources as a unlinked schema. return [ { - name: `${serviceName}/schemas/${SCHEMA_ID}`, + name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`, datasources: [{ postgresql: {} }], source: { files: schemaFiles, @@ -363,7 +363,7 @@ function schemasDeploySequence( // wait for Cloud SQL provision to finish and setup its initial SQL schemas. return [ { - name: `${serviceName}/schemas/${SCHEMA_ID}`, + name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`, datasources: [ { postgresql: { @@ -380,7 +380,7 @@ function schemasDeploySequence( }, }, { - name: `${serviceName}/schemas/${SCHEMA_ID}`, + name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`, datasources: [ { postgresql: { From c9e92f94daa6c9b681117f72500a24d23090e730 Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Fri, 7 Nov 2025 16:17:23 -0800 Subject: [PATCH 03/15] Fix build errors. --- src/commands/dataconnect-sql-diff.ts | 3 ++- src/commands/dataconnect-sql-grant.ts | 3 ++- src/commands/dataconnect-sql-migrate.ts | 3 ++- src/commands/dataconnect-sql-setup.ts | 3 ++- src/commands/dataconnect-sql-shell.ts | 3 ++- src/mcp/prompts/dataconnect/schema.ts | 4 ++-- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/commands/dataconnect-sql-diff.ts b/src/commands/dataconnect-sql-diff.ts index 73035ad52e8..66b7b000342 100644 --- a/src/commands/dataconnect-sql-diff.ts +++ b/src/commands/dataconnect-sql-diff.ts @@ -6,6 +6,7 @@ import { requirePermissions } from "../requirePermissions"; import { pickService } from "../dataconnect/load"; import { diffSchema } from "../dataconnect/schemaMigration"; import { requireAuth } from "../requireAuth"; +import { mainSchema } from "../dataconnect/types"; export const command = new Command("dataconnect:sql:diff [serviceId]") .description( @@ -24,7 +25,7 @@ export const command = new Command("dataconnect:sql:diff [serviceId]") const diffs = await diffSchema( options, - serviceInfo.schema, + mainSchema(serviceInfo), serviceInfo.dataConnectYaml.schema.datasource.postgresql?.schemaValidation, ); return { projectId, serviceId, diffs }; diff --git a/src/commands/dataconnect-sql-grant.ts b/src/commands/dataconnect-sql-grant.ts index 30a25da9e9e..f5ae96253a6 100644 --- a/src/commands/dataconnect-sql-grant.ts +++ b/src/commands/dataconnect-sql-grant.ts @@ -9,6 +9,7 @@ import { requireAuth } from "../requireAuth"; import { FirebaseError } from "../error"; import { fdcSqlRoleMap } from "../gcp/cloudsql/permissionsSetup"; import { iamUserIsCSQLAdmin } from "../gcp/cloudsql/cloudsqladmin"; +import { mainSchema } from "../dataconnect/types"; const allowedRoles = Object.keys(fdcSqlRoleMap); @@ -51,6 +52,6 @@ export const command = new Command("dataconnect:sql:grant [serviceId]") await ensureApis(projectId); const serviceInfo = await pickService(projectId, options.config, serviceId); - await grantRoleToUserInSchema(options, serviceInfo.schema); + await grantRoleToUserInSchema(options, mainSchema(serviceInfo)); return { projectId, serviceId }; }); diff --git a/src/commands/dataconnect-sql-migrate.ts b/src/commands/dataconnect-sql-migrate.ts index bcc72943327..1797d3a05fd 100644 --- a/src/commands/dataconnect-sql-migrate.ts +++ b/src/commands/dataconnect-sql-migrate.ts @@ -8,6 +8,7 @@ import { requireAuth } from "../requireAuth"; import { requirePermissions } from "../requirePermissions"; import { ensureApis } from "../dataconnect/ensureApis"; import { logLabeledSuccess } from "../utils"; +import { mainSchema } from "../dataconnect/types"; export const command = new Command("dataconnect:sql:migrate [serviceId]") .description("migrate your CloudSQL database's schema to match your local Data Connect schema") @@ -32,7 +33,7 @@ export const command = new Command("dataconnect:sql:migrate [serviceId]") } const diffs = await migrateSchema({ options, - schema: serviceInfo.schema, + schema: mainSchema(serviceInfo), validateOnly: true, schemaValidation: serviceInfo.dataConnectYaml.schema.datasource.postgresql?.schemaValidation, }); diff --git a/src/commands/dataconnect-sql-setup.ts b/src/commands/dataconnect-sql-setup.ts index 164a1d55d53..32cc7319953 100644 --- a/src/commands/dataconnect-sql-setup.ts +++ b/src/commands/dataconnect-sql-setup.ts @@ -10,6 +10,7 @@ import { setupSQLPermissions, getSchemaMetadata } from "../gcp/cloudsql/permissi import { DEFAULT_SCHEMA } from "../gcp/cloudsql/permissions"; import { getIdentifiers, ensureServiceIsConnectedToCloudSql } from "../dataconnect/schemaMigration"; import { setupIAMUsers } from "../gcp/cloudsql/connect"; +import { mainSchema } from "../dataconnect/types"; export const command = new Command("dataconnect:sql:setup [serviceId]") .description("set up your CloudSQL database") @@ -32,7 +33,7 @@ export const command = new Command("dataconnect:sql:setup [serviceId]") ); } - const { serviceName, instanceName, databaseId } = getIdentifiers(serviceInfo.schema); + const { serviceName, instanceName, databaseId } = getIdentifiers(mainSchema(serviceInfo)); await ensureServiceIsConnectedToCloudSql( serviceName, instanceName, diff --git a/src/commands/dataconnect-sql-shell.ts b/src/commands/dataconnect-sql-shell.ts index 6023fb44905..62329bb3b1e 100644 --- a/src/commands/dataconnect-sql-shell.ts +++ b/src/commands/dataconnect-sql-shell.ts @@ -17,6 +17,7 @@ import { logger } from "../logger"; import { FirebaseError } from "../error"; import { FBToolsAuthClient } from "../gcp/cloudsql/fbToolsAuthClient"; import { confirmDangerousQuery, interactiveExecuteQuery } from "../gcp/cloudsql/interactive"; +import { mainSchema } from "../dataconnect/types"; // Not a comprehensive list, used for keyword coloring. const sqlKeywords = [ @@ -91,7 +92,7 @@ export const command = new Command("dataconnect:sql:shell [serviceId]") const projectId = needProjectId(options); await ensureApis(projectId); const serviceInfo = await pickService(projectId, options.config, serviceId); - const { instanceId, databaseId } = getIdentifiers(serviceInfo.schema); + const { instanceId, databaseId } = getIdentifiers(mainSchema(serviceInfo)); const { user: username } = await getIAMUser(options); const instance = await cloudSqlAdminClient.getInstance(projectId, instanceId); diff --git a/src/mcp/prompts/dataconnect/schema.ts b/src/mcp/prompts/dataconnect/schema.ts index fd7c0491f3a..416a3c5c1f5 100644 --- a/src/mcp/prompts/dataconnect/schema.ts +++ b/src/mcp/prompts/dataconnect/schema.ts @@ -1,6 +1,6 @@ import { prompt } from "../../prompt"; import { loadAll } from "../../../dataconnect/load"; -import type { ServiceInfo } from "../../../dataconnect/types"; +import { mainSchema, type ServiceInfo } from "../../../dataconnect/types"; import { BUILTIN_SDL, MAIN_INSTRUCTIONS } from "../../util/dataconnect/content"; import { compileErrors } from "../../util/dataconnect/compile"; @@ -11,7 +11,7 @@ function renderServices(fdcServices: ServiceInfo[]) { The following is the up-to-date content of existing schema files (their paths are relative to the Data Connect source directory). -${fdcServices[0].schema.source.files?.map((f) => `\`\`\`graphql ${f.path}\n${f.content}\n\`\`\``).join("\n\n")}`; +${mainSchema(fdcServices[0]).source.files?.map((f) => `\`\`\`graphql ${f.path}\n${f.content}\n\`\`\``).join("\n\n")}`; } function renderErrors(errors?: string) { From 82685adc1182839a0db80ba71f1c1a388b5d3acd Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Mon, 10 Nov 2025 10:56:50 -0800 Subject: [PATCH 04/15] lint --- src/mcp/prompts/dataconnect/schema.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mcp/prompts/dataconnect/schema.ts b/src/mcp/prompts/dataconnect/schema.ts index 416a3c5c1f5..19cb7ad382b 100644 --- a/src/mcp/prompts/dataconnect/schema.ts +++ b/src/mcp/prompts/dataconnect/schema.ts @@ -11,7 +11,9 @@ function renderServices(fdcServices: ServiceInfo[]) { The following is the up-to-date content of existing schema files (their paths are relative to the Data Connect source directory). -${mainSchema(fdcServices[0]).source.files?.map((f) => `\`\`\`graphql ${f.path}\n${f.content}\n\`\`\``).join("\n\n")}`; +${mainSchema(fdcServices[0]) + .source.files?.map((f) => `\`\`\`graphql ${f.path}\n${f.content}\n\`\`\``) + .join("\n\n")}`; } function renderErrors(errors?: string) { From 16f7fae8169f302ac9074a402e9b7e6138fdfece Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Mon, 10 Nov 2025 16:19:45 -0800 Subject: [PATCH 05/15] Support loading schema sources from either `schema` or `schemas` field. --- src/commands/dataconnect-sql-diff.ts | 4 +- src/commands/dataconnect-sql-migrate.ts | 9 +++-- src/commands/dataconnect-sql-setup.ts | 6 +-- src/dataconnect/load.ts | 33 ++++++++++------ src/dataconnect/types.ts | 44 ++++++++++++++++------ src/emulator/dataconnectEmulator.ts | 5 ++- src/mcp/tools/dataconnect/list_services.ts | 13 ++++++- 7 files changed, 79 insertions(+), 35 deletions(-) diff --git a/src/commands/dataconnect-sql-diff.ts b/src/commands/dataconnect-sql-diff.ts index 66b7b000342..cac1a5ffae2 100644 --- a/src/commands/dataconnect-sql-diff.ts +++ b/src/commands/dataconnect-sql-diff.ts @@ -6,7 +6,7 @@ import { requirePermissions } from "../requirePermissions"; import { pickService } from "../dataconnect/load"; import { diffSchema } from "../dataconnect/schemaMigration"; import { requireAuth } from "../requireAuth"; -import { mainSchema } from "../dataconnect/types"; +import { mainSchema, mainSchemaYaml } from "../dataconnect/types"; export const command = new Command("dataconnect:sql:diff [serviceId]") .description( @@ -26,7 +26,7 @@ export const command = new Command("dataconnect:sql:diff [serviceId]") const diffs = await diffSchema( options, mainSchema(serviceInfo), - serviceInfo.dataConnectYaml.schema.datasource.postgresql?.schemaValidation, + mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.schemaValidation, ); return { projectId, serviceId, diffs }; }); diff --git a/src/commands/dataconnect-sql-migrate.ts b/src/commands/dataconnect-sql-migrate.ts index 1797d3a05fd..5f406a16a3d 100644 --- a/src/commands/dataconnect-sql-migrate.ts +++ b/src/commands/dataconnect-sql-migrate.ts @@ -8,7 +8,7 @@ import { requireAuth } from "../requireAuth"; import { requirePermissions } from "../requirePermissions"; import { ensureApis } from "../dataconnect/ensureApis"; import { logLabeledSuccess } from "../utils"; -import { mainSchema } from "../dataconnect/types"; +import { mainSchema, mainSchemaYaml } from "../dataconnect/types"; export const command = new Command("dataconnect:sql:migrate [serviceId]") .description("migrate your CloudSQL database's schema to match your local Data Connect schema") @@ -24,8 +24,8 @@ export const command = new Command("dataconnect:sql:migrate [serviceId]") const projectId = needProjectId(options); await ensureApis(projectId); const serviceInfo = await pickService(projectId, options.config, serviceId); - const instanceId = - serviceInfo.dataConnectYaml.schema.datasource.postgresql?.cloudSql.instanceId; + const instanceId = mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.cloudSql + .instanceId; if (!instanceId) { throw new FirebaseError( "dataconnect.yaml is missing field schema.datasource.postgresql.cloudsql.instanceId", @@ -35,7 +35,8 @@ export const command = new Command("dataconnect:sql:migrate [serviceId]") options, schema: mainSchema(serviceInfo), validateOnly: true, - schemaValidation: serviceInfo.dataConnectYaml.schema.datasource.postgresql?.schemaValidation, + schemaValidation: mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql + ?.schemaValidation, }); if (diffs.length) { logLabeledSuccess( diff --git a/src/commands/dataconnect-sql-setup.ts b/src/commands/dataconnect-sql-setup.ts index 32cc7319953..38c07fd9798 100644 --- a/src/commands/dataconnect-sql-setup.ts +++ b/src/commands/dataconnect-sql-setup.ts @@ -10,7 +10,7 @@ import { setupSQLPermissions, getSchemaMetadata } from "../gcp/cloudsql/permissi import { DEFAULT_SCHEMA } from "../gcp/cloudsql/permissions"; import { getIdentifiers, ensureServiceIsConnectedToCloudSql } from "../dataconnect/schemaMigration"; import { setupIAMUsers } from "../gcp/cloudsql/connect"; -import { mainSchema } from "../dataconnect/types"; +import { mainSchema, mainSchemaYaml } from "../dataconnect/types"; export const command = new Command("dataconnect:sql:setup [serviceId]") .description("set up your CloudSQL database") @@ -25,8 +25,8 @@ export const command = new Command("dataconnect:sql:setup [serviceId]") const projectId = needProjectId(options); await ensureApis(projectId); const serviceInfo = await pickService(projectId, options.config, serviceId); - const instanceId = - serviceInfo.dataConnectYaml.schema.datasource.postgresql?.cloudSql.instanceId; + const instanceId = mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.cloudSql + .instanceId; if (!instanceId) { throw new FirebaseError( "dataconnect.yaml is missing field schema.datasource.postgresql.cloudsql.instanceId", diff --git a/src/dataconnect/load.ts b/src/dataconnect/load.ts index f53480a27c1..aedb487b073 100644 --- a/src/dataconnect/load.ts +++ b/src/dataconnect/load.ts @@ -16,6 +16,7 @@ import { } from "./types"; import { readFileFromDirectory, wrappedSafeLoad } from "../utils"; import { DataConnectMultiple } from "../firebaseConfig"; +import * as experiments from "../experiments"; // pickService reads firebase.json and returns all services with a given serviceId. // If serviceID is not provided and there is a single service, return that. @@ -77,8 +78,20 @@ export async function load( const resolvedDir = config.path(sourceDirectory); const dataConnectYaml = await readDataConnectYaml(resolvedDir); const serviceName = `projects/${projectId}/locations/${dataConnectYaml.location}/services/${dataConnectYaml.serviceId}`; - const schemaDir = path.join(resolvedDir, dataConnectYaml.schema.source); - const schemaGQLs = await readGQLFiles(schemaDir); + const schemaYamls = dataConnectYaml.schema ? [dataConnectYaml.schema] : dataConnectYaml.schemas; + const schemas = await Promise.all( + schemaYamls!.map(async (yaml) => { + const schemaDir = path.join(resolvedDir, yaml.source); + const schemaGQLs = await readGQLFiles(schemaDir); + return { + name: `${serviceName}/schemas/${yaml.id || MAIN_SCHEMA_ID}`, + datasources: [toDatasource(projectId, dataConnectYaml.location, yaml.datasource)], + source: { + files: schemaGQLs, + }, + }; + }), + ); const connectorInfo = await Promise.all( dataConnectYaml.connectorDirs.map(async (dir) => { const connectorDir = path.join(resolvedDir, dir); @@ -100,15 +113,7 @@ export async function load( return { serviceName, sourceDirectory: resolvedDir, - schema: { - name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`, - datasources: [ - toDatasource(projectId, dataConnectYaml.location, dataConnectYaml.schema.datasource), - ], - source: { - files: schemaGQLs, - }, - }, + schemas: schemas, dataConnectYaml, connectorInfo, }; @@ -149,6 +154,12 @@ function validateDataConnectYaml(unvalidated: any): DataConnectYaml { if (!unvalidated["location"]) { throw new FirebaseError("Missing required field 'location' in dataconnect.yaml"); } + if (!experiments.isEnabled("fdcwebhooks") && unvalidated["schemas"]) { + throw new FirebaseError("Unsupported field 'schemas' in dataconnect.yaml"); + } + if (!unvalidated["schema"] && !unvalidated["schemas"]) { + throw new FirebaseError("Either 'schema' or 'schemas' is required in dataconnect.yaml"); + } return unvalidated as DataConnectYaml; } diff --git a/src/dataconnect/types.ts b/src/dataconnect/types.ts index 6274f2f5e93..d2473d0c38f 100644 --- a/src/dataconnect/types.ts +++ b/src/dataconnect/types.ts @@ -15,7 +15,6 @@ export interface Service extends BaseResource { export interface Schema extends BaseResource { name: string; - id?: string; datasources: Datasource[]; source: Source; @@ -124,13 +123,16 @@ export function requiresVector(dm?: DeploymentMetadata): boolean { export interface DataConnectYaml { specVersion?: string; serviceId: string; - schema: SchemaYaml; + // One of `schema` or `schemas` is required. + schema?: SchemaYaml; + schemas?: SchemaYaml[]; location: string; connectorDirs: string[]; } export interface SchemaYaml { source: string; + id?: string; datasource: DatasourceYaml; } @@ -142,6 +144,10 @@ export interface DatasourceYaml { }; schemaValidation?: SchemaValidation; }; + httpGraphql?: { + uri: string; + timeout?: string; + }; } export interface ConnectorYaml { @@ -191,9 +197,7 @@ export interface DartSDK { export interface ServiceInfo { serviceName: string; sourceDirectory: string; - // One of `schema` or `schemas` is required. - schema?: Schema; - schemas?: Schema[]; + schemas: Schema[]; connectorInfo: ConnectorInfo[]; dataConnectYaml: DataConnectYaml; deploymentMetadata?: DeploymentMetadata; @@ -221,18 +225,36 @@ export function toDatasource( }, }; } + if (ds?.httpGraphql) { + return { + httpGraphql: { + uri: ds.httpGraphql.uri, + timeout: ds.httpGraphql.timeout, + }, + }; + } return {}; } /** Returns the main schema for a service info */ -export function mainSchema(serviceInfo: ServiceInfo): Schema { - if (serviceInfo.schema) { - return serviceInfo.schema; +export function mainSchemaYaml(dataconnectYaml: DataConnectYaml): SchemaYaml { + if (dataconnectYaml.schema) { + return dataconnectYaml.schema; } - if (serviceInfo.schemas && serviceInfo.schemas.length > 0) { - return serviceInfo.schemas.find((s) => s.name.endsWith(`/schemas/${MAIN_SCHEMA_ID}`))!; + const mainSch = dataconnectYaml.schemas?.find((s) => s.id === MAIN_SCHEMA_ID || !s.id); + if (!mainSch) { + throw new Error(`Service ${dataconnectYaml.serviceId} has no main schema defined`); + } + return mainSch; +} + +/** Returns the main schema for a service info */ +export function mainSchema(serviceInfo: ServiceInfo): Schema { + const mainSch = serviceInfo.schemas.find((s) => s.name.endsWith(`/schemas/${MAIN_SCHEMA_ID}`)); + if (!mainSch) { + throw new Error(`Service ${serviceInfo.serviceName} has no main schema defined`); } - throw new Error(`Service ${serviceInfo.serviceName} has no schema defined`); + return mainSch; } /** Start Dataplane Client Types */ diff --git a/src/emulator/dataconnectEmulator.ts b/src/emulator/dataconnectEmulator.ts index 554c8405ce5..d950e5a2696 100644 --- a/src/emulator/dataconnectEmulator.ts +++ b/src/emulator/dataconnectEmulator.ts @@ -17,7 +17,7 @@ import { import { EmulatorInfo, EmulatorInstance, Emulators, ListenSpec } from "./types"; import { FirebaseError } from "../error"; import { EmulatorLogger } from "./emulatorLogger"; -import { BuildResult, requiresVector } from "../dataconnect/types"; +import { BuildResult, mainSchemaYaml, requiresVector } from "../dataconnect/types"; import { listenSpecsToString } from "./portUtils"; import { Client, ClientResponse } from "../apiv2"; import { EmulatorRegistry } from "./registry"; @@ -114,7 +114,8 @@ export class DataConnectEmulator implements EmulatorInstance { this.usingExistingEmulator = false; if (this.args.autoconnectToPostgres) { const info = await load(this.args.projectId, this.args.config, this.args.configDir); - const dbId = info.dataConnectYaml.schema.datasource.postgresql?.database || "postgres"; + const dbId = + mainSchemaYaml(info.dataConnectYaml).datasource.postgresql?.database || "postgres"; const serviceId = info.dataConnectYaml.serviceId; const pgPort = this.args.postgresListen?.[0].port; const pgHost = this.args.postgresListen?.[0].address; diff --git a/src/mcp/tools/dataconnect/list_services.ts b/src/mcp/tools/dataconnect/list_services.ts index eecba90ed32..ce996a03d9d 100644 --- a/src/mcp/tools/dataconnect/list_services.ts +++ b/src/mcp/tools/dataconnect/list_services.ts @@ -4,7 +4,13 @@ import { tool } from "../../tool"; import { toContent } from "../../util"; import * as client from "../../../dataconnect/client"; import { loadAll } from "../../../dataconnect/load"; -import { Service, Schema, ServiceInfo, Connector } from "../../../dataconnect/types"; +import { + Service, + Schema, + ServiceInfo, + Connector, + mainSchemaYaml, +} from "../../../dataconnect/types"; import { dump } from "js-yaml"; import { logger } from "../../../logger"; @@ -108,7 +114,10 @@ export const list_services = tool( for (const s of localServices) { const local = s.local!; output.push(dump(local.dataConnectYaml)); - const schemaDir = path.join(local.sourceDirectory, local.dataConnectYaml.schema.source); + const schemaDir = path.join( + local.sourceDirectory, + mainSchemaYaml(local.dataConnectYaml).source, + ); output.push(`You can find all of schema sources under ${schemaDir}/`); if (s.deployed) { output.push(`It's already deployed in the backend:\n`); From 5ebfe69b2d2c0d1f3f932a907b926ee138f33cfd Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Tue, 11 Nov 2025 12:02:16 -0800 Subject: [PATCH 06/15] Fix firebase init --- src/commands/dataconnect-sql-diff.ts | 2 +- src/commands/dataconnect-sql-grant.ts | 2 +- src/commands/dataconnect-sql-migrate.ts | 2 +- src/commands/dataconnect-sql-setup.ts | 4 ++- src/commands/dataconnect-sql-shell.ts | 2 +- src/dataconnect/types.ts | 10 +++--- src/deploy/dataconnect/deploy.ts | 2 +- src/deploy/dataconnect/prepare.ts | 4 +-- src/deploy/dataconnect/release.ts | 6 ++-- src/init/features/dataconnect/index.ts | 31 ++++++++++--------- src/mcp/prompts/dataconnect/schema.ts | 2 +- .../dataconnect/dataconnect-fdcwebhooks.yaml | 13 -------- templates/init/dataconnect/dataconnect.yaml | 18 +++++------ 13 files changed, 45 insertions(+), 53 deletions(-) delete mode 100644 templates/init/dataconnect/dataconnect-fdcwebhooks.yaml diff --git a/src/commands/dataconnect-sql-diff.ts b/src/commands/dataconnect-sql-diff.ts index cac1a5ffae2..f928aadfbdb 100644 --- a/src/commands/dataconnect-sql-diff.ts +++ b/src/commands/dataconnect-sql-diff.ts @@ -25,7 +25,7 @@ export const command = new Command("dataconnect:sql:diff [serviceId]") const diffs = await diffSchema( options, - mainSchema(serviceInfo), + mainSchema(serviceInfo.schemas), mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.schemaValidation, ); return { projectId, serviceId, diffs }; diff --git a/src/commands/dataconnect-sql-grant.ts b/src/commands/dataconnect-sql-grant.ts index f5ae96253a6..0dbaf575787 100644 --- a/src/commands/dataconnect-sql-grant.ts +++ b/src/commands/dataconnect-sql-grant.ts @@ -52,6 +52,6 @@ export const command = new Command("dataconnect:sql:grant [serviceId]") await ensureApis(projectId); const serviceInfo = await pickService(projectId, options.config, serviceId); - await grantRoleToUserInSchema(options, mainSchema(serviceInfo)); + await grantRoleToUserInSchema(options, mainSchema(serviceInfo.schemas)); return { projectId, serviceId }; }); diff --git a/src/commands/dataconnect-sql-migrate.ts b/src/commands/dataconnect-sql-migrate.ts index 5f406a16a3d..cb246527429 100644 --- a/src/commands/dataconnect-sql-migrate.ts +++ b/src/commands/dataconnect-sql-migrate.ts @@ -33,7 +33,7 @@ export const command = new Command("dataconnect:sql:migrate [serviceId]") } const diffs = await migrateSchema({ options, - schema: mainSchema(serviceInfo), + schema: mainSchema(serviceInfo.schemas), validateOnly: true, schemaValidation: mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql ?.schemaValidation, diff --git a/src/commands/dataconnect-sql-setup.ts b/src/commands/dataconnect-sql-setup.ts index 38c07fd9798..cace511cf5d 100644 --- a/src/commands/dataconnect-sql-setup.ts +++ b/src/commands/dataconnect-sql-setup.ts @@ -33,7 +33,9 @@ export const command = new Command("dataconnect:sql:setup [serviceId]") ); } - const { serviceName, instanceName, databaseId } = getIdentifiers(mainSchema(serviceInfo)); + const { serviceName, instanceName, databaseId } = getIdentifiers( + mainSchema(serviceInfo.schemas), + ); await ensureServiceIsConnectedToCloudSql( serviceName, instanceName, diff --git a/src/commands/dataconnect-sql-shell.ts b/src/commands/dataconnect-sql-shell.ts index 62329bb3b1e..49436aac7f2 100644 --- a/src/commands/dataconnect-sql-shell.ts +++ b/src/commands/dataconnect-sql-shell.ts @@ -92,7 +92,7 @@ export const command = new Command("dataconnect:sql:shell [serviceId]") const projectId = needProjectId(options); await ensureApis(projectId); const serviceInfo = await pickService(projectId, options.config, serviceId); - const { instanceId, databaseId } = getIdentifiers(mainSchema(serviceInfo)); + const { instanceId, databaseId } = getIdentifiers(mainSchema(serviceInfo.schemas)); const { user: username } = await getIAMUser(options); const instance = await cloudSqlAdminClient.getInstance(projectId, instanceId); diff --git a/src/dataconnect/types.ts b/src/dataconnect/types.ts index d2473d0c38f..52c2f5c85f4 100644 --- a/src/dataconnect/types.ts +++ b/src/dataconnect/types.ts @@ -236,7 +236,7 @@ export function toDatasource( return {}; } -/** Returns the main schema for a service info */ +/** Returns the main schema YAML for a Data Connect YAML */ export function mainSchemaYaml(dataconnectYaml: DataConnectYaml): SchemaYaml { if (dataconnectYaml.schema) { return dataconnectYaml.schema; @@ -248,11 +248,11 @@ export function mainSchemaYaml(dataconnectYaml: DataConnectYaml): SchemaYaml { return mainSch; } -/** Returns the main schema for a service info */ -export function mainSchema(serviceInfo: ServiceInfo): Schema { - const mainSch = serviceInfo.schemas.find((s) => s.name.endsWith(`/schemas/${MAIN_SCHEMA_ID}`)); +/** Returns the main schema from a list of schemas */ +export function mainSchema(schemas: Schema[]): Schema { + const mainSch = schemas.find((s) => s.name.endsWith(`/schemas/${MAIN_SCHEMA_ID}`)); if (!mainSch) { - throw new Error(`Service ${serviceInfo.serviceName} has no main schema defined`); + throw new Error(`No main schema is defined`); } return mainSch; } diff --git a/src/deploy/dataconnect/deploy.ts b/src/deploy/dataconnect/deploy.ts index 6822e4b1357..4eab22142e2 100644 --- a/src/deploy/dataconnect/deploy.ts +++ b/src/deploy/dataconnect/deploy.ts @@ -88,7 +88,7 @@ export default async function (context: Context, options: Options): Promise { - const postgresDatasource = mainSchema(s).datasources.find((d) => d.postgresql); + const postgresDatasource = mainSchema(s.schemas).datasources.find((d) => d.postgresql); if (postgresDatasource) { const instanceId = postgresDatasource.postgresql?.cloudSql?.instance.split("/").pop(); const databaseId = postgresDatasource.postgresql?.database; diff --git a/src/deploy/dataconnect/prepare.ts b/src/deploy/dataconnect/prepare.ts index 94f02733c83..c4c8aa74cfa 100644 --- a/src/deploy/dataconnect/prepare.ts +++ b/src/deploy/dataconnect/prepare.ts @@ -65,7 +65,7 @@ export default async function (context: Context, options: DeployOptions): Promis for (const si of serviceInfos) { await diffSchema( options, - mainSchema(si), + mainSchema(si.schemas), si.dataConnectYaml.schema?.datasource?.postgresql?.schemaValidation, ); } @@ -76,7 +76,7 @@ export default async function (context: Context, options: DeployOptions): Promis return !filters || filters?.some((f) => si.dataConnectYaml.serviceId === f.serviceId); }) .map(async (s) => { - const postgresDatasource = mainSchema(s).datasources.find((d) => d.postgresql); + const postgresDatasource = mainSchema(s.schemas).datasources.find((d) => d.postgresql); if (postgresDatasource) { const instanceId = postgresDatasource.postgresql?.cloudSql?.instance.split("/").pop(); const databaseId = postgresDatasource.postgresql?.database; diff --git a/src/deploy/dataconnect/release.ts b/src/deploy/dataconnect/release.ts index 314f078b82f..6f5612582a1 100644 --- a/src/deploy/dataconnect/release.ts +++ b/src/deploy/dataconnect/release.ts @@ -1,5 +1,5 @@ import * as utils from "../../utils"; -import { Connector, mainSchema, ServiceInfo } from "../../dataconnect/types"; +import { Connector, mainSchema, mainSchemaYaml, ServiceInfo } from "../../dataconnect/types"; import { listConnectors, upsertConnector } from "../../dataconnect/client"; import { promptDeleteConnector } from "../../dataconnect/prompts"; import { Options } from "../../options"; @@ -35,8 +35,8 @@ export default async function (context: Context, options: Options): Promise ({ - schema: mainSchema(s), - validationMode: s.dataConnectYaml?.schema?.datasource?.postgresql?.schemaValidation, + schema: mainSchema(s.schemas), + validationMode: mainSchemaYaml(s.dataConnectYaml).datasource?.postgresql?.schemaValidation, })); const wantConnectors = serviceInfos.flatMap((si) => si.connectorInfo diff --git a/src/init/features/dataconnect/index.ts b/src/init/features/dataconnect/index.ts index c8903c5be71..f911f5aa38c 100644 --- a/src/init/features/dataconnect/index.ts +++ b/src/init/features/dataconnect/index.ts @@ -9,16 +9,15 @@ import { setupCloudSql } from "../../../dataconnect/provisionCloudSql"; import { checkFreeTrialInstanceUsed, upgradeInstructions } from "../../../dataconnect/freeTrial"; import * as cloudsql from "../../../gcp/cloudsql/cloudsqladmin"; import { ensureApis, ensureGIFApiTos } from "../../../dataconnect/ensureApis"; -import * as experiments from "../../../experiments"; import { listLocations, listAllServices, - getSchema, + listSchemas, listConnectors, createService, upsertSchema, } from "../../../dataconnect/client"; -import { Schema, Service, File, MAIN_SCHEMA_ID } from "../../../dataconnect/types"; +import { Schema, Service, File, MAIN_SCHEMA_ID, mainSchema } from "../../../dataconnect/types"; import { parseCloudSQLInstanceName, parseServiceName } from "../../../dataconnect/names"; import { logger } from "../../../logger"; import { readTemplateSync } from "../../../templates"; @@ -45,9 +44,6 @@ import { trackGA4 } from "../../../track"; export const FDC_DEFAULT_REGION = "us-east4"; const DATACONNECT_YAML_TEMPLATE = readTemplateSync("init/dataconnect/dataconnect.yaml"); -const DATACONNECT_YAML_WEBHOOKS_EXPERIMENT_TEMPLATE = readTemplateSync( - "init/dataconnect/dataconnect-fdcwebhooks.yaml", -); const CONNECTOR_YAML_TEMPLATE = readTemplateSync("init/dataconnect/connector.yaml"); const SCHEMA_TEMPLATE = readTemplateSync("init/dataconnect/schema.gql"); const QUERIES_TEMPLATE = readTemplateSync("init/dataconnect/queries.gql"); @@ -482,9 +478,7 @@ function subDataconnectYamlValues(replacementValues: { cloudSqlInstanceId: "__cloudSqlInstanceId__", connectorDirs: "__connectorDirs__", }; - let replaced = experiments.isEnabled("fdcwebhooks") - ? DATACONNECT_YAML_WEBHOOKS_EXPERIMENT_TEMPLATE - : DATACONNECT_YAML_TEMPLATE; + let replaced = DATACONNECT_YAML_TEMPLATE; for (const [k, v] of Object.entries(replacementValues)) { replaced = replaced.replace(replacements[k], JSON.stringify(v)); } @@ -527,8 +521,15 @@ async function promptForExistingServices(setup: Setup, info: RequiredInfo): Prom } async function downloadService(info: RequiredInfo, serviceName: string): Promise { - const schema = await getSchema(serviceName); - if (!schema) { + let schemas: Schema[] = []; + try { + schemas = await listSchemas(serviceName); + } catch (err: any) { + if (err.status !== 404) { + throw err; + } + } + if (!schemas.length) { return; } info.serviceGql = { @@ -541,13 +542,15 @@ async function downloadService(info: RequiredInfo, serviceName: string): Promise }, ], }; - const primaryDatasource = schema.datasources.find((d) => d.postgresql); + const mainSch = mainSchema(schemas); + const primaryDatasource = mainSch.datasources.find((d) => d.postgresql); if (primaryDatasource?.postgresql?.cloudSql?.instance) { const instanceName = parseCloudSQLInstanceName(primaryDatasource.postgresql.cloudSql.instance); info.cloudSqlInstanceId = instanceName.instanceId; } - if (schema.source.files?.length) { - info.serviceGql.schemaGql = schema.source.files; + // TODO: Update dataconnect.yaml with downloaded secondary schemas as well. + if (mainSch.source.files?.length) { + info.serviceGql.schemaGql = mainSch.source.files; } info.cloudSqlDatabase = primaryDatasource?.postgresql?.database ?? ""; const connectors = await listConnectors(serviceName, [ diff --git a/src/mcp/prompts/dataconnect/schema.ts b/src/mcp/prompts/dataconnect/schema.ts index 19cb7ad382b..46921302358 100644 --- a/src/mcp/prompts/dataconnect/schema.ts +++ b/src/mcp/prompts/dataconnect/schema.ts @@ -11,7 +11,7 @@ function renderServices(fdcServices: ServiceInfo[]) { The following is the up-to-date content of existing schema files (their paths are relative to the Data Connect source directory). -${mainSchema(fdcServices[0]) +${mainSchema(fdcServices[0].schemas) .source.files?.map((f) => `\`\`\`graphql ${f.path}\n${f.content}\n\`\`\``) .join("\n\n")}`; } diff --git a/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml b/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml deleted file mode 100644 index 91c262c1831..00000000000 --- a/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml +++ /dev/null @@ -1,13 +0,0 @@ -specVersion: "v1" -serviceId: __serviceId__ -location: __location__ -schemas: - - source: "./schemas/main" - datasource: - postgresql: - database: __cloudSqlDatabase__ - cloudSql: - instanceId: __cloudSqlInstanceId__ - # schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly. - # schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect. -connectorDirs: __connectorDirs__ diff --git a/templates/init/dataconnect/dataconnect.yaml b/templates/init/dataconnect/dataconnect.yaml index 60a1f4009a6..ed5f6b1d5dc 100644 --- a/templates/init/dataconnect/dataconnect.yaml +++ b/templates/init/dataconnect/dataconnect.yaml @@ -1,13 +1,13 @@ specVersion: "v1" serviceId: __serviceId__ location: __location__ -schema: - source: "./schema" - datasource: - postgresql: - database: __cloudSqlDatabase__ - cloudSql: - instanceId: __cloudSqlInstanceId__ - # schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly. - # schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect. +schemas: + - source: "./schema" + datasource: + postgresql: + database: __cloudSqlDatabase__ + cloudSql: + instanceId: __cloudSqlInstanceId__ + # schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly. + # schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect. connectorDirs: __connectorDirs__ From 58f389982067c05eb5cf8707bfed30687285179a Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Tue, 11 Nov 2025 16:55:36 -0800 Subject: [PATCH 07/15] Properly deploy secondary schemas as well. --- src/dataconnect/types.ts | 5 +++ src/deploy/dataconnect/release.ts | 34 ++++++++++++++++--- src/init/features/dataconnect/index.ts | 8 ++++- .../dataconnect/dataconnect-fdcwebhooks.yaml | 13 +++++++ templates/init/dataconnect/dataconnect.yaml | 18 +++++----- 5 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 templates/init/dataconnect/dataconnect-fdcwebhooks.yaml diff --git a/src/dataconnect/types.ts b/src/dataconnect/types.ts index 52c2f5c85f4..c64493331c1 100644 --- a/src/dataconnect/types.ts +++ b/src/dataconnect/types.ts @@ -257,6 +257,11 @@ export function mainSchema(schemas: Schema[]): Schema { return mainSch; } +/** Returns true if the schema is the main schema */ +export function isMainSchema(schema: Schema): boolean { + return schema.name.endsWith(`/schemas/${MAIN_SCHEMA_ID}`); +} + /** Start Dataplane Client Types */ export interface ExecuteGraphqlRequest { query: string; diff --git a/src/deploy/dataconnect/release.ts b/src/deploy/dataconnect/release.ts index 6f5612582a1..3b9f2a05168 100644 --- a/src/deploy/dataconnect/release.ts +++ b/src/deploy/dataconnect/release.ts @@ -1,6 +1,12 @@ import * as utils from "../../utils"; -import { Connector, mainSchema, mainSchemaYaml, ServiceInfo } from "../../dataconnect/types"; -import { listConnectors, upsertConnector } from "../../dataconnect/client"; +import { + Connector, + isMainSchema, + mainSchema, + mainSchemaYaml, + ServiceInfo, +} from "../../dataconnect/types"; +import { listConnectors, upsertConnector, upsertSchema } from "../../dataconnect/client"; import { promptDeleteConnector } from "../../dataconnect/prompts"; import { Options } from "../../options"; import { migrateSchema } from "../../dataconnect/schemaMigration"; @@ -25,7 +31,7 @@ export default async function (context: Context, options: Options): Promise { return ( !filters || @@ -71,8 +77,26 @@ export default async function (context: Context, options: Options): Promise { + return ( + !filters || + filters.some((f) => { + return f.serviceId === si.dataConnectYaml.serviceId && (f.schemaOnly || f.fullService); + }) + ); + }) + .map((s) => s.schemas.filter((s) => !isMainSchema(s))) + .flatMap((s) => s); + for (const schema of wantSecondarySchemas) { + await upsertSchema(schema, false); + utils.logLabeledSuccess("dataconnect", `Migrated schema ${schema.name}`); + dataconnect.deployStats.numSchemaMigrated++; + } + + // Migrate main schemas. + for (const s of wantMainSchemas) { await migrateSchema({ options, schema: s.schema, diff --git a/src/init/features/dataconnect/index.ts b/src/init/features/dataconnect/index.ts index f911f5aa38c..8ba778a9c84 100644 --- a/src/init/features/dataconnect/index.ts +++ b/src/init/features/dataconnect/index.ts @@ -9,6 +9,7 @@ import { setupCloudSql } from "../../../dataconnect/provisionCloudSql"; import { checkFreeTrialInstanceUsed, upgradeInstructions } from "../../../dataconnect/freeTrial"; import * as cloudsql from "../../../gcp/cloudsql/cloudsqladmin"; import { ensureApis, ensureGIFApiTos } from "../../../dataconnect/ensureApis"; +import * as experiments from "../../../experiments"; import { listLocations, listAllServices, @@ -44,6 +45,9 @@ import { trackGA4 } from "../../../track"; export const FDC_DEFAULT_REGION = "us-east4"; const DATACONNECT_YAML_TEMPLATE = readTemplateSync("init/dataconnect/dataconnect.yaml"); +const DATACONNECT_WEBHOOKS_YAML_TEMPLATE = readTemplateSync( + "init/dataconnect/dataconnect-fdcwebhooks.yaml", +); const CONNECTOR_YAML_TEMPLATE = readTemplateSync("init/dataconnect/connector.yaml"); const SCHEMA_TEMPLATE = readTemplateSync("init/dataconnect/schema.gql"); const QUERIES_TEMPLATE = readTemplateSync("init/dataconnect/queries.gql"); @@ -478,7 +482,9 @@ function subDataconnectYamlValues(replacementValues: { cloudSqlInstanceId: "__cloudSqlInstanceId__", connectorDirs: "__connectorDirs__", }; - let replaced = DATACONNECT_YAML_TEMPLATE; + let replaced = experiments.isEnabled("fdcwebhooks") + ? DATACONNECT_WEBHOOKS_YAML_TEMPLATE + : DATACONNECT_YAML_TEMPLATE; for (const [k, v] of Object.entries(replacementValues)) { replaced = replaced.replace(replacements[k], JSON.stringify(v)); } diff --git a/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml b/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml new file mode 100644 index 00000000000..ed5f6b1d5dc --- /dev/null +++ b/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml @@ -0,0 +1,13 @@ +specVersion: "v1" +serviceId: __serviceId__ +location: __location__ +schemas: + - source: "./schema" + datasource: + postgresql: + database: __cloudSqlDatabase__ + cloudSql: + instanceId: __cloudSqlInstanceId__ + # schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly. + # schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect. +connectorDirs: __connectorDirs__ diff --git a/templates/init/dataconnect/dataconnect.yaml b/templates/init/dataconnect/dataconnect.yaml index ed5f6b1d5dc..60a1f4009a6 100644 --- a/templates/init/dataconnect/dataconnect.yaml +++ b/templates/init/dataconnect/dataconnect.yaml @@ -1,13 +1,13 @@ specVersion: "v1" serviceId: __serviceId__ location: __location__ -schemas: - - source: "./schema" - datasource: - postgresql: - database: __cloudSqlDatabase__ - cloudSql: - instanceId: __cloudSqlInstanceId__ - # schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly. - # schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect. +schema: + source: "./schema" + datasource: + postgresql: + database: __cloudSqlDatabase__ + cloudSql: + instanceId: __cloudSqlInstanceId__ + # schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly. + # schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect. connectorDirs: __connectorDirs__ From 50c99ac31475aa569821301d7e94f94c3da64404 Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Wed, 12 Nov 2025 15:52:09 -0800 Subject: [PATCH 08/15] Make `getSchema` take a schema ID, move secondary schema upsert after main schema upsert, and add some TODOs. --- src/commands/dataconnect-services-list.ts | 4 +++- src/dataconnect/client.spec.ts | 4 ++-- src/dataconnect/client.ts | 9 ++++---- src/dataconnect/schemaMigration.ts | 2 +- src/dataconnect/types.ts | 2 +- src/deploy/dataconnect/release.ts | 26 +++++++++++----------- src/mcp/prompts/dataconnect/schema.ts | 1 + src/mcp/tools/dataconnect/list_services.ts | 1 + 8 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/commands/dataconnect-services-list.ts b/src/commands/dataconnect-services-list.ts index 8a55ad9554b..1c0fd4ff164 100644 --- a/src/commands/dataconnect-services-list.ts +++ b/src/commands/dataconnect-services-list.ts @@ -7,7 +7,9 @@ import { logger } from "../logger"; import { requirePermissions } from "../requirePermissions"; import { ensureApis } from "../dataconnect/ensureApis"; import * as Table from "cli-table3"; +import { MAIN_SCHEMA_ID } from "../dataconnect/types"; +// TODO: Update this command to also list secondary schema information. export const command = new Command("dataconnect:services:list") .description("list all deployed Data Connect services") .before(requirePermissions, [ @@ -32,7 +34,7 @@ export const command = new Command("dataconnect:services:list") }); const jsonOutput: { services: Record[] } = { services: [] }; for (const service of services) { - const schema = (await client.getSchema(service.name)) ?? { + const schema = (await client.getSchema(service.name, MAIN_SCHEMA_ID)) ?? { name: "", datasources: [{}], source: { files: [] }, diff --git a/src/dataconnect/client.spec.ts b/src/dataconnect/client.spec.ts index c6ac6d458e7..5fd72f94bb4 100644 --- a/src/dataconnect/client.spec.ts +++ b/src/dataconnect/client.spec.ts @@ -119,14 +119,14 @@ describe("client", () => { describe("Schema methods", () => { it("getSchema", async () => { getStub.resolves({ body: { name: "schema" } }); - const schema = await client.getSchema("projects/p/locations/l/services/s"); + const schema = await client.getSchema("projects/p/locations/l/services/s", "main"); expect(schema).to.deep.equal({ name: "schema" }); expect(getStub).to.be.calledWith("projects/p/locations/l/services/s/schemas/main"); }); it("getSchema returns undefined if not found", async () => { getStub.rejects(new FirebaseError("err", { status: 404 })); - const schema = await client.getSchema("projects/p/locations/l/services/s"); + const schema = await client.getSchema("projects/p/locations/l/services/s", "main"); expect(schema).to.be.undefined; }); diff --git a/src/dataconnect/client.ts b/src/dataconnect/client.ts index 2cca3813e54..034cd6faddd 100644 --- a/src/dataconnect/client.ts +++ b/src/dataconnect/client.ts @@ -83,11 +83,12 @@ export async function deleteService(serviceName: string): Promise /** Schema methods */ -export async function getSchema(serviceName: string): Promise { +export async function getSchema( + serviceName: string, + schemaId: string, +): Promise { try { - const res = await dataconnectClient().get( - `${serviceName}/schemas/${types.MAIN_SCHEMA_ID}`, - ); + const res = await dataconnectClient().get(`${serviceName}/schemas/${schemaId}`); return res.body; } catch (err: any) { if (err.status !== 404) { diff --git a/src/dataconnect/schemaMigration.ts b/src/dataconnect/schemaMigration.ts index fb9e5c24e8d..f857ed49dd3 100644 --- a/src/dataconnect/schemaMigration.ts +++ b/src/dataconnect/schemaMigration.ts @@ -599,7 +599,7 @@ export async function ensureServiceIsConnectedToCloudSql( databaseId: string, linkIfNotConnected: boolean, ): Promise { - let currentSchema = await getSchema(serviceName); + let currentSchema = await getSchema(serviceName, MAIN_SCHEMA_ID); let postgresql = currentSchema?.datasources?.find((d) => d.postgresql)?.postgresql; if ( currentSchema?.reconciling && // active LRO diff --git a/src/dataconnect/types.ts b/src/dataconnect/types.ts index c64493331c1..3ffdae4b0ea 100644 --- a/src/dataconnect/types.ts +++ b/src/dataconnect/types.ts @@ -250,7 +250,7 @@ export function mainSchemaYaml(dataconnectYaml: DataConnectYaml): SchemaYaml { /** Returns the main schema from a list of schemas */ export function mainSchema(schemas: Schema[]): Schema { - const mainSch = schemas.find((s) => s.name.endsWith(`/schemas/${MAIN_SCHEMA_ID}`)); + const mainSch = schemas.find((s) => isMainSchema(s)); if (!mainSch) { throw new Error(`No main schema is defined`); } diff --git a/src/deploy/dataconnect/release.ts b/src/deploy/dataconnect/release.ts index 3b9f2a05168..a3bec092127 100644 --- a/src/deploy/dataconnect/release.ts +++ b/src/deploy/dataconnect/release.ts @@ -77,6 +77,19 @@ export default async function (context: Context, options: Options): Promise { @@ -95,19 +108,6 @@ export default async function (context: Context, options: Options): Promise { diff --git a/src/mcp/prompts/dataconnect/schema.ts b/src/mcp/prompts/dataconnect/schema.ts index 46921302358..b12630eda85 100644 --- a/src/mcp/prompts/dataconnect/schema.ts +++ b/src/mcp/prompts/dataconnect/schema.ts @@ -7,6 +7,7 @@ import { compileErrors } from "../../util/dataconnect/compile"; function renderServices(fdcServices: ServiceInfo[]) { if (!fdcServices.length) return "Data Connect Status: "; + // TODO: Render secondary schemas as well. return `\n\n## Data Connect Schema The following is the up-to-date content of existing schema files (their paths are relative to the Data Connect source directory). diff --git a/src/mcp/tools/dataconnect/list_services.ts b/src/mcp/tools/dataconnect/list_services.ts index ce996a03d9d..f2ac4af684a 100644 --- a/src/mcp/tools/dataconnect/list_services.ts +++ b/src/mcp/tools/dataconnect/list_services.ts @@ -114,6 +114,7 @@ export const list_services = tool( for (const s of localServices) { const local = s.local!; output.push(dump(local.dataConnectYaml)); + // TODO: Include secondary schema sources here as well. const schemaDir = path.join( local.sourceDirectory, mainSchemaYaml(local.dataConnectYaml).source, From ee2401bb30ded278f395f1c2b74a7876850c6002 Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Wed, 12 Nov 2025 16:28:48 -0800 Subject: [PATCH 09/15] Fix listSchemas in `firebase init`. --- src/init/features/dataconnect/index.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/init/features/dataconnect/index.ts b/src/init/features/dataconnect/index.ts index 8ba778a9c84..683fe699bf6 100644 --- a/src/init/features/dataconnect/index.ts +++ b/src/init/features/dataconnect/index.ts @@ -529,7 +529,12 @@ async function promptForExistingServices(setup: Setup, info: RequiredInfo): Prom async function downloadService(info: RequiredInfo, serviceName: string): Promise { let schemas: Schema[] = []; try { - schemas = await listSchemas(serviceName); + schemas = await listSchemas(serviceName, [ + "schemas.name", + "schemas.datasources.postgresql.database", + "schemas.datasources.postgresql.cloudSql.instance", + "schemas.source.files", + ]); } catch (err: any) { if (err.status !== 404) { throw err; From 8f4a5be0d07dfa0d2b0f90c5dc64ff0c173931a0 Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Thu, 13 Nov 2025 16:19:27 -0800 Subject: [PATCH 10/15] Fix unit tests. --- src/deploy/dataconnect/deploy.spec.ts | 6 +++++- src/deploy/dataconnect/prepare.spec.ts | 1 + src/deploy/dataconnect/release.spec.ts | 8 ++++---- src/init/features/dataconnect/index.spec.ts | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/deploy/dataconnect/deploy.spec.ts b/src/deploy/dataconnect/deploy.spec.ts index 4e81d939cb3..fec93aa542a 100644 --- a/src/deploy/dataconnect/deploy.spec.ts +++ b/src/deploy/dataconnect/deploy.spec.ts @@ -44,7 +44,10 @@ describe("dataconnect deploy", () => { { serviceName: "projects/test-project/locations/l/services/s1", deploymentMetadata: {}, - schema: { datasources: [] }, + schema: { + name: "projects/test-project/locations/l/services/s1/schemas/main", + datasources: [], + }, dataConnectYaml: { serviceId: "s1" }, }, ]; @@ -104,6 +107,7 @@ describe("dataconnect deploy", () => { { serviceName: "projects/test-project/locations/l/services/s1", schema: { + name: "projects/test-project/locations/l/services/s1/schemas/main", datasources: [ { postgresql: { diff --git a/src/deploy/dataconnect/prepare.spec.ts b/src/deploy/dataconnect/prepare.spec.ts index f6b3254ffb0..b5fcea63f94 100644 --- a/src/deploy/dataconnect/prepare.spec.ts +++ b/src/deploy/dataconnect/prepare.spec.ts @@ -105,6 +105,7 @@ describe("dataconnect prepare", () => { const serviceInfos = [ { schema: { + name: "projects/p/locations/l/services/s/schemas/main", datasources: [ { postgresql: { diff --git a/src/deploy/dataconnect/release.spec.ts b/src/deploy/dataconnect/release.spec.ts index 6582eb389f8..8105abac8b6 100644 --- a/src/deploy/dataconnect/release.spec.ts +++ b/src/deploy/dataconnect/release.spec.ts @@ -50,7 +50,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "my-schema" }, + schema: { name: "projects/p/locations/l/services/s1/schemas/main" }, connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, @@ -88,7 +88,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "my-schema" }, + schema: { name: "projects/p/locations/l/services/s1/schemas/main" }, connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, @@ -124,7 +124,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "my-schema" }, + schema: { name: "projects/p/locations/l/services/s1/schemas/main" }, connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, @@ -163,7 +163,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "my-schema" }, + schema: { name: "projects/p/locations/l/services/s1/schemas/main" }, connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, diff --git a/src/init/features/dataconnect/index.spec.ts b/src/init/features/dataconnect/index.spec.ts index f52d04bdd83..ae167df5d1b 100644 --- a/src/init/features/dataconnect/index.spec.ts +++ b/src/init/features/dataconnect/index.spec.ts @@ -31,7 +31,7 @@ describe("init dataconnect", () => { sdkActuateStub = sandbox.stub(sdk, "actuate").resolves(); sandbox.stub(cloudbilling, "isBillingEnabled").resolves(true); sandbox.stub(ensureApis, "ensureApis").resolves(); - sandbox.stub(client, "getSchema").resolves(undefined); + sandbox.stub(client, "listSchemas").resolves([]); }); afterEach(() => { From 852350a0250b5c8d3a315b780365c4ccdb3b8b1f Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Thu, 13 Nov 2025 16:40:02 -0800 Subject: [PATCH 11/15] Actually fix unit tests. --- src/deploy/dataconnect/deploy.spec.ts | 32 +++++++++++++++----------- src/deploy/dataconnect/prepare.spec.ts | 22 ++++++++++-------- src/deploy/dataconnect/release.spec.ts | 8 +++---- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/src/deploy/dataconnect/deploy.spec.ts b/src/deploy/dataconnect/deploy.spec.ts index fec93aa542a..a9faa0233b8 100644 --- a/src/deploy/dataconnect/deploy.spec.ts +++ b/src/deploy/dataconnect/deploy.spec.ts @@ -44,10 +44,12 @@ describe("dataconnect deploy", () => { { serviceName: "projects/test-project/locations/l/services/s1", deploymentMetadata: {}, - schema: { - name: "projects/test-project/locations/l/services/s1/schemas/main", - datasources: [], - }, + schemas: [ + { + name: "projects/test-project/locations/l/services/s1/schemas/main", + datasources: [], + }, + ], dataConnectYaml: { serviceId: "s1" }, }, ]; @@ -106,17 +108,19 @@ describe("dataconnect deploy", () => { const serviceInfos = [ { serviceName: "projects/test-project/locations/l/services/s1", - schema: { - name: "projects/test-project/locations/l/services/s1/schemas/main", - datasources: [ - { - postgresql: { - cloudSql: { instance: "projects/p/locations/l/instances/i" }, - database: "db", + schemas: [ + { + name: "projects/test-project/locations/l/services/s1/schemas/main", + datasources: [ + { + postgresql: { + cloudSql: { instance: "projects/p/locations/l/instances/i" }, + database: "db", + }, }, - }, - ], - }, + ], + }, + ], deploymentMetadata: {}, dataConnectYaml: { serviceId: "s1" }, }, diff --git a/src/deploy/dataconnect/prepare.spec.ts b/src/deploy/dataconnect/prepare.spec.ts index b5fcea63f94..d21cc35638e 100644 --- a/src/deploy/dataconnect/prepare.spec.ts +++ b/src/deploy/dataconnect/prepare.spec.ts @@ -104,17 +104,19 @@ describe("dataconnect prepare", () => { it("should diff schema and setup cloud sql", async () => { const serviceInfos = [ { - schema: { - name: "projects/p/locations/l/services/s/schemas/main", - datasources: [ - { - postgresql: { - cloudSql: { instance: "projects/p/locations/l/instances/i" }, - database: "db", + schemas: [ + { + name: "projects/p/locations/l/services/s/schemas/main", + datasources: [ + { + postgresql: { + cloudSql: { instance: "projects/p/locations/l/instances/i" }, + database: "db", + }, }, - }, - ], - }, + ], + }, + ], serviceName: "projects/p/locations/l/services/s", deploymentMetadata: {}, dataConnectYaml: { diff --git a/src/deploy/dataconnect/release.spec.ts b/src/deploy/dataconnect/release.spec.ts index 8105abac8b6..5ffcda87e76 100644 --- a/src/deploy/dataconnect/release.spec.ts +++ b/src/deploy/dataconnect/release.spec.ts @@ -50,7 +50,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "projects/p/locations/l/services/s1/schemas/main" }, + schemas: [{ name: "projects/p/locations/l/services/s1/schemas/main" }], connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, @@ -88,7 +88,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "projects/p/locations/l/services/s1/schemas/main" }, + schemas: [{ name: "projects/p/locations/l/services/s1/schemas/main" }], connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, @@ -124,7 +124,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "projects/p/locations/l/services/s1/schemas/main" }, + schemas: [{ name: "projects/p/locations/l/services/s1/schemas/main" }], connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, @@ -163,7 +163,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "projects/p/locations/l/services/s1/schemas/main" }, + schemas: [{ name: "projects/p/locations/l/services/s1/schemas/main" }], connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, From 110d9765ee8809b0cfce77f0528984517204f0e7 Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Mon, 17 Nov 2025 12:45:52 -0800 Subject: [PATCH 12/15] Set default parameter for getSchema schemaId. --- src/commands/dataconnect-services-list.ts | 3 +-- src/dataconnect/client.spec.ts | 11 +++++++++-- src/dataconnect/client.ts | 2 +- src/dataconnect/schemaMigration.ts | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/commands/dataconnect-services-list.ts b/src/commands/dataconnect-services-list.ts index 1c0fd4ff164..c1399a480d5 100644 --- a/src/commands/dataconnect-services-list.ts +++ b/src/commands/dataconnect-services-list.ts @@ -7,7 +7,6 @@ import { logger } from "../logger"; import { requirePermissions } from "../requirePermissions"; import { ensureApis } from "../dataconnect/ensureApis"; import * as Table from "cli-table3"; -import { MAIN_SCHEMA_ID } from "../dataconnect/types"; // TODO: Update this command to also list secondary schema information. export const command = new Command("dataconnect:services:list") @@ -34,7 +33,7 @@ export const command = new Command("dataconnect:services:list") }); const jsonOutput: { services: Record[] } = { services: [] }; for (const service of services) { - const schema = (await client.getSchema(service.name, MAIN_SCHEMA_ID)) ?? { + const schema = (await client.getSchema(service.name)) ?? { name: "", datasources: [{}], source: { files: [] }, diff --git a/src/dataconnect/client.spec.ts b/src/dataconnect/client.spec.ts index 5fd72f94bb4..00bf8066d22 100644 --- a/src/dataconnect/client.spec.ts +++ b/src/dataconnect/client.spec.ts @@ -119,14 +119,21 @@ describe("client", () => { describe("Schema methods", () => { it("getSchema", async () => { getStub.resolves({ body: { name: "schema" } }); - const schema = await client.getSchema("projects/p/locations/l/services/s", "main"); + const schema = await client.getSchema("projects/p/locations/l/services/s"); expect(schema).to.deep.equal({ name: "schema" }); expect(getStub).to.be.calledWith("projects/p/locations/l/services/s/schemas/main"); }); + it("getSchema with schemaId", async () => { + getStub.resolves({ body: { name: "schema" } }); + const schema = await client.getSchema("projects/p/locations/l/services/s", "schemaId"); + expect(schema).to.deep.equal({ name: "schema" }); + expect(getStub).to.be.calledWith("projects/p/locations/l/services/s/schemas/schemaId"); + }); + it("getSchema returns undefined if not found", async () => { getStub.rejects(new FirebaseError("err", { status: 404 })); - const schema = await client.getSchema("projects/p/locations/l/services/s", "main"); + const schema = await client.getSchema("projects/p/locations/l/services/s"); expect(schema).to.be.undefined; }); diff --git a/src/dataconnect/client.ts b/src/dataconnect/client.ts index 034cd6faddd..aeaaa28387d 100644 --- a/src/dataconnect/client.ts +++ b/src/dataconnect/client.ts @@ -85,7 +85,7 @@ export async function deleteService(serviceName: string): Promise export async function getSchema( serviceName: string, - schemaId: string, + schemaId: string = types.MAIN_SCHEMA_ID, ): Promise { try { const res = await dataconnectClient().get(`${serviceName}/schemas/${schemaId}`); diff --git a/src/dataconnect/schemaMigration.ts b/src/dataconnect/schemaMigration.ts index f857ed49dd3..fb9e5c24e8d 100644 --- a/src/dataconnect/schemaMigration.ts +++ b/src/dataconnect/schemaMigration.ts @@ -599,7 +599,7 @@ export async function ensureServiceIsConnectedToCloudSql( databaseId: string, linkIfNotConnected: boolean, ): Promise { - let currentSchema = await getSchema(serviceName, MAIN_SCHEMA_ID); + let currentSchema = await getSchema(serviceName); let postgresql = currentSchema?.datasources?.find((d) => d.postgresql)?.postgresql; if ( currentSchema?.reconciling && // active LRO From 740439ece41351a8f721a716e26c76670ea07cd7 Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Mon, 17 Nov 2025 12:59:47 -0800 Subject: [PATCH 13/15] Specify just top-level fields in listSchema call. --- src/init/features/dataconnect/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/init/features/dataconnect/index.ts b/src/init/features/dataconnect/index.ts index 683fe699bf6..2d54ac4c8f4 100644 --- a/src/init/features/dataconnect/index.ts +++ b/src/init/features/dataconnect/index.ts @@ -531,9 +531,8 @@ async function downloadService(info: RequiredInfo, serviceName: string): Promise try { schemas = await listSchemas(serviceName, [ "schemas.name", - "schemas.datasources.postgresql.database", - "schemas.datasources.postgresql.cloudSql.instance", - "schemas.source.files", + "schemas.datasources", + "schemas.source", ]); } catch (err: any) { if (err.status !== 404) { From 2fcebd8f9366182efd7b71352581ab644c90f1c8 Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Mon, 17 Nov 2025 13:19:52 -0800 Subject: [PATCH 14/15] Fix VSCode unit tests. --- firebase-vscode/src/data-connect/config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firebase-vscode/src/data-connect/config.ts b/firebase-vscode/src/data-connect/config.ts index e6849786508..5bf95958547 100644 --- a/firebase-vscode/src/data-connect/config.ts +++ b/firebase-vscode/src/data-connect/config.ts @@ -1,6 +1,6 @@ import { isPathInside } from "./file-utils"; import { DeepReadOnly } from "../metaprogramming"; -import { ConnectorYaml, DataConnectYaml } from "../dataconnect/types"; +import { ConnectorYaml, DataConnectYaml, mainSchemaYaml } from "../dataconnect/types"; import { Result, ResultValue } from "../result"; import { computed, effect, signal } from "@preact/signals-core"; import { @@ -265,7 +265,7 @@ export class ResolvedDataConnectConfig { } get schemaDir(): string { - return this.value.schema.source; + return mainSchemaYaml(this.value).source; } get relativePath(): string { From b45dcd257fe1e7e3b6e407771431f568d4484e3f Mon Sep 17 00:00:00 2001 From: Rosalyn Tan Date: Mon, 17 Nov 2025 14:43:59 -0800 Subject: [PATCH 15/15] Try to fix import path in VSCode. --- firebase-vscode/src/data-connect/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase-vscode/src/data-connect/config.ts b/firebase-vscode/src/data-connect/config.ts index 5bf95958547..f10356a04fa 100644 --- a/firebase-vscode/src/data-connect/config.ts +++ b/firebase-vscode/src/data-connect/config.ts @@ -1,6 +1,6 @@ import { isPathInside } from "./file-utils"; import { DeepReadOnly } from "../metaprogramming"; -import { ConnectorYaml, DataConnectYaml, mainSchemaYaml } from "../dataconnect/types"; +import { ConnectorYaml, DataConnectYaml, mainSchemaYaml } from "../../../src/dataconnect/types"; import { Result, ResultValue } from "../result"; import { computed, effect, signal } from "@preact/signals-core"; import {