diff --git a/firebase-vscode/src/data-connect/config.ts b/firebase-vscode/src/data-connect/config.ts index e6849786508..f10356a04fa 100644 --- a/firebase-vscode/src/data-connect/config.ts +++ b/firebase-vscode/src/data-connect/config.ts @@ -1,6 +1,6 @@ import { isPathInside } from "./file-utils"; import { DeepReadOnly } from "../metaprogramming"; -import { ConnectorYaml, DataConnectYaml } from "../dataconnect/types"; +import { ConnectorYaml, DataConnectYaml, mainSchemaYaml } from "../../../src/dataconnect/types"; import { Result, ResultValue } from "../result"; import { computed, effect, signal } from "@preact/signals-core"; import { @@ -265,7 +265,7 @@ export class ResolvedDataConnectConfig { } get schemaDir(): string { - return this.value.schema.source; + return mainSchemaYaml(this.value).source; } get relativePath(): string { diff --git a/src/commands/dataconnect-services-list.ts b/src/commands/dataconnect-services-list.ts index 8a55ad9554b..c1399a480d5 100644 --- a/src/commands/dataconnect-services-list.ts +++ b/src/commands/dataconnect-services-list.ts @@ -8,6 +8,7 @@ import { requirePermissions } from "../requirePermissions"; import { ensureApis } from "../dataconnect/ensureApis"; import * as Table from "cli-table3"; +// TODO: Update this command to also list secondary schema information. export const command = new Command("dataconnect:services:list") .description("list all deployed Data Connect services") .before(requirePermissions, [ diff --git a/src/commands/dataconnect-sql-diff.ts b/src/commands/dataconnect-sql-diff.ts index 73035ad52e8..f928aadfbdb 100644 --- a/src/commands/dataconnect-sql-diff.ts +++ b/src/commands/dataconnect-sql-diff.ts @@ -6,6 +6,7 @@ import { requirePermissions } from "../requirePermissions"; import { pickService } from "../dataconnect/load"; import { diffSchema } from "../dataconnect/schemaMigration"; import { requireAuth } from "../requireAuth"; +import { mainSchema, mainSchemaYaml } from "../dataconnect/types"; export const command = new Command("dataconnect:sql:diff [serviceId]") .description( @@ -24,8 +25,8 @@ export const command = new Command("dataconnect:sql:diff [serviceId]") const diffs = await diffSchema( options, - serviceInfo.schema, - serviceInfo.dataConnectYaml.schema.datasource.postgresql?.schemaValidation, + mainSchema(serviceInfo.schemas), + mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.schemaValidation, ); return { projectId, serviceId, diffs }; }); diff --git a/src/commands/dataconnect-sql-grant.ts b/src/commands/dataconnect-sql-grant.ts index 30a25da9e9e..0dbaf575787 100644 --- a/src/commands/dataconnect-sql-grant.ts +++ b/src/commands/dataconnect-sql-grant.ts @@ -9,6 +9,7 @@ import { requireAuth } from "../requireAuth"; import { FirebaseError } from "../error"; import { fdcSqlRoleMap } from "../gcp/cloudsql/permissionsSetup"; import { iamUserIsCSQLAdmin } from "../gcp/cloudsql/cloudsqladmin"; +import { mainSchema } from "../dataconnect/types"; const allowedRoles = Object.keys(fdcSqlRoleMap); @@ -51,6 +52,6 @@ export const command = new Command("dataconnect:sql:grant [serviceId]") await ensureApis(projectId); const serviceInfo = await pickService(projectId, options.config, serviceId); - await grantRoleToUserInSchema(options, serviceInfo.schema); + await grantRoleToUserInSchema(options, mainSchema(serviceInfo.schemas)); return { projectId, serviceId }; }); diff --git a/src/commands/dataconnect-sql-migrate.ts b/src/commands/dataconnect-sql-migrate.ts index bcc72943327..cb246527429 100644 --- a/src/commands/dataconnect-sql-migrate.ts +++ b/src/commands/dataconnect-sql-migrate.ts @@ -8,6 +8,7 @@ import { requireAuth } from "../requireAuth"; import { requirePermissions } from "../requirePermissions"; import { ensureApis } from "../dataconnect/ensureApis"; import { logLabeledSuccess } from "../utils"; +import { mainSchema, mainSchemaYaml } from "../dataconnect/types"; export const command = new Command("dataconnect:sql:migrate [serviceId]") .description("migrate your CloudSQL database's schema to match your local Data Connect schema") @@ -23,8 +24,8 @@ export const command = new Command("dataconnect:sql:migrate [serviceId]") const projectId = needProjectId(options); await ensureApis(projectId); const serviceInfo = await pickService(projectId, options.config, serviceId); - const instanceId = - serviceInfo.dataConnectYaml.schema.datasource.postgresql?.cloudSql.instanceId; + const instanceId = mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.cloudSql + .instanceId; if (!instanceId) { throw new FirebaseError( "dataconnect.yaml is missing field schema.datasource.postgresql.cloudsql.instanceId", @@ -32,9 +33,10 @@ export const command = new Command("dataconnect:sql:migrate [serviceId]") } const diffs = await migrateSchema({ options, - schema: serviceInfo.schema, + schema: mainSchema(serviceInfo.schemas), validateOnly: true, - schemaValidation: serviceInfo.dataConnectYaml.schema.datasource.postgresql?.schemaValidation, + schemaValidation: mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql + ?.schemaValidation, }); if (diffs.length) { logLabeledSuccess( diff --git a/src/commands/dataconnect-sql-setup.ts b/src/commands/dataconnect-sql-setup.ts index 164a1d55d53..cace511cf5d 100644 --- a/src/commands/dataconnect-sql-setup.ts +++ b/src/commands/dataconnect-sql-setup.ts @@ -10,6 +10,7 @@ import { setupSQLPermissions, getSchemaMetadata } from "../gcp/cloudsql/permissi import { DEFAULT_SCHEMA } from "../gcp/cloudsql/permissions"; import { getIdentifiers, ensureServiceIsConnectedToCloudSql } from "../dataconnect/schemaMigration"; import { setupIAMUsers } from "../gcp/cloudsql/connect"; +import { mainSchema, mainSchemaYaml } from "../dataconnect/types"; export const command = new Command("dataconnect:sql:setup [serviceId]") .description("set up your CloudSQL database") @@ -24,15 +25,17 @@ export const command = new Command("dataconnect:sql:setup [serviceId]") const projectId = needProjectId(options); await ensureApis(projectId); const serviceInfo = await pickService(projectId, options.config, serviceId); - const instanceId = - serviceInfo.dataConnectYaml.schema.datasource.postgresql?.cloudSql.instanceId; + const instanceId = mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.cloudSql + .instanceId; if (!instanceId) { throw new FirebaseError( "dataconnect.yaml is missing field schema.datasource.postgresql.cloudsql.instanceId", ); } - const { serviceName, instanceName, databaseId } = getIdentifiers(serviceInfo.schema); + const { serviceName, instanceName, databaseId } = getIdentifiers( + mainSchema(serviceInfo.schemas), + ); await ensureServiceIsConnectedToCloudSql( serviceName, instanceName, diff --git a/src/commands/dataconnect-sql-shell.ts b/src/commands/dataconnect-sql-shell.ts index 6023fb44905..49436aac7f2 100644 --- a/src/commands/dataconnect-sql-shell.ts +++ b/src/commands/dataconnect-sql-shell.ts @@ -17,6 +17,7 @@ import { logger } from "../logger"; import { FirebaseError } from "../error"; import { FBToolsAuthClient } from "../gcp/cloudsql/fbToolsAuthClient"; import { confirmDangerousQuery, interactiveExecuteQuery } from "../gcp/cloudsql/interactive"; +import { mainSchema } from "../dataconnect/types"; // Not a comprehensive list, used for keyword coloring. const sqlKeywords = [ @@ -91,7 +92,7 @@ export const command = new Command("dataconnect:sql:shell [serviceId]") const projectId = needProjectId(options); await ensureApis(projectId); const serviceInfo = await pickService(projectId, options.config, serviceId); - const { instanceId, databaseId } = getIdentifiers(serviceInfo.schema); + const { instanceId, databaseId } = getIdentifiers(mainSchema(serviceInfo.schemas)); const { user: username } = await getIAMUser(options); const instance = await cloudSqlAdminClient.getInstance(projectId, instanceId); diff --git a/src/dataconnect/client.spec.ts b/src/dataconnect/client.spec.ts index c6ac6d458e7..00bf8066d22 100644 --- a/src/dataconnect/client.spec.ts +++ b/src/dataconnect/client.spec.ts @@ -124,6 +124,13 @@ describe("client", () => { expect(getStub).to.be.calledWith("projects/p/locations/l/services/s/schemas/main"); }); + it("getSchema with schemaId", async () => { + getStub.resolves({ body: { name: "schema" } }); + const schema = await client.getSchema("projects/p/locations/l/services/s", "schemaId"); + expect(schema).to.deep.equal({ name: "schema" }); + expect(getStub).to.be.calledWith("projects/p/locations/l/services/s/schemas/schemaId"); + }); + it("getSchema returns undefined if not found", async () => { getStub.rejects(new FirebaseError("err", { status: 404 })); const schema = await client.getSchema("projects/p/locations/l/services/s"); diff --git a/src/dataconnect/client.ts b/src/dataconnect/client.ts index b0135de8939..aeaaa28387d 100644 --- a/src/dataconnect/client.ts +++ b/src/dataconnect/client.ts @@ -83,11 +83,12 @@ export async function deleteService(serviceName: string): Promise /** Schema methods */ -export async function getSchema(serviceName: string): Promise { +export async function getSchema( + serviceName: string, + schemaId: string = types.MAIN_SCHEMA_ID, +): Promise { try { - const res = await dataconnectClient().get( - `${serviceName}/schemas/${types.SCHEMA_ID}`, - ); + const res = await dataconnectClient().get(`${serviceName}/schemas/${schemaId}`); return res.body; } catch (err: any) { if (err.status !== 404) { @@ -146,7 +147,7 @@ export async function upsertSchema( export async function deleteSchema(serviceName: string): Promise { const op = await dataconnectClient().delete( - `${serviceName}/schemas/${types.SCHEMA_ID}`, + `${serviceName}/schemas/${types.MAIN_SCHEMA_ID}`, ); await operationPoller.pollOperation({ apiOrigin: dataconnectOrigin(), diff --git a/src/dataconnect/load.ts b/src/dataconnect/load.ts index 829dddd05a6..aedb487b073 100644 --- a/src/dataconnect/load.ts +++ b/src/dataconnect/load.ts @@ -7,7 +7,7 @@ import { Config } from "../config"; import { FirebaseError } from "../error"; import { toDatasource, - SCHEMA_ID, + MAIN_SCHEMA_ID, ConnectorYaml, DataConnectYaml, File, @@ -16,6 +16,7 @@ import { } from "./types"; import { readFileFromDirectory, wrappedSafeLoad } from "../utils"; import { DataConnectMultiple } from "../firebaseConfig"; +import * as experiments from "../experiments"; // pickService reads firebase.json and returns all services with a given serviceId. // If serviceID is not provided and there is a single service, return that. @@ -77,8 +78,20 @@ export async function load( const resolvedDir = config.path(sourceDirectory); const dataConnectYaml = await readDataConnectYaml(resolvedDir); const serviceName = `projects/${projectId}/locations/${dataConnectYaml.location}/services/${dataConnectYaml.serviceId}`; - const schemaDir = path.join(resolvedDir, dataConnectYaml.schema.source); - const schemaGQLs = await readGQLFiles(schemaDir); + const schemaYamls = dataConnectYaml.schema ? [dataConnectYaml.schema] : dataConnectYaml.schemas; + const schemas = await Promise.all( + schemaYamls!.map(async (yaml) => { + const schemaDir = path.join(resolvedDir, yaml.source); + const schemaGQLs = await readGQLFiles(schemaDir); + return { + name: `${serviceName}/schemas/${yaml.id || MAIN_SCHEMA_ID}`, + datasources: [toDatasource(projectId, dataConnectYaml.location, yaml.datasource)], + source: { + files: schemaGQLs, + }, + }; + }), + ); const connectorInfo = await Promise.all( dataConnectYaml.connectorDirs.map(async (dir) => { const connectorDir = path.join(resolvedDir, dir); @@ -100,15 +113,7 @@ export async function load( return { serviceName, sourceDirectory: resolvedDir, - schema: { - name: `${serviceName}/schemas/${SCHEMA_ID}`, - datasources: [ - toDatasource(projectId, dataConnectYaml.location, dataConnectYaml.schema.datasource), - ], - source: { - files: schemaGQLs, - }, - }, + schemas: schemas, dataConnectYaml, connectorInfo, }; @@ -149,6 +154,12 @@ function validateDataConnectYaml(unvalidated: any): DataConnectYaml { if (!unvalidated["location"]) { throw new FirebaseError("Missing required field 'location' in dataconnect.yaml"); } + if (!experiments.isEnabled("fdcwebhooks") && unvalidated["schemas"]) { + throw new FirebaseError("Unsupported field 'schemas' in dataconnect.yaml"); + } + if (!unvalidated["schema"] && !unvalidated["schemas"]) { + throw new FirebaseError("Either 'schema' or 'schemas' is required in dataconnect.yaml"); + } return unvalidated as DataConnectYaml; } diff --git a/src/dataconnect/schemaMigration.ts b/src/dataconnect/schemaMigration.ts index 7de85a86d7f..fb9e5c24e8d 100644 --- a/src/dataconnect/schemaMigration.ts +++ b/src/dataconnect/schemaMigration.ts @@ -1,7 +1,7 @@ import * as clc from "colorette"; import { format } from "sql-formatter"; -import { IncompatibleSqlSchemaError, Diff, SCHEMA_ID, SchemaValidation } from "./types"; +import { IncompatibleSqlSchemaError, Diff, MAIN_SCHEMA_ID, SchemaValidation } from "./types"; import { getSchema, upsertSchema, deleteConnector } from "./client"; import { getIAMUser, @@ -178,7 +178,7 @@ export async function migrateSchema(args: { const postgresql = schema.datasources.find((d) => d.postgresql)?.postgresql; if (!postgresql) { throw new FirebaseError( - `Cannot find Postgres datasource in the schema to deploy: ${serviceName}/schemas/${SCHEMA_ID}.\nIts datasources: ${JSON.stringify(schema.datasources)}`, + `Cannot find Postgres datasource in the schema to deploy: ${serviceName}/schemas/${MAIN_SCHEMA_ID}.\nIts datasources: ${JSON.stringify(schema.datasources)}`, ); } postgresql.schemaValidation = "NONE"; @@ -374,7 +374,7 @@ export function getIdentifiers(schema: Schema): { ); } const instanceId = instanceName.split("/").pop()!; - const serviceName = schema.name.replace(`/schemas/${SCHEMA_ID}`, ""); + const serviceName = schema.name.replace(`/schemas/${MAIN_SCHEMA_ID}`, ""); return { databaseId, instanceId, @@ -626,7 +626,7 @@ export async function ensureServiceIsConnectedToCloudSql( logLabeledBullet("dataconnect", `Linking the Cloud SQL instance...`); // If no schema has been deployed yet, deploy an empty one to get connectivity. currentSchema = { - name: `${serviceName}/schemas/${SCHEMA_ID}`, + name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`, source: { files: [], }, diff --git a/src/dataconnect/types.ts b/src/dataconnect/types.ts index c9b7dfeaa2d..3ffdae4b0ea 100644 --- a/src/dataconnect/types.ts +++ b/src/dataconnect/types.ts @@ -1,5 +1,5 @@ -// Schema is a singleton, so we always call it 'main' -export const SCHEMA_ID = "main"; +// The database schema ID is always 'main' +export const MAIN_SCHEMA_ID = "main"; // API Types interface BaseResource { @@ -26,7 +26,9 @@ export interface Connector extends BaseResource { } export interface Datasource { + // One of postgresql or httpGraphql must be set. postgresql?: PostgreSql; + httpGraphql?: HttpGraphql; } export type SchemaValidation = "STRICT" | "COMPATIBLE"; @@ -39,6 +41,11 @@ export interface PostgreSql { schemaMigration?: "MIGRATE_COMPATIBLE"; } +export interface HttpGraphql { + uri: string; + timeout?: string; +} + export interface CloudSqlInstance { instance: string; } @@ -116,13 +123,16 @@ export function requiresVector(dm?: DeploymentMetadata): boolean { export interface DataConnectYaml { specVersion?: string; serviceId: string; - schema: SchemaYaml; + // One of `schema` or `schemas` is required. + schema?: SchemaYaml; + schemas?: SchemaYaml[]; location: string; connectorDirs: string[]; } export interface SchemaYaml { source: string; + id?: string; datasource: DatasourceYaml; } @@ -134,6 +144,10 @@ export interface DatasourceYaml { }; schemaValidation?: SchemaValidation; }; + httpGraphql?: { + uri: string; + timeout?: string; + }; } export interface ConnectorYaml { @@ -183,7 +197,7 @@ export interface DartSDK { export interface ServiceInfo { serviceName: string; sourceDirectory: string; - schema: Schema; + schemas: Schema[]; connectorInfo: ConnectorInfo[]; dataConnectYaml: DataConnectYaml; deploymentMetadata?: DeploymentMetadata; @@ -211,9 +225,43 @@ export function toDatasource( }, }; } + if (ds?.httpGraphql) { + return { + httpGraphql: { + uri: ds.httpGraphql.uri, + timeout: ds.httpGraphql.timeout, + }, + }; + } return {}; } +/** Returns the main schema YAML for a Data Connect YAML */ +export function mainSchemaYaml(dataconnectYaml: DataConnectYaml): SchemaYaml { + if (dataconnectYaml.schema) { + return dataconnectYaml.schema; + } + const mainSch = dataconnectYaml.schemas?.find((s) => s.id === MAIN_SCHEMA_ID || !s.id); + if (!mainSch) { + throw new Error(`Service ${dataconnectYaml.serviceId} has no main schema defined`); + } + return mainSch; +} + +/** Returns the main schema from a list of schemas */ +export function mainSchema(schemas: Schema[]): Schema { + const mainSch = schemas.find((s) => isMainSchema(s)); + if (!mainSch) { + throw new Error(`No main schema is defined`); + } + return mainSch; +} + +/** Returns true if the schema is the main schema */ +export function isMainSchema(schema: Schema): boolean { + return schema.name.endsWith(`/schemas/${MAIN_SCHEMA_ID}`); +} + /** Start Dataplane Client Types */ export interface ExecuteGraphqlRequest { query: string; diff --git a/src/deploy/dataconnect/deploy.spec.ts b/src/deploy/dataconnect/deploy.spec.ts index 4e81d939cb3..a9faa0233b8 100644 --- a/src/deploy/dataconnect/deploy.spec.ts +++ b/src/deploy/dataconnect/deploy.spec.ts @@ -44,7 +44,12 @@ describe("dataconnect deploy", () => { { serviceName: "projects/test-project/locations/l/services/s1", deploymentMetadata: {}, - schema: { datasources: [] }, + schemas: [ + { + name: "projects/test-project/locations/l/services/s1/schemas/main", + datasources: [], + }, + ], dataConnectYaml: { serviceId: "s1" }, }, ]; @@ -103,16 +108,19 @@ describe("dataconnect deploy", () => { const serviceInfos = [ { serviceName: "projects/test-project/locations/l/services/s1", - schema: { - datasources: [ - { - postgresql: { - cloudSql: { instance: "projects/p/locations/l/instances/i" }, - database: "db", + schemas: [ + { + name: "projects/test-project/locations/l/services/s1/schemas/main", + datasources: [ + { + postgresql: { + cloudSql: { instance: "projects/p/locations/l/instances/i" }, + database: "db", + }, }, - }, - ], - }, + ], + }, + ], deploymentMetadata: {}, dataConnectYaml: { serviceId: "s1" }, }, diff --git a/src/deploy/dataconnect/deploy.ts b/src/deploy/dataconnect/deploy.ts index e2620f99515..4eab22142e2 100644 --- a/src/deploy/dataconnect/deploy.ts +++ b/src/deploy/dataconnect/deploy.ts @@ -1,7 +1,7 @@ import { Options } from "../../options"; import * as client from "../../dataconnect/client"; import * as utils from "../../utils"; -import { Service, ServiceInfo, requiresVector } from "../../dataconnect/types"; +import { Service, ServiceInfo, mainSchema, requiresVector } from "../../dataconnect/types"; import { needProjectId } from "../../projectUtils"; import { setupCloudSql } from "../../dataconnect/provisionCloudSql"; import { parseServiceName } from "../../dataconnect/names"; @@ -88,7 +88,7 @@ export default async function (context: Context, options: Options): Promise { - const postgresDatasource = s.schema.datasources.find((d) => d.postgresql); + const postgresDatasource = mainSchema(s.schemas).datasources.find((d) => d.postgresql); if (postgresDatasource) { const instanceId = postgresDatasource.postgresql?.cloudSql?.instance.split("/").pop(); const databaseId = postgresDatasource.postgresql?.database; diff --git a/src/deploy/dataconnect/prepare.spec.ts b/src/deploy/dataconnect/prepare.spec.ts index f6b3254ffb0..d21cc35638e 100644 --- a/src/deploy/dataconnect/prepare.spec.ts +++ b/src/deploy/dataconnect/prepare.spec.ts @@ -104,16 +104,19 @@ describe("dataconnect prepare", () => { it("should diff schema and setup cloud sql", async () => { const serviceInfos = [ { - schema: { - datasources: [ - { - postgresql: { - cloudSql: { instance: "projects/p/locations/l/instances/i" }, - database: "db", + schemas: [ + { + name: "projects/p/locations/l/services/s/schemas/main", + datasources: [ + { + postgresql: { + cloudSql: { instance: "projects/p/locations/l/instances/i" }, + database: "db", + }, }, - }, - ], - }, + ], + }, + ], serviceName: "projects/p/locations/l/services/s", deploymentMetadata: {}, dataConnectYaml: { diff --git a/src/deploy/dataconnect/prepare.ts b/src/deploy/dataconnect/prepare.ts index 6624ab685c0..c4c8aa74cfa 100644 --- a/src/deploy/dataconnect/prepare.ts +++ b/src/deploy/dataconnect/prepare.ts @@ -14,7 +14,7 @@ import { setupCloudSql } from "../../dataconnect/provisionCloudSql"; import { checkBillingEnabled } from "../../gcp/cloudbilling"; import { parseServiceName } from "../../dataconnect/names"; import { FirebaseError } from "../../error"; -import { requiresVector } from "../../dataconnect/types"; +import { mainSchema, requiresVector } from "../../dataconnect/types"; import { diffSchema } from "../../dataconnect/schemaMigration"; import { upgradeInstructions } from "../../dataconnect/freeTrial"; import { Context, initDeployStats } from "./context"; @@ -65,7 +65,7 @@ export default async function (context: Context, options: DeployOptions): Promis for (const si of serviceInfos) { await diffSchema( options, - si.schema, + mainSchema(si.schemas), si.dataConnectYaml.schema?.datasource?.postgresql?.schemaValidation, ); } @@ -76,7 +76,7 @@ export default async function (context: Context, options: DeployOptions): Promis return !filters || filters?.some((f) => si.dataConnectYaml.serviceId === f.serviceId); }) .map(async (s) => { - const postgresDatasource = s.schema.datasources.find((d) => d.postgresql); + const postgresDatasource = mainSchema(s.schemas).datasources.find((d) => d.postgresql); if (postgresDatasource) { const instanceId = postgresDatasource.postgresql?.cloudSql?.instance.split("/").pop(); const databaseId = postgresDatasource.postgresql?.database; diff --git a/src/deploy/dataconnect/release.spec.ts b/src/deploy/dataconnect/release.spec.ts index 6582eb389f8..5ffcda87e76 100644 --- a/src/deploy/dataconnect/release.spec.ts +++ b/src/deploy/dataconnect/release.spec.ts @@ -50,7 +50,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "my-schema" }, + schemas: [{ name: "projects/p/locations/l/services/s1/schemas/main" }], connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, @@ -88,7 +88,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "my-schema" }, + schemas: [{ name: "projects/p/locations/l/services/s1/schemas/main" }], connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, @@ -124,7 +124,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "my-schema" }, + schemas: [{ name: "projects/p/locations/l/services/s1/schemas/main" }], connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, @@ -163,7 +163,7 @@ describe("dataconnect release", () => { serviceId: "s1", schema: { datasource: { postgresql: { schemaValidation: "STRICT" } } }, }, - schema: { name: "my-schema" }, + schemas: [{ name: "projects/p/locations/l/services/s1/schemas/main" }], connectorInfo: [ { connector: { name: "projects/p/locations/l/services/s1/connectors/c1" }, diff --git a/src/deploy/dataconnect/release.ts b/src/deploy/dataconnect/release.ts index 044ed039b3f..a3bec092127 100644 --- a/src/deploy/dataconnect/release.ts +++ b/src/deploy/dataconnect/release.ts @@ -1,6 +1,12 @@ import * as utils from "../../utils"; -import { Connector, ServiceInfo } from "../../dataconnect/types"; -import { listConnectors, upsertConnector } from "../../dataconnect/client"; +import { + Connector, + isMainSchema, + mainSchema, + mainSchemaYaml, + ServiceInfo, +} from "../../dataconnect/types"; +import { listConnectors, upsertConnector, upsertSchema } from "../../dataconnect/client"; import { promptDeleteConnector } from "../../dataconnect/prompts"; import { Options } from "../../options"; import { migrateSchema } from "../../dataconnect/schemaMigration"; @@ -25,7 +31,7 @@ export default async function (context: Context, options: Options): Promise { return ( !filters || @@ -35,8 +41,8 @@ export default async function (context: Context, options: Options): Promise ({ - schema: s.schema, - validationMode: s.dataConnectYaml?.schema?.datasource?.postgresql?.schemaValidation, + schema: mainSchema(s.schemas), + validationMode: mainSchemaYaml(s.dataConnectYaml).datasource?.postgresql?.schemaValidation, })); const wantConnectors = serviceInfos.flatMap((si) => si.connectorInfo @@ -71,8 +77,8 @@ export default async function (context: Context, options: Options): Promise { + return ( + !filters || + filters.some((f) => { + return f.serviceId === si.dataConnectYaml.serviceId && (f.schemaOnly || f.fullService); + }) + ); + }) + .map((s) => s.schemas.filter((s) => !isMainSchema(s))) + .flatMap((s) => s); + for (const schema of wantSecondarySchemas) { + await upsertSchema(schema, false); + utils.logLabeledSuccess("dataconnect", `Migrated schema ${schema.name}`); + dataconnect.deployStats.numSchemaMigrated++; + } + // Lastly, deploy the remaining connectors that relies on the latest schema. await Promise.all( remainingConnectors.map(async (c) => { diff --git a/src/emulator/dataconnectEmulator.ts b/src/emulator/dataconnectEmulator.ts index 554c8405ce5..d950e5a2696 100644 --- a/src/emulator/dataconnectEmulator.ts +++ b/src/emulator/dataconnectEmulator.ts @@ -17,7 +17,7 @@ import { import { EmulatorInfo, EmulatorInstance, Emulators, ListenSpec } from "./types"; import { FirebaseError } from "../error"; import { EmulatorLogger } from "./emulatorLogger"; -import { BuildResult, requiresVector } from "../dataconnect/types"; +import { BuildResult, mainSchemaYaml, requiresVector } from "../dataconnect/types"; import { listenSpecsToString } from "./portUtils"; import { Client, ClientResponse } from "../apiv2"; import { EmulatorRegistry } from "./registry"; @@ -114,7 +114,8 @@ export class DataConnectEmulator implements EmulatorInstance { this.usingExistingEmulator = false; if (this.args.autoconnectToPostgres) { const info = await load(this.args.projectId, this.args.config, this.args.configDir); - const dbId = info.dataConnectYaml.schema.datasource.postgresql?.database || "postgres"; + const dbId = + mainSchemaYaml(info.dataConnectYaml).datasource.postgresql?.database || "postgres"; const serviceId = info.dataConnectYaml.serviceId; const pgPort = this.args.postgresListen?.[0].port; const pgHost = this.args.postgresListen?.[0].address; diff --git a/src/experiments.ts b/src/experiments.ts index a18e4ca8eb7..7ec9aa6975c 100644 --- a/src/experiments.ts +++ b/src/experiments.ts @@ -148,6 +148,11 @@ export const ALL_EXPERIMENTS = experiments({ shortDescription: "Adds experimental App Testing feature", public: true, }, + fdcwebhooks: { + shortDescription: "Enable Firebase Data Connect webhooks feature.", + default: false, + public: false, + }, }); export type ExperimentName = keyof typeof ALL_EXPERIMENTS; diff --git a/src/init/features/dataconnect/index.spec.ts b/src/init/features/dataconnect/index.spec.ts index f52d04bdd83..ae167df5d1b 100644 --- a/src/init/features/dataconnect/index.spec.ts +++ b/src/init/features/dataconnect/index.spec.ts @@ -31,7 +31,7 @@ describe("init dataconnect", () => { sdkActuateStub = sandbox.stub(sdk, "actuate").resolves(); sandbox.stub(cloudbilling, "isBillingEnabled").resolves(true); sandbox.stub(ensureApis, "ensureApis").resolves(); - sandbox.stub(client, "getSchema").resolves(undefined); + sandbox.stub(client, "listSchemas").resolves([]); }); afterEach(() => { diff --git a/src/init/features/dataconnect/index.ts b/src/init/features/dataconnect/index.ts index 11f460ba1ef..2d54ac4c8f4 100644 --- a/src/init/features/dataconnect/index.ts +++ b/src/init/features/dataconnect/index.ts @@ -9,15 +9,16 @@ import { setupCloudSql } from "../../../dataconnect/provisionCloudSql"; import { checkFreeTrialInstanceUsed, upgradeInstructions } from "../../../dataconnect/freeTrial"; import * as cloudsql from "../../../gcp/cloudsql/cloudsqladmin"; import { ensureApis, ensureGIFApiTos } from "../../../dataconnect/ensureApis"; +import * as experiments from "../../../experiments"; import { listLocations, listAllServices, - getSchema, + listSchemas, listConnectors, createService, upsertSchema, } from "../../../dataconnect/client"; -import { Schema, Service, File, SCHEMA_ID } from "../../../dataconnect/types"; +import { Schema, Service, File, MAIN_SCHEMA_ID, mainSchema } from "../../../dataconnect/types"; import { parseCloudSQLInstanceName, parseServiceName } from "../../../dataconnect/names"; import { logger } from "../../../logger"; import { readTemplateSync } from "../../../templates"; @@ -44,6 +45,9 @@ import { trackGA4 } from "../../../track"; export const FDC_DEFAULT_REGION = "us-east4"; const DATACONNECT_YAML_TEMPLATE = readTemplateSync("init/dataconnect/dataconnect.yaml"); +const DATACONNECT_WEBHOOKS_YAML_TEMPLATE = readTemplateSync( + "init/dataconnect/dataconnect-fdcwebhooks.yaml", +); const CONNECTOR_YAML_TEMPLATE = readTemplateSync("init/dataconnect/connector.yaml"); const SCHEMA_TEMPLATE = readTemplateSync("init/dataconnect/schema.gql"); const QUERIES_TEMPLATE = readTemplateSync("init/dataconnect/queries.gql"); @@ -346,7 +350,7 @@ function schemasDeploySequence( // No Cloud SQL is being provisioned, just deploy the schema sources as a unlinked schema. return [ { - name: `${serviceName}/schemas/${SCHEMA_ID}`, + name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`, datasources: [{ postgresql: {} }], source: { files: schemaFiles, @@ -359,7 +363,7 @@ function schemasDeploySequence( // wait for Cloud SQL provision to finish and setup its initial SQL schemas. return [ { - name: `${serviceName}/schemas/${SCHEMA_ID}`, + name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`, datasources: [ { postgresql: { @@ -376,7 +380,7 @@ function schemasDeploySequence( }, }, { - name: `${serviceName}/schemas/${SCHEMA_ID}`, + name: `${serviceName}/schemas/${MAIN_SCHEMA_ID}`, datasources: [ { postgresql: { @@ -478,7 +482,9 @@ function subDataconnectYamlValues(replacementValues: { cloudSqlInstanceId: "__cloudSqlInstanceId__", connectorDirs: "__connectorDirs__", }; - let replaced = DATACONNECT_YAML_TEMPLATE; + let replaced = experiments.isEnabled("fdcwebhooks") + ? DATACONNECT_WEBHOOKS_YAML_TEMPLATE + : DATACONNECT_YAML_TEMPLATE; for (const [k, v] of Object.entries(replacementValues)) { replaced = replaced.replace(replacements[k], JSON.stringify(v)); } @@ -521,8 +527,19 @@ async function promptForExistingServices(setup: Setup, info: RequiredInfo): Prom } async function downloadService(info: RequiredInfo, serviceName: string): Promise { - const schema = await getSchema(serviceName); - if (!schema) { + let schemas: Schema[] = []; + try { + schemas = await listSchemas(serviceName, [ + "schemas.name", + "schemas.datasources", + "schemas.source", + ]); + } catch (err: any) { + if (err.status !== 404) { + throw err; + } + } + if (!schemas.length) { return; } info.serviceGql = { @@ -535,13 +552,15 @@ async function downloadService(info: RequiredInfo, serviceName: string): Promise }, ], }; - const primaryDatasource = schema.datasources.find((d) => d.postgresql); + const mainSch = mainSchema(schemas); + const primaryDatasource = mainSch.datasources.find((d) => d.postgresql); if (primaryDatasource?.postgresql?.cloudSql?.instance) { const instanceName = parseCloudSQLInstanceName(primaryDatasource.postgresql.cloudSql.instance); info.cloudSqlInstanceId = instanceName.instanceId; } - if (schema.source.files?.length) { - info.serviceGql.schemaGql = schema.source.files; + // TODO: Update dataconnect.yaml with downloaded secondary schemas as well. + if (mainSch.source.files?.length) { + info.serviceGql.schemaGql = mainSch.source.files; } info.cloudSqlDatabase = primaryDatasource?.postgresql?.database ?? ""; const connectors = await listConnectors(serviceName, [ diff --git a/src/mcp/prompts/dataconnect/schema.ts b/src/mcp/prompts/dataconnect/schema.ts index fd7c0491f3a..b12630eda85 100644 --- a/src/mcp/prompts/dataconnect/schema.ts +++ b/src/mcp/prompts/dataconnect/schema.ts @@ -1,17 +1,20 @@ import { prompt } from "../../prompt"; import { loadAll } from "../../../dataconnect/load"; -import type { ServiceInfo } from "../../../dataconnect/types"; +import { mainSchema, type ServiceInfo } from "../../../dataconnect/types"; import { BUILTIN_SDL, MAIN_INSTRUCTIONS } from "../../util/dataconnect/content"; import { compileErrors } from "../../util/dataconnect/compile"; function renderServices(fdcServices: ServiceInfo[]) { if (!fdcServices.length) return "Data Connect Status: "; + // TODO: Render secondary schemas as well. return `\n\n## Data Connect Schema The following is the up-to-date content of existing schema files (their paths are relative to the Data Connect source directory). -${fdcServices[0].schema.source.files?.map((f) => `\`\`\`graphql ${f.path}\n${f.content}\n\`\`\``).join("\n\n")}`; +${mainSchema(fdcServices[0].schemas) + .source.files?.map((f) => `\`\`\`graphql ${f.path}\n${f.content}\n\`\`\``) + .join("\n\n")}`; } function renderErrors(errors?: string) { diff --git a/src/mcp/tools/dataconnect/list_services.ts b/src/mcp/tools/dataconnect/list_services.ts index eecba90ed32..f2ac4af684a 100644 --- a/src/mcp/tools/dataconnect/list_services.ts +++ b/src/mcp/tools/dataconnect/list_services.ts @@ -4,7 +4,13 @@ import { tool } from "../../tool"; import { toContent } from "../../util"; import * as client from "../../../dataconnect/client"; import { loadAll } from "../../../dataconnect/load"; -import { Service, Schema, ServiceInfo, Connector } from "../../../dataconnect/types"; +import { + Service, + Schema, + ServiceInfo, + Connector, + mainSchemaYaml, +} from "../../../dataconnect/types"; import { dump } from "js-yaml"; import { logger } from "../../../logger"; @@ -108,7 +114,11 @@ export const list_services = tool( for (const s of localServices) { const local = s.local!; output.push(dump(local.dataConnectYaml)); - const schemaDir = path.join(local.sourceDirectory, local.dataConnectYaml.schema.source); + // TODO: Include secondary schema sources here as well. + const schemaDir = path.join( + local.sourceDirectory, + mainSchemaYaml(local.dataConnectYaml).source, + ); output.push(`You can find all of schema sources under ${schemaDir}/`); if (s.deployed) { output.push(`It's already deployed in the backend:\n`); diff --git a/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml b/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml new file mode 100644 index 00000000000..ed5f6b1d5dc --- /dev/null +++ b/templates/init/dataconnect/dataconnect-fdcwebhooks.yaml @@ -0,0 +1,13 @@ +specVersion: "v1" +serviceId: __serviceId__ +location: __location__ +schemas: + - source: "./schema" + datasource: + postgresql: + database: __cloudSqlDatabase__ + cloudSql: + instanceId: __cloudSqlInstanceId__ + # schemaValidation: "STRICT" # STRICT mode makes Postgres schema match Data Connect exactly. + # schemaValidation: "COMPATIBLE" # COMPATIBLE mode makes Postgres schema compatible with Data Connect. +connectorDirs: __connectorDirs__