diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a3641dbe98..150bf41efc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1 +1,2 @@ -- Improve performance and reliability when deploying multiple 2nd gen functions using single builds. (#6275) +- Fixed issue where the Extensions emulator would error when emualting local extensions with no params. (#6271) +- Improved performance and reliability when deploying multiple 2nd gen functions using single builds. (#6275) diff --git a/src/emulator/extensionsEmulator.ts b/src/emulator/extensionsEmulator.ts index 3c522dbf001..17a710b8954 100644 --- a/src/emulator/extensionsEmulator.ts +++ b/src/emulator/extensionsEmulator.ts @@ -143,11 +143,7 @@ export class ExtensionsEmulator implements EmulatorInstance { private hasValidSource(args: { path: string; extTarget: string }): boolean { // TODO(lihes): Source code can technically exist in other than "functions" dir. // https://source.corp.google.com/piper///depot/google3/firebase/mods/go/worker/fetch_mod_source.go;l=451 - const requiredFiles = [ - "./extension.yaml", - "./functions/package.json", - "./functions/node_modules", - ]; + const requiredFiles = ["./extension.yaml", "./functions/package.json"]; // If the directory isn't found, no need to check for files or print errors. if (!fs.existsSync(args.path)) { return false; @@ -226,10 +222,12 @@ export class ExtensionsEmulator implements EmulatorInstance { instance: planner.DeploymentInstanceSpec ): Promise { const extensionDir = await this.ensureSourceCode(instance); + // TODO: This should find package.json, then use that as functionsDir. const functionsDir = path.join(extensionDir, "functions"); // TODO(b/213335255): For local extensions, this should include extensionSpec instead of extensionVersion const env = Object.assign(this.autoPopulatedParams(instance), instance.params); + const { extensionTriggers, runtime, nonSecretEnv, secretEnvVariables } = await getExtensionFunctionInfo(instance, env); const emulatableBackend: EmulatableBackend = { @@ -248,6 +246,7 @@ export class ExtensionsEmulator implements EmulatorInstance { } else if (instance.localPath) { emulatableBackend.extensionSpec = await planner.getExtensionSpec(instance); } + return emulatableBackend; } diff --git a/src/extensions/emulator/optionsHelper.ts b/src/extensions/emulator/optionsHelper.ts index 202909e4f1a..dfef10bc05a 100644 --- a/src/extensions/emulator/optionsHelper.ts +++ b/src/extensions/emulator/optionsHelper.ts @@ -1,49 +1,13 @@ -import * as fs from "fs-extra"; import { ParsedTriggerDefinition } from "../../emulator/functionsEmulatorShared"; -import * as path from "path"; import * as paramHelper from "../paramHelper"; import * as specHelper from "./specHelper"; -import * as localHelper from "../localHelper"; import * as triggerHelper from "./triggerHelper"; -import { ExtensionSpec, Param, ParamType, Resource } from "../types"; +import { ExtensionSpec, Param, ParamType } from "../types"; import * as extensionsHelper from "../extensionsHelper"; import * as planner from "../../deploy/extensions/planner"; -import { Config } from "../../config"; -import { FirebaseError } from "../../error"; -import { EmulatorLogger } from "../../emulator/emulatorLogger"; import { needProjectId } from "../../projectUtils"; -import { Emulators } from "../../emulator/types"; import { SecretEnvVar } from "../../deploy/functions/backend"; -/** - * Build firebase options based on the extension configuration. - */ -export async function buildOptions(options: any): Promise { - const extDevDir = localHelper.findExtensionYaml(process.cwd()); - options.extDevDir = extDevDir; - const spec = await specHelper.readExtensionYaml(extDevDir); - extensionsHelper.validateSpec(spec); - - const params = getParams(options, spec); - - extensionsHelper.validateCommandLineParams(params, spec.params); - - const functionResources = specHelper.getFunctionResourcesWithParamSubstitution(spec, params); - let testConfig; - if (options.testConfig) { - testConfig = readTestConfigFile(options.testConfig); - checkTestConfig(testConfig, functionResources); - } - options.config = buildConfig(functionResources, testConfig); - options.extDevEnv = params; - const functionEmuTriggerDefs: ParsedTriggerDefinition[] = functionResources.map((r) => - triggerHelper.functionResourceToEmulatedTriggerDefintion(r) - ); - options.extDevTriggers = functionEmuTriggerDefs; - options.extDevRuntime = specHelper.getRuntime(functionResources); - return options; -} - /** * TODO: Better name? Also, should this be in extensionsEmulator instead? */ @@ -66,8 +30,8 @@ export async function getExtensionFunctionInfo( }); const runtime = specHelper.getRuntime(functionResources); - const nonSecretEnv = getNonSecretEnv(spec.params, paramValues); - const secretEnvVariables = getSecretEnvVars(spec.params, paramValues); + const nonSecretEnv = getNonSecretEnv(spec.params ?? [], paramValues); + const secretEnvVariables = getSecretEnvVars(spec.params ?? [], paramValues); return { extensionTriggers, runtime, @@ -144,148 +108,3 @@ export function getParams(options: any, extensionSpec: ExtensionSpec) { // Run a substitution to support params that reference other params. return extensionsHelper.substituteParams>(unsubbedParams, unsubbedParams); } - -/** - * Checks and warns if the test config is missing fields - * that are relevant for the extension being emulated. - */ -function checkTestConfig(testConfig: { [key: string]: any }, functionResources: Resource[]) { - const logger = EmulatorLogger.forEmulator(Emulators.FUNCTIONS); - if (!testConfig.functions && functionResources.length) { - logger.log( - "WARN", - "This extension uses functions," + - "but 'firebase.json' provided by --test-config is missing a top-level 'functions' object." + - "Functions will not be emulated." - ); - } - - if (!testConfig.firestore && shouldEmulateFirestore(functionResources)) { - logger.log( - "WARN", - "This extension interacts with Cloud Firestore," + - "but 'firebase.json' provided by --test-config is missing a top-level 'firestore' object." + - "Cloud Firestore will not be emulated." - ); - } - - if (!testConfig.database && shouldEmulateDatabase(functionResources)) { - logger.log( - "WARN", - "This extension interacts with Realtime Database," + - "but 'firebase.json' provided by --test-config is missing a top-level 'database' object." + - "Realtime Database will not be emulated." - ); - } - - if (!testConfig.storage && shouldEmulateStorage(functionResources)) { - logger.log( - "WARN", - "This extension interacts with Cloud Storage," + - "but 'firebase.json' provided by --test-config is missing a top-level 'storage' object." + - "Cloud Storage will not be emulated." - ); - } -} - -/** - * Reads a test config file. - * @param testConfigPath filepath to a firebase.json style config file. - */ -function readTestConfigFile(testConfigPath: string): { [key: string]: any } { - try { - const buf = fs.readFileSync(path.resolve(testConfigPath)); - return JSON.parse(buf.toString()); - } catch (err: any) { - throw new FirebaseError(`Error reading --test-config file: ${err.message}\n`, { - original: err, - }); - } -} - -function buildConfig( - functionResources: Resource[], - testConfig?: { [key: string]: string } -): Config { - const config = new Config(testConfig || {}, { projectDir: process.cwd(), cwd: process.cwd() }); - - const emulateFunctions = shouldEmulateFunctions(functionResources); - if (!testConfig) { - // If testConfig was provided, don't add any new blocks. - if (emulateFunctions) { - config.set("functions", {}); - } - if (shouldEmulateFirestore(functionResources)) { - config.set("firestore", {}); - } - if (shouldEmulateDatabase(functionResources)) { - config.set("database", {}); - } - if (shouldEmulatePubsub(functionResources)) { - config.set("pubsub", {}); - } - if (shouldEmulateStorage(functionResources)) { - config.set("storage", {}); - } - } - - if (config.src.functions) { - // Switch functions source to what is provided in the extension.yaml - // to match the behavior of deployed extensions. - const sourceDirectory = getFunctionSourceDirectory(functionResources); - config.set("functions.source", sourceDirectory); - } - return config; -} - -/** - * Finds the source directory from extension.yaml to use for emulating functions. - * Errors if the extension.yaml contins function resources with different or missing - * values for properties.sourceDirectory. - * @param functionResources An array of function type resources - */ -function getFunctionSourceDirectory(functionResources: Resource[]): string { - let sourceDirectory; - for (const r of functionResources) { - // If not specified, default sourceDirectory to "functions" - const dir = r.properties?.sourceDirectory || "functions"; - if (!sourceDirectory) { - sourceDirectory = dir; - } else if (sourceDirectory !== dir) { - throw new FirebaseError( - `Found function resources with different sourceDirectories: '${sourceDirectory}' and '${dir}'. The extensions emulator only supports a single sourceDirectory.` - ); - } - } - return sourceDirectory || "functions"; -} - -function shouldEmulateFunctions(resources: Resource[]): boolean { - return resources.length > 0; -} - -function shouldEmulate(emulatorName: string, resources: Resource[]): boolean { - for (const r of resources) { - const eventType: string = r.properties?.eventTrigger?.eventType || ""; - if (eventType.includes(emulatorName)) { - return true; - } - } - return false; -} - -function shouldEmulateFirestore(resources: Resource[]): boolean { - return shouldEmulate("cloud.firestore", resources); -} - -function shouldEmulateDatabase(resources: Resource[]): boolean { - return shouldEmulate("google.firebase.database", resources); -} - -function shouldEmulatePubsub(resources: Resource[]): boolean { - return shouldEmulate("google.pubsub", resources); -} - -function shouldEmulateStorage(resources: Resource[]): boolean { - return shouldEmulate("google.storage", resources); -} diff --git a/src/extensions/emulator/specHelper.ts b/src/extensions/emulator/specHelper.ts index 47aeef51ba8..badecb4945e 100644 --- a/src/extensions/emulator/specHelper.ts +++ b/src/extensions/emulator/specHelper.ts @@ -37,7 +37,19 @@ function wrappedSafeLoad(source: string): any { export async function readExtensionYaml(directory: string): Promise { const extensionYaml = await readFileFromDirectory(directory, SPEC_FILE); const source = extensionYaml.source; - return wrappedSafeLoad(source); + const spec = wrappedSafeLoad(source); + // Ensure that any omitted array fields are initialized as empty arrays + spec.params = spec.params ?? []; + spec.systemParams = spec.systemParams ?? []; + spec.resources = spec.resources ?? []; + spec.apis = spec.apis ?? []; + spec.roles = spec.roles ?? []; + spec.externalServices = spec.externalServices ?? []; + spec.events = spec.events ?? []; + spec.lifecycleEvents = spec.lifecycleEvents ?? []; + spec.contributors = spec.contributors ?? []; + + return spec; } /** diff --git a/src/test/emulators/extensions/firebase/storage-resize-images@0.1.18/functions/node_modules/.gitkeep b/src/test/emulators/extensions/firebase/storage-resize-images@0.1.18/functions/node_modules/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/test/emulators/extensions/firebase/storage-resize-images@0.1.18/functions/package.json b/src/test/emulators/extensions/firebase/storage-resize-images@0.1.18/functions/package.json index a821a111f1a..e8ba8a23d5b 100644 --- a/src/test/emulators/extensions/firebase/storage-resize-images@0.1.18/functions/package.json +++ b/src/test/emulators/extensions/firebase/storage-resize-images@0.1.18/functions/package.json @@ -1,8 +1,5 @@ { "name": "storage-resize-images", "vresion": "0.1.18", - "description": "Package file for testing only", - "dependencies": { - "firebase-tools": "file:../../../../../../.." - } + "description": "Package file for testing only" } diff --git a/src/test/emulators/extensionsEmulator.spec.ts b/src/test/emulators/extensionsEmulator.spec.ts index 5e2f17d59e7..49a27f3b71d 100644 --- a/src/test/emulators/extensionsEmulator.spec.ts +++ b/src/test/emulators/extensionsEmulator.spec.ts @@ -1,5 +1,4 @@ import { expect } from "chai"; -import { existsSync } from "node:fs"; import { join } from "node:path"; import * as planner from "../../deploy/extensions/planner"; @@ -147,30 +146,4 @@ describe("Extensions Emulator", () => { }); } }); - - describe("installAndBuildSourceCode", () => { - const extensionPath = "src/test/emulators/extensions/firebase/storage-resize-images@0.1.18"; - it("installs dependecies", () => { - // creating a subclass of ext emulator - // to be able to test private method - class DependencyInstallingExtensionsEmulator extends ExtensionsEmulator { - constructor(extensionPath: string) { - super({ - projectId: "test-project", - projectNumber: "1234567", - projectDir: ".", - extensions: {}, - aliases: [], - }); - - this.installAndBuildSourceCode(extensionPath); - } - } - new DependencyInstallingExtensionsEmulator(extensionPath); - const nodeModulesFolderExists = existsSync( - `${extensionPath}/functions/node_modules/firebase-tools` - ); - expect(nodeModulesFolderExists).to.be.true; - }).timeout(60_000); - }); }); diff --git a/src/test/extensions/emulator/specHelper.spec.ts b/src/test/extensions/emulator/specHelper.spec.ts index de4c917326c..6c6720b0f08 100644 --- a/src/test/extensions/emulator/specHelper.spec.ts +++ b/src/test/extensions/emulator/specHelper.spec.ts @@ -1,4 +1,5 @@ import { expect } from "chai"; +import * as path from "path"; import * as specHelper from "../../../extensions/emulator/specHelper"; import { Resource } from "../../../extensions/types"; @@ -15,6 +16,87 @@ const testResource: Resource = { }, }; +describe("readExtensionYaml", () => { + const testCases: { + desc: string; + directory: string; + expected: any; // ExtensionSpec + }[] = [ + { + desc: "should read a minimal extension.yaml", + directory: path.resolve(__dirname, "../../fixtures/extension-yamls/minimal"), + expected: { + apis: [], + contributors: [], + description: "Sends the world a greeting.", + displayName: "Greet the world", + events: [], + externalServices: [], + license: "Apache-2.0", + lifecycleEvents: [], + name: "greet-the-world", + params: [], + resources: [], + roles: [], + specVersion: "v1beta", + systemParams: [], + version: "0.0.1", + }, + }, + { + desc: "should read a hello-world extension.yaml", + directory: path.resolve(__dirname, "../../fixtures/extension-yamls/hello-world"), + expected: { + apis: [], + billingRequired: true, + contributors: [], + description: "Sends the world a greeting.", + displayName: "Greet the world", + events: [], + externalServices: [], + license: "Apache-2.0", + lifecycleEvents: [], + name: "greet-the-world", + params: [ + { + default: "Hello", + description: + "What do you want to say to the world? For example, Hello world? or What's up, world?", + immutable: false, + label: "Greeting for the world", + param: "GREETING", + required: true, + type: "string", + }, + ], + resources: [ + { + description: + "HTTP request-triggered function that responds with a specified greeting message", + name: "greetTheWorld", + properties: { + httpsTrigger: {}, + runtime: "nodejs16", + }, + type: "firebaseextensions.v1beta.function", + }, + ], + roles: [], + sourceUrl: "https://github.com/ORG_OR_USER/REPO_NAME", + specVersion: "v1beta", + systemParams: [], + version: "0.0.1", + }, + }, + ]; + for (const tc of testCases) { + it(tc.desc, async () => { + const spec = await specHelper.readExtensionYaml(tc.directory); + expect(spec).to.deep.equal(tc.expected); + }); + } +}); + describe("getRuntime", () => { it("gets runtime of resources", () => { const r1 = { diff --git a/src/test/fixtures/extension-yamls/hello-world/extension.yaml b/src/test/fixtures/extension-yamls/hello-world/extension.yaml new file mode 100644 index 00000000000..7ebd008f641 --- /dev/null +++ b/src/test/fixtures/extension-yamls/hello-world/extension.yaml @@ -0,0 +1,61 @@ +# Learn detailed information about the fields of an extension.yaml file in the docs: +# https://firebase.google.com/docs/extensions/reference/extension-yaml + +# Identifier for your extension +# TODO: Replace this with an descriptive name for your extension. +name: greet-the-world +version: 0.0.1 # Follow semver versioning +specVersion: v1beta # Version of the Firebase Extensions specification + +# Friendly display name for your extension (~3-5 words) +displayName: Greet the world + +# Brief description of the task your extension performs (~1 sentence) +description: >- + Sends the world a greeting. + +license: Apache-2.0 # https://spdx.org/licenses/ + +# Public URL for the source code of your extension. +# TODO: Replace this with your GitHub repo. +sourceUrl: https://github.com/ORG_OR_USER/REPO_NAME + +# Specify whether a paid-tier billing plan is required to use your extension. +# Learn more in the docs: https://firebase.google.com/docs/extensions/reference/extension-yaml#billing-required-field +billingRequired: true + +# In an `apis` field, list any Google APIs (like Cloud Translation, BigQuery, etc.) +# required for your extension to operate. +# Learn more in the docs: +# https://firebase.google.com/docs/extensions/reference/extension-yaml#apis-field + +# In a `roles` field, list any IAM access roles required for your extension to operate. +# Learn more in the docs: +# https://firebase.google.com/docs/extensions/reference/extension-yaml#roles-field + +# In the `resources` field, list each of your extension's functions, including the trigger for each function. +# Learn more in the docs: +# https://firebase.google.com/docs/extensions/reference/extension-yaml#resources-field +resources: + - name: greetTheWorld + type: firebaseextensions.v1beta.function + description: >- + HTTP request-triggered function that responds with a specified greeting message + properties: + # httpsTrigger is used for an HTTP triggered function. + httpsTrigger: {} + runtime: "nodejs16" + +# In the `params` field, set up your extension's user-configured parameters. +# Learn more in the docs: +# https://firebase.google.com/docs/extensions/reference/extension-yaml#params-field +params: + - param: GREETING + label: Greeting for the world + description: >- + What do you want to say to the world? + For example, Hello world? or What's up, world? + type: string + default: Hello + required: true + immutable: false diff --git a/src/test/fixtures/extension-yamls/minimal/extension.yaml b/src/test/fixtures/extension-yamls/minimal/extension.yaml new file mode 100644 index 00000000000..6cdde1230c1 --- /dev/null +++ b/src/test/fixtures/extension-yamls/minimal/extension.yaml @@ -0,0 +1,17 @@ +# Learn detailed information about the fields of an extension.yaml file in the docs: +# https://firebase.google.com/docs/extensions/reference/extension-yaml + +# Identifier for your extension +# TODO: Replace this with an descriptive name for your extension. +name: greet-the-world +version: 0.0.1 # Follow semver versioning +specVersion: v1beta # Version of the Firebase Extensions specification + +# Friendly display name for your extension (~3-5 words) +displayName: Greet the world + +# Brief description of the task your extension performs (~1 sentence) +description: >- + Sends the world a greeting. + +license: Apache-2.0 # https://spdx.org/licenses/