From 2de70201470dd01709949358d7824441817e5cdb Mon Sep 17 00:00:00 2001 From: Russell Wheatley Date: Fri, 31 Jul 2020 00:26:41 +0100 Subject: [PATCH 01/24] fix(firestore-bigquery-export): Fix document id bug when upgrading BQ extension version (#398) --- .../package.json | 2 +- .../src/bigquery/index.ts | 90 ++++++++++++++++++- .../src/bigquery/schema.ts | 21 ++--- .../src/logs.ts | 10 +++ 4 files changed, 108 insertions(+), 15 deletions(-) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json index 69ffe0f43..099484058 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/package.json @@ -5,7 +5,7 @@ "url": "github.com/firebase/extensions.git", "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker" }, - "version": "1.1.9", + "version": "1.1.10", "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports", "main": "./lib/index.js", "scripts": { diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts index 3b3dd248d..3cf3317e9 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/index.ts @@ -17,7 +17,11 @@ import * as bigquery from "@google-cloud/bigquery"; import * as firebase from "firebase-admin"; import * as traverse from "traverse"; -import { RawChangelogSchema, RawChangelogViewSchema } from "./schema"; +import { + RawChangelogSchema, + RawChangelogViewSchema, + documentIdField, +} from "./schema"; import { latestConsistentSnapshotView } from "./snapshot"; import { @@ -26,6 +30,7 @@ import { FirestoreDocumentChangeEvent, } from "../tracker"; import * as logs from "../logs"; +import { InsertRowsOptions } from "@google-cloud/bigquery/build/src/table"; export { RawChangelogSchema, RawChangelogViewSchema } from "./schema"; @@ -96,14 +101,57 @@ export class FirestoreBigQueryEventHistoryTracker return data; } + /** + * Check whether a failed operation is retryable or not. + * Reasons for retrying: + * 1) We added a new column to our schema. Sometimes BQ is not ready to stream insertion records immediately + * after adding a new column to an existing table (https://issuetracker.google.com/35905247) + */ + private async isRetryableInsertionError(e) { + let isRetryable = true; + const expectedErrors = [ + { + message: "no such field.", + location: "document_id", + }, + ]; + if ( + e.response && + e.response.insertErrors && + e.response.insertErrors.errors + ) { + const errors = e.response.insertErrors.errors; + errors.forEach((error) => { + let isExpected = false; + expectedErrors.forEach((expectedError) => { + if ( + error.message === expectedError.message && + error.location === expectedError.location + ) { + isExpected = true; + } + }); + if (!isExpected) { + isRetryable = false; + } + }); + } + return isRetryable; + } + /** * Inserts rows of data into the BigQuery raw change log table. */ - private async insertData(rows: bigquery.RowMetadata[]) { + private async insertData( + rows: bigquery.RowMetadata[], + overrideOptions: InsertRowsOptions = {}, + retry: boolean = true + ) { const options = { skipInvalidRows: false, ignoreUnknownValues: false, raw: true, + ...overrideOptions, }; try { const dataset = this.bq.dataset(this.config.datasetId); @@ -112,6 +160,15 @@ export class FirestoreBigQueryEventHistoryTracker await table.insert(rows, options); logs.dataInserted(rows.length); } catch (e) { + if (retry && this.isRetryableInsertionError(e)) { + retry = false; + logs.dataInsertRetried(rows.length); + return this.insertData( + rows, + { ...overrideOptions, ignoreUnknownValues: true }, + retry + ); + } // Reinitializing in case the destintation table is modified. this.initialized = false; throw e; @@ -160,6 +217,19 @@ export class FirestoreBigQueryEventHistoryTracker if (tableExists) { logs.bigQueryTableAlreadyExists(table.id, dataset.id); + + const [metadata] = await table.getMetadata(); + const fields = metadata.schema.fields; + + const documentIdColExists = fields.find( + (column) => column.name === "document_id" + ); + + if (!documentIdColExists) { + fields.push(documentIdField); + await table.setMetadata(metadata); + logs.addDocumentIdColumn(this.rawChangeLogTableName()); + } } else { logs.bigQueryTableCreating(changelogName); const options = { @@ -184,6 +254,22 @@ export class FirestoreBigQueryEventHistoryTracker if (viewExists) { logs.bigQueryViewAlreadyExists(view.id, dataset.id); + const [metadata] = await view.getMetadata(); + const fields = metadata.schema.fields; + + const documentIdColExists = fields.find( + (column) => column.name === "document_id" + ); + + if (!documentIdColExists) { + metadata.view = latestConsistentSnapshotView( + this.config.datasetId, + this.rawChangeLogTableName() + ); + + await view.setMetadata(metadata); + logs.addDocumentIdColumn(this.rawLatestView()); + } } else { const latestSnapshot = latestConsistentSnapshotView( this.config.datasetId, diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts index 0a96c324d..57f8dc74b 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/schema.ts @@ -59,6 +59,13 @@ export const timestampField = bigQueryField( export const latitudeField = bigQueryField("latitude", "NUMERIC"); export const longitudeField = bigQueryField("longitude", "NUMERIC"); +export const documentIdField = { + name: "document_id", + mode: "NULLABLE", + type: "STRING", + description: "The document id as defined in the firestore database.", +}; + /* * We cannot specify a schema for view creation, and all view columns default * to the NULLABLE mode. @@ -99,12 +106,7 @@ export const RawChangelogViewSchema: any = { description: "The full JSON representation of the current document state.", }, - { - name: "document_id", - mode: "NULLABLE", - type: "STRING", - description: "The document id as defined in the firestore database.", - }, + documentIdField, ], }; @@ -144,11 +146,6 @@ export const RawChangelogSchema: any = { description: "The full JSON representation of the document state after the indicated operation is applied. This field will be null for DELETE operations.", }, - { - name: "document_id", - mode: "NULLABLE", - type: "STRING", - description: "The document id as defined in the firestore database.", - }, + documentIdField, ], }; diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts index 13fad4ee2..abb3fb6f9 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts @@ -135,6 +135,12 @@ export const dataInserted = (rowCount: number) => { console.log(`Inserted ${rowCount} row(s) of data into BigQuery`); }; +export const dataInsertRetried = (rowCount: number) => { + console.log( + `Retried to insert ${rowCount} row(s) of data into BigQuery (ignoring uknown columns)` + ); +}; + export const dataInserting = (rowCount: number) => { console.log(`Inserting ${rowCount} row(s) of data into BigQuery`); }; @@ -158,3 +164,7 @@ export const timestampMissingValue = (fieldName: string) => { `Missing value for timestamp field: ${fieldName}, using default timestamp instead.` ); }; + +export const addDocumentIdColumn = (table) => { + console.log(`Updated '${table}' table with a 'document_id' column`); +}; From 5759cbc95f9c4ad59b1133a1a04a9ad28129cffa Mon Sep 17 00:00:00 2001 From: huangjeff5 <64040981+huangjeff5@users.noreply.github.com> Date: Thu, 30 Jul 2020 16:51:34 -0700 Subject: [PATCH 02/24] Bump firestore-bigquery-export version (#403) * Bump firestore-bigquery-export version * bump version on package.json --- firestore-bigquery-export/extension.yaml | 2 +- firestore-bigquery-export/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index f7e16dfb1..e845a482f 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-bigquery-export -version: 0.1.6 +version: 0.1.7 specVersion: v1beta displayName: Export Collections to BigQuery diff --git a/firestore-bigquery-export/package.json b/firestore-bigquery-export/package.json index 1e4ff49da..ede9a91bc 100644 --- a/firestore-bigquery-export/package.json +++ b/firestore-bigquery-export/package.json @@ -13,7 +13,7 @@ "author": "Jan Wyszynski ", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.6", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.10", "@google-cloud/bigquery": "^2.1.0", "@types/chai": "^4.1.6", "chai": "^4.2.0", From 13195e2fcc52c802bdb9415037ff293522ae2807 Mon Sep 17 00:00:00 2001 From: markarndt <50713862+markarndt@users.noreply.github.com> Date: Tue, 18 Aug 2020 10:07:40 -0700 Subject: [PATCH 03/24] chore(*): Update Cloud Firestore console deeplinks for three other extensions. (#392) Co-authored-by: Jeff Co-authored-by: Daniel Lee --- firestore-bigquery-export/POSTINSTALL.md | 4 ++-- firestore-shorten-urls-bitly/POSTINSTALL.md | 2 +- firestore-translate-text/POSTINSTALL.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/firestore-bigquery-export/POSTINSTALL.md b/firestore-bigquery-export/POSTINSTALL.md index 9434b21a3..2e82e7292 100644 --- a/firestore-bigquery-export/POSTINSTALL.md +++ b/firestore-bigquery-export/POSTINSTALL.md @@ -2,7 +2,7 @@ You can test out this extension right away! -1. Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data) in the Firebase console. +1. Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/firestore/data) in the Firebase console. 1. If it doesn't already exist, create the collection you specified during installation: `${param:COLLECTION_PATH}` @@ -24,7 +24,7 @@ You can test out this extension right away! FROM `${param:PROJECT_ID}.${param:DATASET_ID}.${param:TABLE_ID}_raw_latest` ``` -1. Delete the `bigquery-mirror-test` document from [Cloud Firestore](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data). +1. Delete the `bigquery-mirror-test` document from [Cloud Firestore](https://console.firebase.google.com/project/${param:PROJECT_ID}/firestore/data). The `bigquery-mirror-test` document will disappear from the **latest view** and a `DELETE` event will be added to the **raw changelog table**. 1. You can check the changelogs of a single document with this query: diff --git a/firestore-shorten-urls-bitly/POSTINSTALL.md b/firestore-shorten-urls-bitly/POSTINSTALL.md index 598736a5a..0a84c01d7 100644 --- a/firestore-shorten-urls-bitly/POSTINSTALL.md +++ b/firestore-shorten-urls-bitly/POSTINSTALL.md @@ -2,7 +2,7 @@ You can test out this extension right away! -1. Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data) in the Firebase console. +1. Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/firestore/data) in the Firebase console. 1. If it doesn't exist already, create a collection called `${param:COLLECTION_PATH}`. diff --git a/firestore-translate-text/POSTINSTALL.md b/firestore-translate-text/POSTINSTALL.md index 759d82791..7f379a332 100644 --- a/firestore-translate-text/POSTINSTALL.md +++ b/firestore-translate-text/POSTINSTALL.md @@ -2,7 +2,7 @@ You can test out this extension right away! -1. Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data) in the Firebase console. +1. Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/firestore/data) in the Firebase console. 1. If it doesn't exist already, create a collection called `${param:COLLECTION_PATH}`. From 8b59c05f2daec67a0315be03d35b7979b11eb24c Mon Sep 17 00:00:00 2001 From: markarndt <50713862+markarndt@users.noreply.github.com> Date: Tue, 18 Aug 2020 10:12:16 -0700 Subject: [PATCH 04/24] chore(firestore-send-email): Update console deep links for Cloud Firestore. (#390) Co-authored-by: Jeff Co-authored-by: Daniel Lee --- firestore-send-email/POSTINSTALL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firestore-send-email/POSTINSTALL.md b/firestore-send-email/POSTINSTALL.md index 904c49b2b..c9e65f321 100644 --- a/firestore-send-email/POSTINSTALL.md +++ b/firestore-send-email/POSTINSTALL.md @@ -2,7 +2,7 @@ You can test out this extension right away! -1. Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/database/firestore/data) in the Firebase console. +1. Go to your [Cloud Firestore dashboard](https://console.firebase.google.com/project/${param:PROJECT_ID}/firestore/data) in the Firebase console. 1. If it doesn't already exist, create the collection you specified during installation: `${param:MAIL_COLLECTION}`. @@ -132,7 +132,7 @@ There are instances in which email delivery fails in a recoverable fashion or th As a best practice, you can [monitor the activity](https://firebase.google.com/docs/extensions/manage-installed-extensions#monitor) of your installed extension, including checks on its health, usage, and logs. -[mail_collection]: https://console.firebase.google.com/project/_/database/firestore/data~2F${param:MAIL_COLLECTION} +[mail_collection]: https://console.firebase.google.com/project/_/firestore/data~2F${param:MAIL_COLLECTION} [admin_sdk]: https://firebase.google.com/docs/admin/setup [amp4email]: https://amp.dev/documentation/guides-and-tutorials/learn/email-spec/amp-email-format/ [handlebars]: https://handlebarsjs.com/ From dbeb5392e97921bb6d4907974ee418cb55cc31bd Mon Sep 17 00:00:00 2001 From: Russell Wheatley Date: Tue, 18 Aug 2020 18:15:52 +0100 Subject: [PATCH 05/24] docs(firestore-bigquery-export): add BigQuery NPM packages' README files (#420) --- .../firestore-bigquery-change-tracker/README.md | 4 ++++ firestore-bigquery-export/scripts/gen-schema-view/README.md | 6 ++++++ firestore-bigquery-export/scripts/import/README.md | 5 +++++ 3 files changed, 15 insertions(+) create mode 100644 firestore-bigquery-export/firestore-bigquery-change-tracker/README.md create mode 100644 firestore-bigquery-export/scripts/gen-schema-view/README.md create mode 100644 firestore-bigquery-export/scripts/import/README.md diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/README.md b/firestore-bigquery-export/firestore-bigquery-change-tracker/README.md new file mode 100644 index 000000000..cadbd70fc --- /dev/null +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/README.md @@ -0,0 +1,4 @@ +The `firestore-bigquery-change-tracker` package is a dependency for the official Firebase Extension [_Export Collections to BigQuery_](https://github.com/firebase/extensions/tree/master/firestore-bigquery-export), [_schema views script_](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md) & the [_import Firestore documents script_](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/IMPORT_EXISTING_DOCUMENTS.md). + +Its main purpose is to initialize & update the BigQuery table & view generated by using the `firestore-bigquery-export` extension. + diff --git a/firestore-bigquery-export/scripts/gen-schema-view/README.md b/firestore-bigquery-export/scripts/gen-schema-view/README.md new file mode 100644 index 000000000..89819c647 --- /dev/null +++ b/firestore-bigquery-export/scripts/gen-schema-view/README.md @@ -0,0 +1,6 @@ +The `fs-bq-schema-views` script is for use with the official Firebase Extension +[_Export Collections to BigQuery_](https://github.com/firebase/extensions/tree/master/firestore-bigquery-export). + +This script creates a BigQuery view based on a provided JSON schema configuration file. It queries the data from the `firestore-bigquery-export` extension changelog table to generate the view. + +A guide on how to use the script can be found [here](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md). \ No newline at end of file diff --git a/firestore-bigquery-export/scripts/import/README.md b/firestore-bigquery-export/scripts/import/README.md new file mode 100644 index 000000000..0f42cbd0d --- /dev/null +++ b/firestore-bigquery-export/scripts/import/README.md @@ -0,0 +1,5 @@ +The `fs-bq-import-collection` script is for use with the official Firebase Extension [_Export Collections to BiqQuery_](https://github.com/firebase/extensions/tree/master/firestore-bigquery-export). + +This script reads all existing documents in a specified Firestore Collection, and updates the changelog table used by the `firestore-bigquery-export` extension. + +A guide on how to use the script can be found [here](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/IMPORT_EXISTING_DOCUMENTS.md). \ No newline at end of file From 4c4f4b7316c10104b21bbbc270fce8e49ff2277f Mon Sep 17 00:00:00 2001 From: Russell Wheatley Date: Tue, 18 Aug 2020 18:16:20 +0100 Subject: [PATCH 06/24] test(firestore-bigquery-export): add Big Query sub-module tests (#280) --- jest.config.js | 4 ++-- lerna.json | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/jest.config.js b/jest.config.js index aacd86729..7dde9f78d 100644 --- a/jest.config.js +++ b/jest.config.js @@ -2,12 +2,12 @@ module.exports = { projects: [ "/*/jest.config.js", "/*/functions/jest.config.js", + "/firestore-bigquery-export/*/jest.config.js", + "/firestore-bigquery-export/scripts/*/jest.config.js", ], testPathIgnorePatterns: [ ".*/bin/", ".*/lib/", - // ignore until existing tests migrated - ".*/firestore-bigquery-export/", ".*/firestore-counter/", // Ignoring otherwise tests duplicate due to Jest `projects` ".*/__tests__/.*.ts", diff --git a/lerna.json b/lerna.json index f1d969ed5..5ba9c8df3 100644 --- a/lerna.json +++ b/lerna.json @@ -7,7 +7,8 @@ "auth-mailchimp-sync/functions", "firestore-send-email/functions", "firestore-counter/functions", - "delete-user-data/functions" + "delete-user-data/functions", + "firestore-bigquery-export/scripts/gen-schema-view" ], "version": "0.0.0" } From 2673ac8c32c4285e81ba98b970b3b649c3da5039 Mon Sep 17 00:00:00 2001 From: Russell Wheatley Date: Tue, 18 Aug 2020 18:27:25 +0100 Subject: [PATCH 07/24] feat(firestore-bigquery-export): Add validation regex for collection path parameter. (#418) --- firestore-bigquery-export/extension.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index e845a482f..9f420815a 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -97,6 +97,8 @@ params: collection (for example: `chatrooms/{chatid}/posts`). type: string example: posts + validationRegex: "^[^/]+(/[^/]+/[^/]+)*$" + validationErrorMessage: Firestore collection paths must be an odd number of segments separated by slashes, e.g. "path/to/collection". default: posts required: true From e30da83d7d1339c6571eff05c8ccf80f0ca861f5 Mon Sep 17 00:00:00 2001 From: Mike Diarmid Date: Tue, 18 Aug 2020 18:28:22 +0100 Subject: [PATCH 08/24] chore(*): disable patch status coverage reporting (#407) --- codecov.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codecov.yml b/codecov.yml index 79d8e2cbc..c290ab4c8 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,12 +3,13 @@ codecov: coverage: precision: 2 - round: down - range: "40...100" + round: up + range: "35...100" status: project: default: true + patch: off comment: layout: "reach, diff, files" From 8410baa9756491d98f57e1e03bd28a7653ddff19 Mon Sep 17 00:00:00 2001 From: Lauren Long Date: Tue, 18 Aug 2020 10:35:29 -0700 Subject: [PATCH 09/24] chore(*): Regenerate JS files --- auth-mailchimp-sync/functions/README.md | 2 ++ delete-user-data/extension.yaml | 1 + firestore-bigquery-export/README.md | 2 ++ firestore-bigquery-export/functions/lib/logs.js | 1 - firestore-bigquery-export/functions/lib/util.js | 1 - firestore-counter/functions/README.md | 2 ++ firestore-send-email/functions/README.md | 2 ++ firestore-shorten-urls-bitly/functions/README.md | 2 ++ firestore-translate-text/README.md | 2 ++ firestore-translate-text/functions/lib/index.js | 1 - firestore-translate-text/functions/lib/logs/index.js | 1 - firestore-translate-text/functions/lib/logs/messages.js | 1 - firestore-translate-text/functions/lib/validators.js | 1 - rtdb-limit-child-nodes/functions/README.md | 2 ++ storage-resize-images/README.md | 2 ++ 15 files changed, 17 insertions(+), 6 deletions(-) diff --git a/auth-mailchimp-sync/functions/README.md b/auth-mailchimp-sync/functions/README.md index 941dd5892..e35eab874 100644 --- a/auth-mailchimp-sync/functions/README.md +++ b/auth-mailchimp-sync/functions/README.md @@ -1,5 +1,7 @@ # Sync with Mailchimp +**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) + **Description**: Adds new users from Firebase Authentication to a specified Mailchimp audience. diff --git a/delete-user-data/extension.yaml b/delete-user-data/extension.yaml index e8c29b9ec..ae0eff466 100644 --- a/delete-user-data/extension.yaml +++ b/delete-user-data/extension.yaml @@ -63,6 +63,7 @@ resources: eventTrigger: eventType: providers/firebase.auth/eventTypes/user.delete resource: projects/${PROJECT_ID} + runtime: nodejs10 params: - param: LOCATION diff --git a/firestore-bigquery-export/README.md b/firestore-bigquery-export/README.md index 513405ed3..f04c552d6 100644 --- a/firestore-bigquery-export/README.md +++ b/firestore-bigquery-export/README.md @@ -1,5 +1,7 @@ # Export Collections to BigQuery +**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) + **Description**: Sends realtime, incremental updates from a specified Cloud Firestore collection to BigQuery. diff --git a/firestore-bigquery-export/functions/lib/logs.js b/firestore-bigquery-export/functions/lib/logs.js index 090409138..10abf8917 100644 --- a/firestore-bigquery-export/functions/lib/logs.js +++ b/firestore-bigquery-export/functions/lib/logs.js @@ -15,7 +15,6 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); -exports.timestampMissingValue = exports.start = exports.init = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0; const config_1 = require("./config"); exports.arrayFieldInvalid = (fieldName) => { console.warn(`Array field '${fieldName}' does not contain an array, skipping`); diff --git a/firestore-bigquery-export/functions/lib/util.js b/firestore-bigquery-export/functions/lib/util.js index fc29c0d8b..440e1709d 100644 --- a/firestore-bigquery-export/functions/lib/util.js +++ b/firestore-bigquery-export/functions/lib/util.js @@ -15,7 +15,6 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); -exports.getDocumentId = exports.getChangeType = void 0; const firestore_bigquery_change_tracker_1 = require("@firebaseextensions/firestore-bigquery-change-tracker"); function getChangeType(change) { if (!change.after.exists) { diff --git a/firestore-counter/functions/README.md b/firestore-counter/functions/README.md index a8fbe0aa9..60fbe3f28 100644 --- a/firestore-counter/functions/README.md +++ b/firestore-counter/functions/README.md @@ -1,5 +1,7 @@ # Distributed Counter +**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) + **Description**: Records event counters at scale to accommodate high-velocity writes to Cloud Firestore. diff --git a/firestore-send-email/functions/README.md b/firestore-send-email/functions/README.md index 5ecd26cf3..ec9dff2fd 100644 --- a/firestore-send-email/functions/README.md +++ b/firestore-send-email/functions/README.md @@ -1,5 +1,7 @@ # Trigger Email +**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) + **Description**: Composes and sends an email based on the contents of a document written to a specified Cloud Firestore collection. diff --git a/firestore-shorten-urls-bitly/functions/README.md b/firestore-shorten-urls-bitly/functions/README.md index d08a1a0ba..cf1b70abf 100644 --- a/firestore-shorten-urls-bitly/functions/README.md +++ b/firestore-shorten-urls-bitly/functions/README.md @@ -1,5 +1,7 @@ # Shorten URLs +**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) + **Description**: Shortens URLs written to a specified Cloud Firestore collection (uses Bitly). diff --git a/firestore-translate-text/README.md b/firestore-translate-text/README.md index 9fe9d5e97..198de98df 100644 --- a/firestore-translate-text/README.md +++ b/firestore-translate-text/README.md @@ -1,5 +1,7 @@ # Translate Text +**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) + **Description**: Translates strings written to a Cloud Firestore collection into multiple languages (uses Cloud Translation API). diff --git a/firestore-translate-text/functions/lib/index.js b/firestore-translate-text/functions/lib/index.js index 6ed701aaa..04c31f294 100644 --- a/firestore-translate-text/functions/lib/index.js +++ b/firestore-translate-text/functions/lib/index.js @@ -24,7 +24,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.fstranslate = void 0; const admin = require("firebase-admin"); const functions = require("firebase-functions"); const translate_1 = require("@google-cloud/translate"); diff --git a/firestore-translate-text/functions/lib/logs/index.js b/firestore-translate-text/functions/lib/logs/index.js index cd8220b71..1f3ce2786 100644 --- a/firestore-translate-text/functions/lib/logs/index.js +++ b/firestore-translate-text/functions/lib/logs/index.js @@ -15,7 +15,6 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); -exports.updateDocumentComplete = exports.updateDocument = exports.translateInputToAllLanguagesError = exports.translateInputToAllLanguagesComplete = exports.translateInputStringToAllLanguages = exports.translateStringError = exports.translateStringComplete = exports.translateInputString = exports.start = exports.inputFieldNameIsOutputPath = exports.init = exports.fieldNamesNotDifferent = exports.error = exports.documentUpdatedUnchangedInput = exports.documentUpdatedNoInput = exports.documentUpdatedDeletedInput = exports.documentUpdatedChangedInput = exports.documentDeleted = exports.documentCreatedWithInput = exports.documentCreatedNoInput = exports.complete = void 0; const config_1 = require("../config"); const messages_1 = require("./messages"); exports.complete = () => { diff --git a/firestore-translate-text/functions/lib/logs/messages.js b/firestore-translate-text/functions/lib/logs/messages.js index 5b25303c7..0cee416f5 100644 --- a/firestore-translate-text/functions/lib/logs/messages.js +++ b/firestore-translate-text/functions/lib/logs/messages.js @@ -1,6 +1,5 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.messages = void 0; exports.messages = { complete: () => "Completed execution of extension", documentCreatedNoInput: () => "Document was created without an input string, no processing is required", diff --git a/firestore-translate-text/functions/lib/validators.js b/firestore-translate-text/functions/lib/validators.js index e900c3520..ddc5469d2 100644 --- a/firestore-translate-text/functions/lib/validators.js +++ b/firestore-translate-text/functions/lib/validators.js @@ -15,7 +15,6 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); -exports.fieldNameIsTranslationPath = exports.fieldNamesMatch = void 0; exports.fieldNamesMatch = (field1, field2) => field1 === field2; exports.fieldNameIsTranslationPath = (inputFieldName, outputFieldName, languages) => { for (const language of languages) { diff --git a/rtdb-limit-child-nodes/functions/README.md b/rtdb-limit-child-nodes/functions/README.md index 45b32d9f5..e5fc12fca 100644 --- a/rtdb-limit-child-nodes/functions/README.md +++ b/rtdb-limit-child-nodes/functions/README.md @@ -1,5 +1,7 @@ # Limit Child Nodes +**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) + **Description**: Limits the number of nodes to a specified maximum count in a specified Realtime Database path. diff --git a/storage-resize-images/README.md b/storage-resize-images/README.md index acdb9c5c2..193205f7d 100644 --- a/storage-resize-images/README.md +++ b/storage-resize-images/README.md @@ -1,5 +1,7 @@ # Resize Images +**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) + **Description**: Resizes images uploaded to Cloud Storage to a specified size, and optionally keeps or deletes the original image. From 41123b2d41f6bfabd473fb6f51d661f1f34f4147 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 24 Aug 2020 17:10:04 +0100 Subject: [PATCH 10/24] chore(firestore-translate-text): migration to Node.js v10 (#413) --- firestore-translate-text/PREINSTALL.md | 13 ++-- firestore-translate-text/README.md | 13 ++-- firestore-translate-text/extension.yaml | 1 + .../functions/__tests__/config.test.ts | 12 +++- .../functions/__tests__/functions.test.ts | 57 +++++++---------- .../functions/__tests__/jest.setup.ts | 7 --- .../functions/__tests__/mocks/console.ts | 5 -- .../functions/__tests__/tsconfig.json | 3 +- .../functions/lib/index.js | 61 ++++++++----------- .../functions/lib/logs/index.js | 49 +++++++-------- .../functions/package.json | 2 +- .../functions/src/index.ts | 7 +-- .../functions/src/logs/index.ts | 48 +++++++-------- .../functions/tsconfig.json | 2 + 14 files changed, 127 insertions(+), 153 deletions(-) delete mode 100644 firestore-translate-text/functions/__tests__/mocks/console.ts diff --git a/firestore-translate-text/PREINSTALL.md b/firestore-translate-text/PREINSTALL.md index c21772c56..0b127b240 100644 --- a/firestore-translate-text/PREINSTALL.md +++ b/firestore-translate-text/PREINSTALL.md @@ -14,11 +14,10 @@ If the original non-translated field of the document is updated, then the transl Before installing this extension, make sure that you've [set up a Cloud Firestore database](https://firebase.google.com/docs/firestore/quickstart) in your Firebase project. #### Billing +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Translation API -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Translation API + - Cloud Firestore + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/firestore-translate-text/README.md b/firestore-translate-text/README.md index 198de98df..4ef1ae19b 100644 --- a/firestore-translate-text/README.md +++ b/firestore-translate-text/README.md @@ -22,14 +22,13 @@ If the original non-translated field of the document is updated, then the transl Before installing this extension, make sure that you've [set up a Cloud Firestore database](https://firebase.google.com/docs/firestore/quickstart) in your Firebase project. #### Billing +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Translation API -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Translation API + - Cloud Firestore + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/firestore-translate-text/extension.yaml b/firestore-translate-text/extension.yaml index 35ae8f42e..eb3ab2594 100644 --- a/firestore-translate-text/extension.yaml +++ b/firestore-translate-text/extension.yaml @@ -55,6 +55,7 @@ resources: then writes the translated strings back to the same document. properties: location: ${LOCATION} + runtime: nodejs10 eventTrigger: eventType: providers/cloud.firestore/eventTypes/document.write resource: projects/${PROJECT_ID}/databases/(default)/documents/${COLLECTION_PATH}/{messageId} diff --git a/firestore-translate-text/functions/__tests__/config.test.ts b/firestore-translate-text/functions/__tests__/config.test.ts index 6fad0c0fd..349ac46ac 100644 --- a/firestore-translate-text/functions/__tests__/config.test.ts +++ b/firestore-translate-text/functions/__tests__/config.test.ts @@ -19,11 +19,13 @@ const environment = { OUTPUT_FIELD_NAME: "translated", }; -const { mockConsoleLog, config } = global; +const { config } = global; functionsTestInit(); describe("extension config", () => { + let logMock; + beforeAll(() => { extensionYaml = yaml.safeLoad( readFileSync(pathResolve(__dirname, "../../extension.yaml"), "utf8") @@ -37,7 +39,11 @@ describe("extension config", () => { beforeEach(() => { restoreEnv = mockedEnv(environment); - mockConsoleLog.mockClear(); + logMock = jest.fn(); + + require("firebase-functions").logger = { + log: logMock, + }; }); afterEach(() => restoreEnv()); @@ -53,7 +59,7 @@ describe("extension config", () => { const functionsConfig = config(); - expect(mockConsoleLog).toBeCalledWith(...messages.init(functionsConfig)); + expect(logMock).toBeCalledWith(...messages.init(functionsConfig)); }); // LANGUAGES diff --git a/firestore-translate-text/functions/__tests__/functions.test.ts b/firestore-translate-text/functions/__tests__/functions.test.ts index bfdf24783..8166a36c7 100644 --- a/firestore-translate-text/functions/__tests__/functions.test.ts +++ b/firestore-translate-text/functions/__tests__/functions.test.ts @@ -20,8 +20,6 @@ const { mockTranslate, mockTranslateClassMethod, mockTranslateClass, - mockConsoleError, - mockConsoleLog, mockFirestoreUpdate, mockFirestoreTransaction, mockTranslateModule, @@ -45,6 +43,8 @@ describe("extension", () => { }); describe("functions.fstranslate", () => { + let logMock; + let errorLogMock; let admin; let wrappedMockTranslate; let beforeSnapshot; @@ -58,7 +58,8 @@ describe("extension", () => { functionsTest = functionsTestInit(); admin = require("firebase-admin"); wrappedMockTranslate = mockTranslate(); - + logMock = jest.fn(); + errorLogMock = jest.fn(); beforeSnapshot = snapshot({}); afterSnapshot = snapshot(); @@ -68,6 +69,11 @@ describe("extension", () => { mockDocumentSnapshotFactory(afterSnapshot) ); admin.firestore().runTransaction = mockFirestoreTransaction(); + + require("firebase-functions").logger = { + log: logMock, + error: errorLogMock, + }; }); test("initializes Google Translation API with PROJECT_ID on function load", () => { @@ -87,9 +93,7 @@ describe("extension", () => { const callResult = await wrappedMockTranslate(documentChange); expect(callResult).toBeUndefined(); - expect(mockConsoleLog).toHaveBeenCalledWith( - "Document was deleted, no processing is required" - ); + expect(logMock).toHaveBeenCalledWith(messages.documentDeleted()); expect(mockTranslateClassMethod).not.toHaveBeenCalled(); expect(mockFirestoreUpdate).not.toHaveBeenCalled(); @@ -107,8 +111,8 @@ describe("extension", () => { const callResult = await wrappedMockTranslate(documentChange); expect(callResult).toBeUndefined(); - expect(mockConsoleLog).toHaveBeenCalledWith( - "Document was updated, input string has not changed, no processing is required" + expect(logMock).toHaveBeenCalledWith( + expect.stringContaining(messages.documentUpdatedUnchangedInput()) ); expect(mockTranslateClassMethod).not.toHaveBeenCalled(); @@ -126,18 +130,13 @@ describe("extension", () => { expect(callResult).toBeUndefined(); - expect(mockConsoleLog).toHaveBeenCalledWith( - "Document was created without an input string, no processing is required" - ); + expect(logMock).toHaveBeenCalledWith(messages.documentCreatedNoInput()); expect(mockTranslateClassMethod).not.toHaveBeenCalled(); expect(mockFirestoreUpdate).not.toHaveBeenCalled(); }); test("function exits early if input & output fields are the same", async () => { - // reset modules again - jest.resetModules(); - // so ENV variables can be reset restoreEnv = mockedEnv({ ...defaultEnvironment, INPUT_FIELD_NAME: "input", @@ -149,15 +148,9 @@ describe("extension", () => { const callResult = await wrappedMockTranslate(documentChange); expect(callResult).toBeUndefined(); - expect(mockConsoleError).toHaveBeenCalledWith( - "The `Input` and `Output` field names must be different for this extension to function correctly" - ); }); test("function exits early if input field is a translation output path", async () => { - // reset modules again - jest.resetModules(); - // so ENV variables can be reset restoreEnv = mockedEnv({ ...defaultEnvironment, INPUT_FIELD_NAME: "output.en", @@ -167,11 +160,7 @@ describe("extension", () => { wrappedMockTranslate = mockTranslate(); const callResult = await wrappedMockTranslate(documentChange); - expect(callResult).toBeUndefined(); - expect(mockConsoleError).toHaveBeenCalledWith( - "The `Input` field name must not be the same as an `Output` path for this extension to function correctly" - ); }); test("function updates translation document with translations", async () => { @@ -192,23 +181,23 @@ describe("extension", () => { // confirm logs were printed Object.keys(testTranslations).forEach((language) => { // logs.translateInputString - expect(mockConsoleLog).toHaveBeenCalledWith( + expect(logMock).toHaveBeenCalledWith( messages.translateInputString("hello", language) ); // logs.translateStringComplete - expect(mockConsoleLog).toHaveBeenCalledWith( + expect(logMock).toHaveBeenCalledWith( messages.translateStringComplete("hello", language) ); }); // logs.translateInputStringToAllLanguages - expect(mockConsoleLog).toHaveBeenCalledWith( + expect(logMock).toHaveBeenCalledWith( messages.translateInputStringToAllLanguages( "hello", defaultEnvironment.LANGUAGES.split(",") ) ); // logs.translateInputToAllLanguagesComplete - expect(mockConsoleLog).toHaveBeenCalledWith( + expect(logMock).toHaveBeenCalledWith( messages.translateInputToAllLanguagesComplete("hello") ); }); @@ -223,7 +212,7 @@ describe("extension", () => { await wrappedMockTranslate(documentChange); // logs.documentUpdatedChangedInput - expect(mockConsoleLog).toHaveBeenCalledWith( + expect(logMock).toHaveBeenCalledWith( messages.documentUpdatedChangedInput() ); @@ -258,7 +247,7 @@ describe("extension", () => { ); // logs.documentUpdatedDeletedInput - expect(mockConsoleLog).toHaveBeenCalledWith( + expect(logMock).toHaveBeenCalledWith( messages.documentUpdatedDeletedInput() ); }); @@ -279,9 +268,7 @@ describe("extension", () => { expect(mockTranslateClassMethod).not.toHaveBeenCalled(); // logs.documentUpdatedNoInput - expect(mockConsoleLog).toHaveBeenCalledWith( - messages.documentUpdatedNoInput() - ); + expect(logMock).toHaveBeenCalledWith(messages.documentUpdatedNoInput()); }); test("function handles Google Translation API errors", async () => { @@ -293,12 +280,12 @@ describe("extension", () => { await wrappedMockTranslate(documentChange); // logs.translateStringError - expect(mockConsoleError).toHaveBeenCalledWith( + expect(errorLogMock).toHaveBeenCalledWith( ...messages.translateStringError("hello", "en", error) ); // logs.translateInputToAllLanguagesError - expect(mockConsoleError).toHaveBeenCalledWith( + expect(errorLogMock).toHaveBeenCalledWith( ...messages.translateInputToAllLanguagesError("hello", error) ); }); diff --git a/firestore-translate-text/functions/__tests__/jest.setup.ts b/firestore-translate-text/functions/__tests__/jest.setup.ts index 868f884d9..0a4d905be 100644 --- a/firestore-translate-text/functions/__tests__/jest.setup.ts +++ b/firestore-translate-text/functions/__tests__/jest.setup.ts @@ -4,7 +4,6 @@ import { mockFirestoreUpdate, mockFirestoreTransaction, } from "./mocks/firestore"; -import { mockConsoleError, mockConsoleLog } from "./mocks/console"; import { testTranslations, mockTranslate, @@ -30,10 +29,6 @@ global.mockTranslateClass = mockTranslateClass; global.mockTranslateModule = () => jest.mock("@google-cloud/translate", mockTranslateModuleFactory); -global.mockConsoleError = mockConsoleError; - -global.mockConsoleLog = mockConsoleLog; - global.mockFirestoreUpdate = mockFirestoreUpdate; global.mockFirestoreTransaction = mockFirestoreTransaction; @@ -41,6 +36,4 @@ global.mockFirestoreTransaction = mockFirestoreTransaction; global.clearMocks = () => { mockFirestoreUpdate.mockClear(); mockTranslateClassMethod.mockClear(); - mockConsoleLog.mockClear(); - mockConsoleError.mockClear(); }; diff --git a/firestore-translate-text/functions/__tests__/mocks/console.ts b/firestore-translate-text/functions/__tests__/mocks/console.ts deleted file mode 100644 index e811ff751..000000000 --- a/firestore-translate-text/functions/__tests__/mocks/console.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const mockConsoleLog = jest.spyOn(console, "log").mockImplementation(); - -export const mockConsoleError = jest - .spyOn(console, "error") - .mockImplementation(); diff --git a/firestore-translate-text/functions/__tests__/tsconfig.json b/firestore-translate-text/functions/__tests__/tsconfig.json index 5f5aa2da6..c599a00c2 100644 --- a/firestore-translate-text/functions/__tests__/tsconfig.json +++ b/firestore-translate-text/functions/__tests__/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../tsconfig.json", "compilerOptions": { - "outDir": "lib" + "outDir": "lib", + "target": "es2018" }, "files": ["test.types.d.ts"], "include": ["**/*"] diff --git a/firestore-translate-text/functions/lib/index.js b/firestore-translate-text/functions/lib/index.js index 04c31f294..aaceec6bf 100644 --- a/firestore-translate-text/functions/lib/index.js +++ b/firestore-translate-text/functions/lib/index.js @@ -14,15 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); const admin = require("firebase-admin"); const functions = require("firebase-functions"); @@ -39,9 +30,9 @@ var ChangeType; const translate = new translate_1.Translate({ projectId: process.env.PROJECT_ID }); // Initialize the Firebase Admin SDK admin.initializeApp(); -logs.init(); -exports.fstranslate = functions.handler.firestore.document.onWrite((change) => __awaiter(void 0, void 0, void 0, function* () { - logs.start(); +logs.init(config_1.default); +exports.fstranslate = functions.handler.firestore.document.onWrite(async (change) => { + logs.start(config_1.default); const { languages, inputFieldName, outputFieldName } = config_1.default; if (validators.fieldNamesMatch(inputFieldName, outputFieldName)) { logs.fieldNamesNotDifferent(); @@ -55,13 +46,13 @@ exports.fstranslate = functions.handler.firestore.document.onWrite((change) => _ try { switch (changeType) { case ChangeType.CREATE: - yield handleCreateDocument(change.after); + await handleCreateDocument(change.after); break; case ChangeType.DELETE: handleDeleteDocument(); break; case ChangeType.UPDATE: - yield handleUpdateDocument(change.before, change.after); + await handleUpdateDocument(change.before, change.after); break; } logs.complete(); @@ -69,7 +60,7 @@ exports.fstranslate = functions.handler.firestore.document.onWrite((change) => _ catch (err) { logs.error(err); } -})); +}); const extractInput = (snapshot) => { return snapshot.get(config_1.default.inputFieldName); }; @@ -82,20 +73,20 @@ const getChangeType = (change) => { } return ChangeType.UPDATE; }; -const handleCreateDocument = (snapshot) => __awaiter(void 0, void 0, void 0, function* () { +const handleCreateDocument = async (snapshot) => { const input = extractInput(snapshot); if (input) { logs.documentCreatedWithInput(); - yield translateDocument(snapshot); + await translateDocument(snapshot); } else { logs.documentCreatedNoInput(); } -}); +}; const handleDeleteDocument = () => { logs.documentDeleted(); }; -const handleUpdateDocument = (before, after) => __awaiter(void 0, void 0, void 0, function* () { +const handleUpdateDocument = async (before, after) => { const inputAfter = extractInput(after); const inputBefore = extractInput(before); const inputHasChanged = inputAfter !== inputBefore; @@ -107,43 +98,43 @@ const handleUpdateDocument = (before, after) => __awaiter(void 0, void 0, void 0 } if (inputAfter) { logs.documentUpdatedChangedInput(); - yield translateDocument(after); + await translateDocument(after); } else if (inputBefore) { logs.documentUpdatedDeletedInput(); - yield updateTranslations(after, admin.firestore.FieldValue.delete()); + await updateTranslations(after, admin.firestore.FieldValue.delete()); } else { logs.documentUpdatedNoInput(); } -}); -const translateDocument = (snapshot) => __awaiter(void 0, void 0, void 0, function* () { +}; +const translateDocument = async (snapshot) => { const input = extractInput(snapshot); logs.translateInputStringToAllLanguages(input, config_1.default.languages); - const tasks = config_1.default.languages.map((targetLanguage) => __awaiter(void 0, void 0, void 0, function* () { + const tasks = config_1.default.languages.map(async (targetLanguage) => { return { language: targetLanguage, - output: yield translateString(input, targetLanguage), + output: await translateString(input, targetLanguage), }; - })); + }); try { - const translations = yield Promise.all(tasks); + const translations = await Promise.all(tasks); logs.translateInputToAllLanguagesComplete(input); const translationsMap = translations.reduce((output, translation) => { output[translation.language] = translation.output; return output; }, {}); - yield updateTranslations(snapshot, translationsMap); + await updateTranslations(snapshot, translationsMap); } catch (err) { logs.translateInputToAllLanguagesError(input, err); throw err; } -}); -const translateString = (string, targetLanguage) => __awaiter(void 0, void 0, void 0, function* () { +}; +const translateString = async (string, targetLanguage) => { try { logs.translateInputString(string, targetLanguage); - const [translatedString] = yield translate.translate(string, targetLanguage); + const [translatedString] = await translate.translate(string, targetLanguage); logs.translateStringComplete(string, targetLanguage); return translatedString; } @@ -151,13 +142,13 @@ const translateString = (string, targetLanguage) => __awaiter(void 0, void 0, vo logs.translateStringError(string, targetLanguage, err); throw err; } -}); -const updateTranslations = (snapshot, translations) => __awaiter(void 0, void 0, void 0, function* () { +}; +const updateTranslations = async (snapshot, translations) => { logs.updateDocument(snapshot.ref.path); // Wrapping in transaction to allow for automatic retries (#48) - yield admin.firestore().runTransaction((transaction) => { + await admin.firestore().runTransaction((transaction) => { transaction.update(snapshot.ref, config_1.default.outputFieldName, translations); return Promise.resolve(); }); logs.updateDocumentComplete(snapshot.ref.path); -}); +}; diff --git a/firestore-translate-text/functions/lib/logs/index.js b/firestore-translate-text/functions/lib/logs/index.js index 1f3ce2786..99ca0ecc9 100644 --- a/firestore-translate-text/functions/lib/logs/index.js +++ b/firestore-translate-text/functions/lib/logs/index.js @@ -15,68 +15,69 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); -const config_1 = require("../config"); +exports.updateDocumentComplete = exports.updateDocument = exports.translateInputToAllLanguagesError = exports.translateInputToAllLanguagesComplete = exports.translateInputStringToAllLanguages = exports.translateStringError = exports.translateStringComplete = exports.translateInputString = exports.start = exports.inputFieldNameIsOutputPath = exports.init = exports.fieldNamesNotDifferent = exports.error = exports.documentUpdatedUnchangedInput = exports.documentUpdatedNoInput = exports.documentUpdatedDeletedInput = exports.documentUpdatedChangedInput = exports.documentDeleted = exports.documentCreatedWithInput = exports.documentCreatedNoInput = exports.complete = void 0; +const firebase_functions_1 = require("firebase-functions"); const messages_1 = require("./messages"); exports.complete = () => { - console.log(messages_1.messages.complete()); + firebase_functions_1.logger.log(messages_1.messages.complete()); }; exports.documentCreatedNoInput = () => { - console.log(messages_1.messages.documentCreatedNoInput()); + firebase_functions_1.logger.log(messages_1.messages.documentCreatedNoInput()); }; exports.documentCreatedWithInput = () => { - console.log(messages_1.messages.documentCreatedWithInput()); + firebase_functions_1.logger.log(messages_1.messages.documentCreatedWithInput()); }; exports.documentDeleted = () => { - console.log(messages_1.messages.documentDeleted()); + firebase_functions_1.logger.log(messages_1.messages.documentDeleted()); }; exports.documentUpdatedChangedInput = () => { - console.log(messages_1.messages.documentUpdatedChangedInput()); + firebase_functions_1.logger.log(messages_1.messages.documentUpdatedChangedInput()); }; exports.documentUpdatedDeletedInput = () => { - console.log(messages_1.messages.documentUpdatedDeletedInput()); + firebase_functions_1.logger.log(messages_1.messages.documentUpdatedDeletedInput()); }; exports.documentUpdatedNoInput = () => { - console.log(messages_1.messages.documentUpdatedNoInput()); + firebase_functions_1.logger.log(messages_1.messages.documentUpdatedNoInput()); }; exports.documentUpdatedUnchangedInput = () => { - console.log(messages_1.messages.documentUpdatedUnchangedInput()); + firebase_functions_1.logger.log(messages_1.messages.documentUpdatedUnchangedInput()); }; exports.error = (err) => { - console.error(...messages_1.messages.error(err)); + firebase_functions_1.logger.error(...messages_1.messages.error(err)); }; exports.fieldNamesNotDifferent = () => { - console.error(messages_1.messages.fieldNamesNotDifferent()); + firebase_functions_1.logger.error(messages_1.messages.fieldNamesNotDifferent()); }; -exports.init = () => { - console.log(...messages_1.messages.init(config_1.default)); +exports.init = (config) => { + firebase_functions_1.logger.log(...messages_1.messages.init(config)); }; exports.inputFieldNameIsOutputPath = () => { - console.error(messages_1.messages.inputFieldNameIsOutputPath()); + firebase_functions_1.logger.error(messages_1.messages.inputFieldNameIsOutputPath()); }; -exports.start = () => { - console.log(...messages_1.messages.start(config_1.default)); +exports.start = (config) => { + firebase_functions_1.logger.log(...messages_1.messages.start(config)); }; exports.translateInputString = (string, language) => { - console.log(messages_1.messages.translateInputString(string, language)); + firebase_functions_1.logger.log(messages_1.messages.translateInputString(string, language)); }; exports.translateStringComplete = (string, language) => { - console.log(messages_1.messages.translateStringComplete(string, language)); + firebase_functions_1.logger.log(messages_1.messages.translateStringComplete(string, language)); }; exports.translateStringError = (string, language, err) => { - console.error(...messages_1.messages.translateStringError(string, language, err)); + firebase_functions_1.logger.error(...messages_1.messages.translateStringError(string, language, err)); }; exports.translateInputStringToAllLanguages = (string, languages) => { - console.log(messages_1.messages.translateInputStringToAllLanguages(string, languages)); + firebase_functions_1.logger.log(messages_1.messages.translateInputStringToAllLanguages(string, languages)); }; exports.translateInputToAllLanguagesComplete = (string) => { - console.log(messages_1.messages.translateInputToAllLanguagesComplete(string)); + firebase_functions_1.logger.log(messages_1.messages.translateInputToAllLanguagesComplete(string)); }; exports.translateInputToAllLanguagesError = (string, err) => { - console.error(...messages_1.messages.translateInputToAllLanguagesError(string, err)); + firebase_functions_1.logger.error(...messages_1.messages.translateInputToAllLanguagesError(string, err)); }; exports.updateDocument = (path) => { - console.log(...messages_1.messages.updateDocument(path)); + firebase_functions_1.logger.log(messages_1.messages.updateDocument(path)); }; exports.updateDocumentComplete = (path) => { - console.log(...messages_1.messages.updateDocumentComplete(path)); + firebase_functions_1.logger.log(messages_1.messages.updateDocumentComplete(path)); }; diff --git a/firestore-translate-text/functions/package.json b/firestore-translate-text/functions/package.json index 2c363cb5a..16aff0d5e 100644 --- a/firestore-translate-text/functions/package.json +++ b/firestore-translate-text/functions/package.json @@ -13,7 +13,7 @@ "dependencies": { "@google-cloud/translate": "^4.0.1", "firebase-admin": "^8.0.0", - "firebase-functions": "^3.3.0" + "firebase-functions": "^3.7.0" }, "devDependencies": { "firebase-functions-test": "^0.1.7", diff --git a/firestore-translate-text/functions/src/index.ts b/firestore-translate-text/functions/src/index.ts index 3793b054e..ea1f16965 100644 --- a/firestore-translate-text/functions/src/index.ts +++ b/firestore-translate-text/functions/src/index.ts @@ -38,19 +38,17 @@ const translate = new Translate({ projectId: process.env.PROJECT_ID }); // Initialize the Firebase Admin SDK admin.initializeApp(); -logs.init(); +logs.init(config); export const fstranslate = functions.handler.firestore.document.onWrite( async (change): Promise => { - logs.start(); - + logs.start(config); const { languages, inputFieldName, outputFieldName } = config; if (validators.fieldNamesMatch(inputFieldName, outputFieldName)) { logs.fieldNamesNotDifferent(); return; } - if ( validators.fieldNameIsTranslationPath( inputFieldName, @@ -124,6 +122,7 @@ const handleUpdateDocument = async ( const inputBefore = extractInput(before); const inputHasChanged = inputAfter !== inputBefore; + if ( !inputHasChanged && inputAfter !== undefined && diff --git a/firestore-translate-text/functions/src/logs/index.ts b/firestore-translate-text/functions/src/logs/index.ts index 261b4381f..d8d68810a 100644 --- a/firestore-translate-text/functions/src/logs/index.ts +++ b/firestore-translate-text/functions/src/logs/index.ts @@ -14,67 +14,67 @@ * limitations under the License. */ -import config from "../config"; +import { logger } from "firebase-functions"; import { messages } from "./messages"; export const complete = () => { - console.log(messages.complete()); + logger.log(messages.complete()); }; export const documentCreatedNoInput = () => { - console.log(messages.documentCreatedNoInput()); + logger.log(messages.documentCreatedNoInput()); }; export const documentCreatedWithInput = () => { - console.log(messages.documentCreatedWithInput()); + logger.log(messages.documentCreatedWithInput()); }; export const documentDeleted = () => { - console.log(messages.documentDeleted()); + logger.log(messages.documentDeleted()); }; export const documentUpdatedChangedInput = () => { - console.log(messages.documentUpdatedChangedInput()); + logger.log(messages.documentUpdatedChangedInput()); }; export const documentUpdatedDeletedInput = () => { - console.log(messages.documentUpdatedDeletedInput()); + logger.log(messages.documentUpdatedDeletedInput()); }; export const documentUpdatedNoInput = () => { - console.log(messages.documentUpdatedNoInput()); + logger.log(messages.documentUpdatedNoInput()); }; export const documentUpdatedUnchangedInput = () => { - console.log(messages.documentUpdatedUnchangedInput()); + logger.log(messages.documentUpdatedUnchangedInput()); }; export const error = (err: Error) => { - console.error(...messages.error(err)); + logger.error(...messages.error(err)); }; export const fieldNamesNotDifferent = () => { - console.error(messages.fieldNamesNotDifferent()); + logger.error(messages.fieldNamesNotDifferent()); }; -export const init = () => { - console.log(...messages.init(config)); +export const init = (config) => { + logger.log(...messages.init(config)); }; export const inputFieldNameIsOutputPath = () => { - console.error(messages.inputFieldNameIsOutputPath()); + logger.error(messages.inputFieldNameIsOutputPath()); }; -export const start = () => { - console.log(...messages.start(config)); +export const start = (config) => { + logger.log(...messages.start(config)); }; export const translateInputString = (string: string, language: string) => { - console.log(messages.translateInputString(string, language)); + logger.log(messages.translateInputString(string, language)); }; export const translateStringComplete = (string: string, language: string) => { - console.log(messages.translateStringComplete(string, language)); + logger.log(messages.translateStringComplete(string, language)); }; export const translateStringError = ( @@ -82,31 +82,31 @@ export const translateStringError = ( language: string, err: Error ) => { - console.error(...messages.translateStringError(string, language, err)); + logger.error(...messages.translateStringError(string, language, err)); }; export const translateInputStringToAllLanguages = ( string: string, languages: string[] ) => { - console.log(messages.translateInputStringToAllLanguages(string, languages)); + logger.log(messages.translateInputStringToAllLanguages(string, languages)); }; export const translateInputToAllLanguagesComplete = (string: string) => { - console.log(messages.translateInputToAllLanguagesComplete(string)); + logger.log(messages.translateInputToAllLanguagesComplete(string)); }; export const translateInputToAllLanguagesError = ( string: string, err: Error ) => { - console.error(...messages.translateInputToAllLanguagesError(string, err)); + logger.error(...messages.translateInputToAllLanguagesError(string, err)); }; export const updateDocument = (path: string) => { - console.log(...messages.updateDocument(path)); + logger.log(messages.updateDocument(path)); }; export const updateDocumentComplete = (path: string) => { - console.log(...messages.updateDocumentComplete(path)); + logger.log(messages.updateDocumentComplete(path)); }; diff --git a/firestore-translate-text/functions/tsconfig.json b/firestore-translate-text/functions/tsconfig.json index 1b551f4e9..14b97491f 100644 --- a/firestore-translate-text/functions/tsconfig.json +++ b/firestore-translate-text/functions/tsconfig.json @@ -1,6 +1,8 @@ { "extends": "../../tsconfig.json", "compilerOptions": { + "target": "es2018", + "lib": ["es2018"], "outDir": "lib" }, "include": ["src"] From 4e94f0f2626108f9e8c7128ef491beaed2246173 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 24 Aug 2020 17:32:39 +0100 Subject: [PATCH 11/24] chore(firestore-bigquery-change-export): Added node 10 updates (#436) --- firestore-bigquery-export/PREINSTALL.md | 16 ++--- firestore-bigquery-export/README.md | 17 +++-- firestore-bigquery-export/extension.yaml | 1 + .../src/logs.ts | 70 +++++++++---------- .../functions/lib/index.js | 15 +--- .../functions/lib/logs.js | 1 + .../functions/lib/util.js | 1 + firestore-bigquery-export/package.json | 2 +- firestore-bigquery-export/tsconfig.json | 3 +- 9 files changed, 60 insertions(+), 66 deletions(-) diff --git a/firestore-bigquery-export/PREINSTALL.md b/firestore-bigquery-export/PREINSTALL.md index 6beb3eff0..23fe30dc9 100644 --- a/firestore-bigquery-export/PREINSTALL.md +++ b/firestore-bigquery-export/PREINSTALL.md @@ -27,11 +27,11 @@ This extension only sends the content of documents that have been changed -- it After your data is in BigQuery, you can run the [schema-views script](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md) (provided by this extension) to create views that make it easier to query relevant data. You only need to provide a JSON schema file that describes your data structure, and the schema-views script will create the views. #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- BigQuery (this extension writes to BigQuery with [streaming inserts](https://cloud.google.com/bigquery/pricing#streaming_pricing)) -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - BigQuery (this extension writes to BigQuery with [streaming inserts](https://cloud.google.com/bigquery/pricing#streaming_pricing)) + - Cloud Firestore + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) \ No newline at end of file diff --git a/firestore-bigquery-export/README.md b/firestore-bigquery-export/README.md index f04c552d6..82baf34af 100644 --- a/firestore-bigquery-export/README.md +++ b/firestore-bigquery-export/README.md @@ -35,15 +35,14 @@ This extension only sends the content of documents that have been changed -- it After your data is in BigQuery, you can run the [schema-views script](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md) (provided by this extension) to create views that make it easier to query relevant data. You only need to provide a JSON schema file that describes your data structure, and the schema-views script will create the views. #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- BigQuery (this extension writes to BigQuery with [streaming inserts](https://cloud.google.com/bigquery/pricing#streaming_pricing)) -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) - + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - BigQuery (this extension writes to BigQuery with [streaming inserts](https://cloud.google.com/bigquery/pricing#streaming_pricing)) + - Cloud Firestore + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index 9f420815a..3211e610a 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -54,6 +54,7 @@ resources: properties: sourceDirectory: . location: ${LOCATION} + runtime: nodejs10 eventTrigger: eventType: providers/cloud.firestore/eventTypes/document.write resource: projects/${PROJECT_ID}/databases/(default)/documents/${COLLECTION_PATH}/{documentId} diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts index abb3fb6f9..be1f260c9 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/logs.ts @@ -14,135 +14,135 @@ * limitations under the License. */ +import { logger } from "firebase-functions"; + export const arrayFieldInvalid = (fieldName: string) => { - console.warn( - `Array field '${fieldName}' does not contain an array, skipping` - ); + logger.warn(`Array field '${fieldName}' does not contain an array, skipping`); }; export const bigQueryDatasetCreated = (datasetId: string) => { - console.log(`Created BigQuery dataset: ${datasetId}`); + logger.log(`Created BigQuery dataset: ${datasetId}`); }; export const bigQueryDatasetCreating = (datasetId: string) => { - console.log(`Creating BigQuery dataset: ${datasetId}`); + logger.log(`Creating BigQuery dataset: ${datasetId}`); }; export const bigQueryDatasetExists = (datasetId: string) => { - console.log(`BigQuery dataset already exists: ${datasetId}`); + logger.log(`BigQuery dataset already exists: ${datasetId}`); }; export const bigQueryErrorRecordingDocumentChange = (e: Error) => { - console.error(`Error recording document changes.`, e); + logger.error(`Error recording document changes.`, e); }; export const bigQueryLatestSnapshotViewQueryCreated = (query: string) => { - console.log(`BigQuery latest snapshot view query:\n${query}`); + logger.log(`BigQuery latest snapshot view query:\n${query}`); }; export const bigQuerySchemaViewCreated = (name: string) => { - console.log(`BigQuery created schema view ${name}\n`); + logger.log(`BigQuery created schema view ${name}\n`); }; export const bigQueryTableAlreadyExists = ( tableName: string, datasetName: string ) => { - console.log( + logger.log( `BigQuery table with name ${tableName} already ` + `exists in dataset ${datasetName}!` ); }; export const bigQueryTableCreated = (tableName: string) => { - console.log(`Created BigQuery table: ${tableName}`); + logger.log(`Created BigQuery table: ${tableName}`); }; export const bigQueryTableCreating = (tableName: string) => { - console.log(`Creating BigQuery table: ${tableName}`); + logger.log(`Creating BigQuery table: ${tableName}`); }; export const bigQueryTableUpdated = (tableName: string) => { - console.log(`Updated existing BigQuery table: ${tableName}`); + logger.log(`Updated existing BigQuery table: ${tableName}`); }; export const bigQueryTableUpdating = (tableName: string) => { - console.log(`Updating existing BigQuery table: ${tableName}`); + logger.log(`Updating existing BigQuery table: ${tableName}`); }; export const bigQueryTableUpToDate = (tableName: string) => { - console.log(`BigQuery table: ${tableName} is up to date`); + logger.log(`BigQuery table: ${tableName} is up to date`); }; export const bigQueryTableValidated = (tableName: string) => { - console.log(`Validated existing BigQuery table: ${tableName}`); + logger.log(`Validated existing BigQuery table: ${tableName}`); }; export const bigQueryTableValidating = (tableName: string) => { - console.log(`Validating existing BigQuery table: ${tableName}`); + logger.log(`Validating existing BigQuery table: ${tableName}`); }; export const bigQueryUserDefinedFunctionCreating = (functionName: string) => { - console.log(`Creating BigQuery user-defined function ${functionName}`); + logger.log(`Creating BigQuery user-defined function ${functionName}`); }; export const bigQueryUserDefinedFunctionCreated = (functionName: string) => { - console.log(`Created BigQuery user-defined function ${functionName}`); + logger.log(`Created BigQuery user-defined function ${functionName}`); }; export const bigQueryViewCreated = (viewName: string) => { - console.log(`Created BigQuery view: ${viewName}`); + logger.log(`Created BigQuery view: ${viewName}`); }; export const bigQueryViewCreating = (viewName: string, query: string) => { - console.log(`Creating BigQuery view: ${viewName}\nQuery:\n${query}`); + logger.log(`Creating BigQuery view: ${viewName}\nQuery:\n${query}`); }; export const bigQueryViewAlreadyExists = ( viewName: string, datasetName: string ) => { - console.log( + logger.log( `View with id ${viewName} already exists in dataset ${datasetName}.` ); }; export const bigQueryViewUpdated = (viewName: string) => { - console.log(`Updated existing BigQuery view: ${viewName}`); + logger.log(`Updated existing BigQuery view: ${viewName}`); }; export const bigQueryViewUpdating = (viewName: string) => { - console.log(`Updating existing BigQuery view: ${viewName}`); + logger.log(`Updating existing BigQuery view: ${viewName}`); }; export const bigQueryViewUpToDate = (viewName: string) => { - console.log(`BigQuery view: ${viewName} is up to date`); + logger.log(`BigQuery view: ${viewName} is up to date`); }; export const bigQueryViewValidated = (viewName: string) => { - console.log(`Validated existing BigQuery view: ${viewName}`); + logger.log(`Validated existing BigQuery view: ${viewName}`); }; export const bigQueryViewValidating = (viewName: string) => { - console.log(`Validating existing BigQuery view: ${viewName}`); + logger.log(`Validating existing BigQuery view: ${viewName}`); }; export const complete = () => { - console.log("Completed mod execution"); + logger.log("Completed mod execution"); }; export const dataInserted = (rowCount: number) => { - console.log(`Inserted ${rowCount} row(s) of data into BigQuery`); + logger.log(`Inserted ${rowCount} row(s) of data into BigQuery`); }; export const dataInsertRetried = (rowCount: number) => { - console.log( + logger.log( `Retried to insert ${rowCount} row(s) of data into BigQuery (ignoring uknown columns)` ); }; export const dataInserting = (rowCount: number) => { - console.log(`Inserting ${rowCount} row(s) of data into BigQuery`); + logger.log(`Inserting ${rowCount} row(s) of data into BigQuery`); }; export const dataTypeInvalid = ( @@ -150,21 +150,21 @@ export const dataTypeInvalid = ( fieldType: string, dataType: string ) => { - console.warn( + logger.warn( `Field '${fieldName}' has invalid data. Expected: ${fieldType}, received: ${dataType}` ); }; export const error = (err: Error) => { - console.error("Error when mirroring data to BigQuery", err); + logger.error("Error when mirroring data to BigQuery", err); }; export const timestampMissingValue = (fieldName: string) => { - console.warn( + logger.warn( `Missing value for timestamp field: ${fieldName}, using default timestamp instead.` ); }; export const addDocumentIdColumn = (table) => { - console.log(`Updated '${table}' table with a 'document_id' column`); + logger.log(`Updated '${table}' table with a 'document_id' column`); }; diff --git a/firestore-bigquery-export/functions/lib/index.js b/firestore-bigquery-export/functions/lib/index.js index 653f6e224..019d25d85 100644 --- a/firestore-bigquery-export/functions/lib/index.js +++ b/firestore-bigquery-export/functions/lib/index.js @@ -14,15 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); const config_1 = require("./config"); const functions = require("firebase-functions"); @@ -34,12 +25,12 @@ const eventTracker = new firestore_bigquery_change_tracker_1.FirestoreBigQueryEv datasetId: config_1.default.datasetId, }); logs.init(); -exports.fsexportbigquery = functions.handler.firestore.document.onWrite((change, context) => __awaiter(void 0, void 0, void 0, function* () { +exports.fsexportbigquery = functions.handler.firestore.document.onWrite(async (change, context) => { logs.start(); try { const changeType = util_1.getChangeType(change); const documentId = util_1.getDocumentId(change); - yield eventTracker.record([ + await eventTracker.record([ { timestamp: context.timestamp, operation: changeType, @@ -54,4 +45,4 @@ exports.fsexportbigquery = functions.handler.firestore.document.onWrite((change, catch (err) { logs.error(err); } -})); +}); diff --git a/firestore-bigquery-export/functions/lib/logs.js b/firestore-bigquery-export/functions/lib/logs.js index 10abf8917..090409138 100644 --- a/firestore-bigquery-export/functions/lib/logs.js +++ b/firestore-bigquery-export/functions/lib/logs.js @@ -15,6 +15,7 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); +exports.timestampMissingValue = exports.start = exports.init = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0; const config_1 = require("./config"); exports.arrayFieldInvalid = (fieldName) => { console.warn(`Array field '${fieldName}' does not contain an array, skipping`); diff --git a/firestore-bigquery-export/functions/lib/util.js b/firestore-bigquery-export/functions/lib/util.js index 440e1709d..fc29c0d8b 100644 --- a/firestore-bigquery-export/functions/lib/util.js +++ b/firestore-bigquery-export/functions/lib/util.js @@ -15,6 +15,7 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); +exports.getDocumentId = exports.getChangeType = void 0; const firestore_bigquery_change_tracker_1 = require("@firebaseextensions/firestore-bigquery-change-tracker"); function getChangeType(change) { if (!change.after.exists) { diff --git a/firestore-bigquery-export/package.json b/firestore-bigquery-export/package.json index ede9a91bc..b609bdd51 100644 --- a/firestore-bigquery-export/package.json +++ b/firestore-bigquery-export/package.json @@ -18,7 +18,7 @@ "@types/chai": "^4.1.6", "chai": "^4.2.0", "firebase-admin": "^8.0.0", - "firebase-functions": "^3.3.0", + "firebase-functions": "^3.7.0", "generate-schema": "^2.6.0", "inquirer": "^6.4.0", "lodash": "^4.17.14", diff --git a/firestore-bigquery-export/tsconfig.json b/firestore-bigquery-export/tsconfig.json index 172c7cbc0..c6744df48 100644 --- a/firestore-bigquery-export/tsconfig.json +++ b/firestore-bigquery-export/tsconfig.json @@ -2,7 +2,8 @@ "extends": "../tsconfig.json", "compilerOptions": { "outDir": "functions/lib", - "types": ["node", "jest", "chai"] + "types": ["node", "jest", "chai"], + "target": "es2018" }, "include": ["functions/src"] } From 6e6aac9aec5e79e3ce8a7cf2624dce246cc97197 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 24 Aug 2020 17:33:08 +0100 Subject: [PATCH 12/24] chore(firestore-counter): migration to Node.js v10 (#414) --- firestore-counter/PREINSTALL.md | 14 +- firestore-counter/README.md | 73 ---- firestore-counter/extension.yaml | 6 +- firestore-counter/functions/README.md | 15 +- firestore-counter/functions/lib/controller.js | 364 +++++++++--------- firestore-counter/functions/lib/index.js | 35 +- firestore-counter/functions/lib/worker.js | 64 ++- firestore-counter/functions/package.json | 2 +- firestore-counter/functions/src/controller.ts | 27 +- firestore-counter/functions/src/worker.ts | 19 +- firestore-counter/functions/tsconfig.json | 2 + 11 files changed, 262 insertions(+), 359 deletions(-) delete mode 100644 firestore-counter/README.md diff --git a/firestore-counter/PREINSTALL.md b/firestore-counter/PREINSTALL.md index 4e2a8ec3e..5b6f17d0a 100644 --- a/firestore-counter/PREINSTALL.md +++ b/firestore-counter/PREINSTALL.md @@ -25,10 +25,10 @@ Detailed information for these post-installation tasks are provided after you in #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Firestore + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) \ No newline at end of file diff --git a/firestore-counter/README.md b/firestore-counter/README.md deleted file mode 100644 index a8fbe0aa9..000000000 --- a/firestore-counter/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Distributed Counter - -**Description**: Records event counters at scale to accommodate high-velocity writes to Cloud Firestore. - - - -**Details**: Use this extension to add a highly scalable counter service to your app. This is ideal for applications that count viral actions or any very high-velocity action such as views, likes, or shares. - -Since Cloud Firestore has a limit of one sustained write per second, per document, this extension instead shards your writes across documents in a `_counter_shards_` subcollection. Each client only increments their own unique shard while the background workers (provided by this extension) monitor and aggregate these shards into a main document. - -Here are some features of this extension: - -- Scales from 0 updates per second to a maximum of 10,000 per second. -- Supports an arbitrary number of counters in your app. -- Works offline and provides latency compensation for the main counter. - -Note that this extension requires client-side logic to work. We provide a [TypeScript client sample implementation](https://github.com/firebase/extensions/blob/master/firestore-counter/clients/web/src/index.ts) and its [compiled minified JavaScript](https://github.com/firebase/extensions/blob/master/firestore-counter/clients/web/dist/sharded-counter.js). You can use this extension on other platforms if you'd like to develop your own client code based on the provided client sample. - - -#### Additional setup - -Before installing this extension, make sure that you've [set up a Cloud Firestore database](https://firebase.google.com/docs/firestore/quickstart) in your Firebase project. - -After installing this extension, you'll need to: - -- Update your [database security rules](https://firebase.google.com/docs/rules). -- Set up a [Cloud Scheduler job](https://cloud.google.com/scheduler/docs/quickstart) to regularly call the controllerCore function, which is created by this extension. It works by either aggregating shards itself or scheduling and monitoring workers to aggregate shards. -- Use the provided [client sample](https://github.com/firebase/extensions/blob/master/firestore-counter/clients/web/src/index.ts) or your own client code to specify your document path and increment values. - -Detailed information for these post-installation tasks are provided after you install this extension. - - -#### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) - - - - -**Configuration Parameters:** - -* Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). - -* Document path for internal state: What is the path to the document where the extension can keep its internal state? - - - -**Cloud Functions:** - -* **controllerCore:** Scheduled to run every minute. This function either aggregates shards itself, or it schedules and monitors workers to aggregate shards. - -* **controller:** Maintained for backwards compatibility. This function relays a message to the extension's Pub/Sub topic to trigger the controllerCore function. - -* **onWrite:** Listens for changes on counter shards that may need aggregating. This function is limited to max 1 instance. - -* **worker:** Monitors a range of shards and aggregates them, as needed. There may be 0 or more worker functions running at any point in time. The controllerCore function is responsible for scheduling and monitoring these workers. - - - -**Access Required**: - - - -This extension will operate with the following project IAM roles: - -* datastore.user (Reason: Allows the extension to aggregate Cloud Firestore counter shards.) - -* pubsub.publisher (Reason: Allows the HTTPS controller function to publish a message to the extension's Pub/Sub topic, which triggers the controllerCore function.) diff --git a/firestore-counter/extension.yaml b/firestore-counter/extension.yaml index eaf473a37..969138371 100644 --- a/firestore-counter/extension.yaml +++ b/firestore-counter/extension.yaml @@ -38,7 +38,7 @@ contributors: url: https://github.com/invertase -billingRequired: false +billingRequired: true roles: - role: datastore.user @@ -57,6 +57,7 @@ resources: This function either aggregates shards itself, or it schedules and monitors workers to aggregate shards. properties: location: ${LOCATION} + runtime: nodejs10 maxInstances: 1 eventTrigger: eventType: google.pubsub.topic.publish @@ -69,6 +70,7 @@ resources: This function relays a message to the extension's Pub/Sub topic to trigger the controllerCore function. properties: location: ${LOCATION} + runtime: nodejs10 maxInstances: 1 httpsTrigger: {} @@ -78,6 +80,7 @@ resources: Listens for changes on counter shards that may need aggregating. This function is limited to max 1 instance. properties: location: ${LOCATION} + runtime: nodejs10 maxInstances: 1 timeout: 120s eventTrigger: @@ -92,6 +95,7 @@ resources: The controllerCore function is responsible for scheduling and monitoring these workers. properties: location: ${LOCATION} + runtime: nodejs10 eventTrigger: eventType: providers/cloud.firestore/eventTypes/document.write resource: projects/${PROJECT_ID}/databases/(default)/documents/${INTERNAL_STATE_PATH}/workers/{workerId} diff --git a/firestore-counter/functions/README.md b/firestore-counter/functions/README.md index 60fbe3f28..3b2f98ff0 100644 --- a/firestore-counter/functions/README.md +++ b/firestore-counter/functions/README.md @@ -33,14 +33,13 @@ Detailed information for these post-installation tasks are provided after you in #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) - + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Firestore + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/firestore-counter/functions/lib/controller.js b/firestore-counter/functions/lib/controller.js index 8bfa6ad8b..3f6f2a2fd 100644 --- a/firestore-counter/functions/lib/controller.js +++ b/firestore-counter/functions/lib/controller.js @@ -14,21 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); exports.ShardedCounterController = exports.ControllerStatus = void 0; const firebase_admin_1 = require("firebase-admin"); const common_1 = require("./common"); const planner_1 = require("./planner"); const aggregator_1 = require("./aggregator"); +const firebase_functions_1 = require("firebase-functions"); const EMPTY_CONTROLLER_DATA = { workers: [], timestamp: 0 }; var ControllerStatus; (function (ControllerStatus) { @@ -58,212 +50,206 @@ class ShardedCounterController { * Try aggregating up to 'limit' shards for a fixed amount of time. If workers * are running or there's too many shards, it won't run aggregations at all. */ - aggregateContinuously(slice, limit, timeoutMillis) { - return __awaiter(this, void 0, void 0, function* () { - return new Promise((resolve, reject) => { - let aggrPromise = null; - let controllerData = EMPTY_CONTROLLER_DATA; - let rounds = 0; - let skippedRoundsDueToWorkers = 0; - let shardsCount = 0; - let unsubscribeControllerListener = this.controllerDocRef.onSnapshot((snap) => { - if (snap.exists) { - controllerData = snap.data(); - } - }); - let unsubscribeSliceListener = common_1.queryRange(this.db, this.shardCollectionId, slice.start, slice.end, limit).onSnapshot((snap) => __awaiter(this, void 0, void 0, function* () { - if (snap.docs.length == limit) - return; - if (controllerData.workers.length > 0) { - skippedRoundsDueToWorkers++; - return; - } - if (aggrPromise === null) { - aggrPromise = this.aggregateOnce(slice, limit); - const status = yield aggrPromise; - aggrPromise = null; - if (status === ControllerStatus.SUCCESS) { - shardsCount += snap.docs.length; - rounds++; - } + async aggregateContinuously(slice, limit, timeoutMillis) { + return new Promise((resolve, reject) => { + let aggrPromise = null; + let controllerData = EMPTY_CONTROLLER_DATA; + let rounds = 0; + let skippedRoundsDueToWorkers = 0; + let shardsCount = 0; + let unsubscribeControllerListener = this.controllerDocRef.onSnapshot((snap) => { + if (snap.exists) { + controllerData = snap.data(); + } + }); + let unsubscribeSliceListener = common_1.queryRange(this.db, this.shardCollectionId, slice.start, slice.end, limit).onSnapshot(async (snap) => { + if (snap.docs.length == limit) + return; + if (controllerData.workers.length > 0) { + skippedRoundsDueToWorkers++; + return; + } + if (aggrPromise === null) { + aggrPromise = this.aggregateOnce(slice, limit); + const status = await aggrPromise; + aggrPromise = null; + if (status === ControllerStatus.SUCCESS) { + shardsCount += snap.docs.length; + rounds++; } - })); - const shutdown = () => __awaiter(this, void 0, void 0, function* () { - console.log("Successfully ran " + - rounds + - " rounds. Aggregated " + - shardsCount + - " shards."); - console.log("Skipped " + - skippedRoundsDueToWorkers + - " rounds due to workers running."); - unsubscribeControllerListener(); - unsubscribeSliceListener(); - if (aggrPromise === null) - yield aggrPromise; - resolve(); - }); - setTimeout(shutdown, timeoutMillis); + } }); + const shutdown = async () => { + firebase_functions_1.logger.log("Successfully ran " + + rounds + + " rounds. Aggregated " + + shardsCount + + " shards."); + firebase_functions_1.logger.log("Skipped " + + skippedRoundsDueToWorkers + + " rounds due to workers running."); + unsubscribeControllerListener(); + unsubscribeSliceListener(); + if (aggrPromise === null) + await aggrPromise; + resolve(); + }; + setTimeout(shutdown, timeoutMillis); }); } /** * Try aggregating up to 'limit' shards. If workers are running or there's too many * shards return with appropriate status code. */ - aggregateOnce(slice, limit) { - return __awaiter(this, void 0, void 0, function* () { - try { - const status = yield this.db.runTransaction((t) => __awaiter(this, void 0, void 0, function* () { - let controllerDoc = null; - try { - controllerDoc = yield t.get(this.controllerDocRef); - } - catch (err) { - console.log("Failed to read controller doc: " + this.controllerDocRef.path); - throw Error("Failed to read controller doc."); - } - const controllerData = controllerDoc.exists - ? controllerDoc.data() - : EMPTY_CONTROLLER_DATA; - if (controllerData.workers.length > 0) - return ControllerStatus.WORKERS_RUNNING; - let shards = null; - try { - shards = yield t.get(common_1.queryRange(this.db, this.shardCollectionId, slice.start, slice.end, limit)); - } - catch (err) { - console.log("Query to find shards to aggregate failed.", err); - throw Error("Query to find shards to aggregate failed."); - } - if (shards.docs.length == 200) - return ControllerStatus.TOO_MANY_SHARDS; - const plans = planner_1.Planner.planAggregations("", shards.docs); - const promises = plans.map((plan) => __awaiter(this, void 0, void 0, function* () { - if (plan.isPartial) { - throw Error("Aggregation plan in controller run resulted in partial shard, " + - "this should never happen!"); - } - let counter = null; - try { - counter = yield t.get(this.db.doc(plan.aggregate)); - } - catch (err) { - console.log("Failed to read document: " + plan.aggregate, err); - throw Error("Failed to read counter " + plan.aggregate); - } - // Calculate aggregated value and save to aggregate shard. - const aggr = new aggregator_1.Aggregator(); - const update = aggr.aggregate(counter, plan.partials, plan.shards); - t.set(this.db.doc(plan.aggregate), update, { merge: true }); - // Delete shards that have been aggregated. - plan.shards.forEach((snap) => t.delete(snap.ref)); - plan.partials.forEach((snap) => t.delete(snap.ref)); - })); - try { - yield Promise.all(promises); - } - catch (err) { - console.log("Some counter aggregation failed, bailing out."); - throw Error("Some counter aggregation failed, bailing out."); - } - t.set(this.controllerDocRef, { timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp() }, { merge: true }); - console.log("Aggregated " + plans.length + " counters."); - return ControllerStatus.SUCCESS; - })); - return status; - } - catch (err) { - console.log("Transaction to aggregate shards failed.", err); - return ControllerStatus.FAILURE; - } - }); - } - /** - * Reschedule workers based on stats in their metadata docs. - */ - rescheduleWorkers() { - return __awaiter(this, void 0, void 0, function* () { - const timestamp = Date.now(); - yield this.db.runTransaction((t) => __awaiter(this, void 0, void 0, function* () { - // Read controller document to prevent race conditions. + async aggregateOnce(slice, limit) { + try { + const status = await this.db.runTransaction(async (t) => { + let controllerDoc = null; try { - yield t.get(this.controllerDocRef); + controllerDoc = await t.get(this.controllerDocRef); } catch (err) { - console.log("Failed to read controller doc " + this.controllerDocRef.path); + firebase_functions_1.logger.log("Failed to read controller doc: " + this.controllerDocRef.path); throw Error("Failed to read controller doc."); } - // Read all workers' metadata and construct sharding info based on collected stats. - let query = null; + const controllerData = controllerDoc.exists + ? controllerDoc.data() + : EMPTY_CONTROLLER_DATA; + if (controllerData.workers.length > 0) + return ControllerStatus.WORKERS_RUNNING; + let shards = null; try { - query = yield t.get(this.workersRef.orderBy(firebase_admin_1.firestore.FieldPath.documentId())); + shards = await t.get(common_1.queryRange(this.db, this.shardCollectionId, slice.start, slice.end, limit)); } catch (err) { - console.log("Failed to read worker docs.", err); - throw Error("Failed to read worker docs."); + firebase_functions_1.logger.log("Query to find shards to aggregate failed.", err); + throw Error("Query to find shards to aggregate failed."); } - let shardingInfo = yield Promise.all(query.docs.map((worker) => __awaiter(this, void 0, void 0, function* () { - const slice = worker.get("slice"); - const stats = worker.get("stats"); - // This workers hasn't had a chance to finish its run yet. Bail out. - if (!stats) { - return { - slice: slice, - hasData: false, - overloaded: false, - splits: [], - }; + if (shards.docs.length == 200) + return ControllerStatus.TOO_MANY_SHARDS; + const plans = planner_1.Planner.planAggregations("", shards.docs); + const promises = plans.map(async (plan) => { + if (plan.isPartial) { + throw Error("Aggregation plan in controller run resulted in partial shard, " + + "this should never happen!"); } - const hasData = true; - const overloaded = stats.rounds === stats.roundsCapped; - const splits = stats.splits; - // If a worker is overloaded, we don't have reliable splits for that range. - // Fetch extra shards to make better balancing decision. + let counter = null; try { - if (overloaded && splits.length > 0) { - const snap = yield common_1.queryRange(this.db, this.shardCollectionId, splits[splits.length - 1], slice.end, 100000).get(); - for (let i = 100; i < snap.docs.length; i += 100) { - splits.push(snap.docs[i].ref.path); - } - } + counter = await t.get(this.db.doc(plan.aggregate)); } catch (err) { - console.log("Failed to calculate additional splits for worker: " + worker.id); + firebase_functions_1.logger.log("Failed to read document: " + plan.aggregate, err); + throw Error("Failed to read counter " + plan.aggregate); } - return { slice, hasData, overloaded, splits }; - }))); - let [reshard, slices] = ShardedCounterController.balanceWorkers(shardingInfo); - if (reshard) { - console.log("Resharding workers, new workers: " + - slices.length + - " prev num workers: " + - query.docs.length); - query.docs.forEach((snap) => t.delete(snap.ref)); - slices.forEach((slice, index) => { - t.set(this.workersRef.doc(ShardedCounterController.encodeWorkerKey(index)), { - slice: slice, - timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp(), - }); - }); - t.set(this.controllerDocRef, { - workers: slices, - timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp(), - }); + // Calculate aggregated value and save to aggregate shard. + const aggr = new aggregator_1.Aggregator(); + const update = aggr.aggregate(counter, plan.partials, plan.shards); + t.set(this.db.doc(plan.aggregate), update, { merge: true }); + // Delete shards that have been aggregated. + plan.shards.forEach((snap) => t.delete(snap.ref)); + plan.partials.forEach((snap) => t.delete(snap.ref)); + }); + try { + await Promise.all(promises); } - else { - // Check workers that haven't updated stats for over 90s - they most likely failed. - let failures = 0; - query.docs.forEach((snap) => { - if (timestamp / 1000 - snap.updateTime.seconds > 90) { - t.set(snap.ref, { timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp() }, { merge: true }); - failures++; + catch (err) { + firebase_functions_1.logger.log("Some counter aggregation failed, bailing out."); + throw Error("Some counter aggregation failed, bailing out."); + } + t.set(this.controllerDocRef, { timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp() }, { merge: true }); + firebase_functions_1.logger.log("Aggregated " + plans.length + " counters."); + return ControllerStatus.SUCCESS; + }); + return status; + } + catch (err) { + firebase_functions_1.logger.log("Transaction to aggregate shards failed.", err); + return ControllerStatus.FAILURE; + } + } + /** + * Reschedule workers based on stats in their metadata docs. + */ + async rescheduleWorkers() { + const timestamp = Date.now(); + await this.db.runTransaction(async (t) => { + // Read controller document to prevent race conditions. + try { + await t.get(this.controllerDocRef); + } + catch (err) { + firebase_functions_1.logger.log("Failed to read controller doc " + this.controllerDocRef.path); + throw Error("Failed to read controller doc."); + } + // Read all workers' metadata and construct sharding info based on collected stats. + let query = null; + try { + query = await t.get(this.workersRef.orderBy(firebase_admin_1.firestore.FieldPath.documentId())); + } + catch (err) { + firebase_functions_1.logger.log("Failed to read worker docs.", err); + throw Error("Failed to read worker docs."); + } + let shardingInfo = await Promise.all(query.docs.map(async (worker) => { + const slice = worker.get("slice"); + const stats = worker.get("stats"); + // This workers hasn't had a chance to finish its run yet. Bail out. + if (!stats) { + return { + slice: slice, + hasData: false, + overloaded: false, + splits: [], + }; + } + const hasData = true; + const overloaded = stats.rounds === stats.roundsCapped; + const splits = stats.splits; + // If a worker is overloaded, we don't have reliable splits for that range. + // Fetch extra shards to make better balancing decision. + try { + if (overloaded && splits.length > 0) { + const snap = await common_1.queryRange(this.db, this.shardCollectionId, splits[splits.length - 1], slice.end, 100000).get(); + for (let i = 100; i < snap.docs.length; i += 100) { + splits.push(snap.docs[i].ref.path); } - }); - console.log("Detected " + failures + " failed workers."); - t.set(this.controllerDocRef, { timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp() }, { merge: true }); + } + } + catch (err) { + firebase_functions_1.logger.log("Failed to calculate additional splits for worker: " + worker.id); } + return { slice, hasData, overloaded, splits }; })); + let [reshard, slices] = ShardedCounterController.balanceWorkers(shardingInfo); + if (reshard) { + firebase_functions_1.logger.log("Resharding workers, new workers: " + + slices.length + + " prev num workers: " + + query.docs.length); + query.docs.forEach((snap) => t.delete(snap.ref)); + slices.forEach((slice, index) => { + t.set(this.workersRef.doc(ShardedCounterController.encodeWorkerKey(index)), { + slice: slice, + timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp(), + }); + }); + t.set(this.controllerDocRef, { + workers: slices, + timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp(), + }); + } + else { + // Check workers that haven't updated stats for over 90s - they most likely failed. + let failures = 0; + query.docs.forEach((snap) => { + if (timestamp / 1000 - snap.updateTime.seconds > 90) { + t.set(snap.ref, { timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp() }, { merge: true }); + failures++; + } + }); + firebase_functions_1.logger.log("Detected " + failures + " failed workers."); + t.set(this.controllerDocRef, { timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp() }, { merge: true }); + } }); } /** diff --git a/firestore-counter/functions/lib/index.js b/firestore-counter/functions/lib/index.js index 090d8ef12..83f354919 100644 --- a/firestore-counter/functions/lib/index.js +++ b/firestore-counter/functions/lib/index.js @@ -14,15 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); exports.onWrite = exports.worker = exports.controller = exports.controllerCore = void 0; const functions = require("firebase-functions"); @@ -41,29 +32,29 @@ const WORKERS_COLLECTION_ID = "_counter_workers_"; * there's less than 200 of them. Otherwise it is scheduling and monitoring * workers to do the aggregation. */ -exports.controllerCore = functions.handler.pubsub.topic.onPublish(() => __awaiter(void 0, void 0, void 0, function* () { +exports.controllerCore = functions.handler.pubsub.topic.onPublish(async () => { const metadocRef = firestore.doc(process.env.INTERNAL_STATE_PATH); const controller = new controller_1.ShardedCounterController(metadocRef, SHARDS_COLLECTION_ID); - let status = yield controller.aggregateOnce({ start: "", end: "" }, 200); + let status = await controller.aggregateOnce({ start: "", end: "" }, 200); if (status === controller_1.ControllerStatus.WORKERS_RUNNING || status === controller_1.ControllerStatus.TOO_MANY_SHARDS || status === controller_1.ControllerStatus.FAILURE) { - yield controller.rescheduleWorkers(); + await controller.rescheduleWorkers(); } return null; -})); +}); /** * Backwards compatible HTTPS function */ -exports.controller = functions.handler.https.onRequest((req, res) => __awaiter(void 0, void 0, void 0, function* () { +exports.controller = functions.handler.https.onRequest(async (req, res) => { if (!pubsub) { pubsub = new pubsub_1.PubSub(); } - yield pubsub + await pubsub .topic(process.env.EXT_INSTANCE_ID) .publish(Buffer.from(JSON.stringify({}))); res.status(200).send("Ok"); -})); +}); /** * Worker is responsible for aggregation of a defined range of shards. It is controlled * by a worker metadata document. At the end of its run (that lasts for 45s) it writes @@ -72,20 +63,20 @@ exports.controller = functions.handler.https.onRequest((req, res) => __awaiter(v * ControllerCore is monitoring these metadata documents to detect overload that requires * resharding and to detect failed workers that need poking. */ -exports.worker = functions.handler.firestore.document.onWrite((change, context) => __awaiter(void 0, void 0, void 0, function* () { +exports.worker = functions.handler.firestore.document.onWrite(async (change, context) => { // stop worker if document got deleted if (!change.after.exists) return; const worker = new worker_1.ShardedCounterWorker(change.after, SHARDS_COLLECTION_ID); - yield worker.run(); -})); + await worker.run(); +}); /** * This is an additional function that is triggered for every shard write. It is * limited to one concurrent run at the time. This helps reduce latency for workloads * that are below the threshold for workers. */ -exports.onWrite = functions.handler.firestore.document.onWrite((change, context) => __awaiter(void 0, void 0, void 0, function* () { +exports.onWrite = functions.handler.firestore.document.onWrite(async (change, context) => { const metadocRef = firestore.doc(process.env.INTERNAL_STATE_PATH); const controller = new controller_1.ShardedCounterController(metadocRef, SHARDS_COLLECTION_ID); - yield controller.aggregateContinuously({ start: "", end: "" }, 200, 60000); -})); + await controller.aggregateContinuously({ start: "", end: "" }, 200, 60000); +}); diff --git a/firestore-counter/functions/lib/worker.js b/firestore-counter/functions/lib/worker.js index 1a2d1eb45..c75844090 100644 --- a/firestore-counter/functions/lib/worker.js +++ b/firestore-counter/functions/lib/worker.js @@ -14,19 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); exports.ShardedCounterWorker = void 0; const firebase_admin_1 = require("firebase-admin"); const deepEqual = require("deep-equal"); +const firebase_functions_1 = require("firebase-functions"); const common_1 = require("./common"); const planner_1 = require("./planner"); const aggregator_1 = require("./aggregator"); @@ -67,21 +59,21 @@ class ShardedCounterWorker { let timeoutTimer; let unsubscribeMetadataListener; let unsubscribeSliceListener; - const shutdown = () => __awaiter(this, void 0, void 0, function* () { + const shutdown = async () => { clearInterval(intervalTimer); clearTimeout(timeoutTimer); unsubscribeMetadataListener(); unsubscribeSliceListener(); if (this.aggregation != null) { try { - yield this.aggregation; + await this.aggregation; } catch (err) { // Not much here we can do, transaction is over. } } - }); - const writeStats = () => __awaiter(this, void 0, void 0, function* () { + }; + const writeStats = async () => { this.allPaths.sort(); let splits = this.allPaths.filter((val, idx) => idx !== 0 && idx % 100 === 0); let stats = { @@ -92,9 +84,9 @@ class ShardedCounterWorker { roundsCapped: this.roundsCapped, }; try { - yield this.db.runTransaction((t) => __awaiter(this, void 0, void 0, function* () { + await this.db.runTransaction(async (t) => { try { - const snap = yield t.get(this.metadoc.ref); + const snap = await t.get(this.metadoc.ref); if (snap.exists && deepEqual(snap.data(), this.metadata)) { t.update(snap.ref, { timestamp: firebase_admin_1.firestore.FieldValue.serverTimestamp(), @@ -103,14 +95,14 @@ class ShardedCounterWorker { } } catch (err) { - console.log("Failed to save writer stats.", err); + firebase_functions_1.logger.log("Failed to save writer stats.", err); } - })); + }); } catch (err) { - console.log("Failed to save writer stats.", err); + firebase_functions_1.logger.log("Failed to save writer stats.", err); } - }); + }; intervalTimer = setInterval(() => { this.maybeAggregate(); }, 1000); @@ -121,7 +113,7 @@ class ShardedCounterWorker { unsubscribeMetadataListener = this.metadoc.ref.onSnapshot((snap) => { // if something's changed in the worker metadata since we were called, abort. if (!snap.exists || !deepEqual(snap.data(), this.metadata)) { - console.log("Shutting down because metadoc changed."); + firebase_functions_1.logger.log("Shutting down because metadoc changed."); shutdown() .then(resolve) .catch(reject); @@ -130,7 +122,7 @@ class ShardedCounterWorker { unsubscribeSliceListener = common_1.queryRange(this.db, this.shardCollection, this.metadata.slice.start, this.metadata.slice.end, SHARDS_LIMIT).onSnapshot((snap) => { this.shards = snap.docs; if (this.singleRun && this.shards.length === 0) { - console.log("Shutting down, single run mode."); + firebase_functions_1.logger.log("Shutting down, single run mode."); shutdown() .then(writeStats) .then(resolve) @@ -149,9 +141,9 @@ class ShardedCounterWorker { const [toAggregate, toCleanup] = ShardedCounterWorker.categorizeShards(this.shards, this.singleRun); const cleanupPromises = ShardedCounterWorker.cleanupPartials(this.db, toCleanup); const plans = planner_1.Planner.planAggregations(this.metadata.slice.start, toAggregate); - const promises = plans.map((plan) => __awaiter(this, void 0, void 0, function* () { + const promises = plans.map(async (plan) => { try { - const paths = yield this.db.runTransaction((t) => __awaiter(this, void 0, void 0, function* () { + const paths = await this.db.runTransaction(async (t) => { const paths = []; // Read metadata document in transaction to guarantee ownership of the slice. const metadocPromise = t.get(this.metadoc.ref); @@ -169,19 +161,19 @@ class ShardedCounterWorker { let counter; let metadoc; try { - [shards, counter, metadoc] = yield Promise.all([ + [shards, counter, metadoc] = await Promise.all([ shardsPromise, counterPromise, metadocPromise, ]); } catch (err) { - console.log("Unable to read shards during aggregation round, skipping...", err); + firebase_functions_1.logger.log("Unable to read shards during aggregation round, skipping...", err); return []; } // Check that we still own the slice. if (!metadoc.exists || !deepEqual(metadoc.data(), this.metadata)) { - console.log("Metadata has changed, bailing out..."); + firebase_functions_1.logger.log("Metadata has changed, bailing out..."); return []; } // Calculate aggregated value and save to aggregate shard. @@ -203,13 +195,13 @@ class ShardedCounterWorker { } }); return paths; - })); + }); this.allPaths.push(...paths); } catch (err) { - console.log("transaction to: " + plan.aggregate + " failed, skipping...", err); + firebase_functions_1.logger.log("transaction to: " + plan.aggregate + " failed, skipping...", err); } - })); + }); if (promises.length === 0 && cleanupPromises.length === 0) return; this.aggregation = Promise.all(promises.concat(cleanupPromises)).then(() => { @@ -244,11 +236,11 @@ class ShardedCounterWorker { * Deletes empty partials and compacts non-empty ones. */ static cleanupPartials(db, toCleanup) { - return toCleanup.map((partial) => __awaiter(this, void 0, void 0, function* () { + return toCleanup.map(async (partial) => { try { - yield db.runTransaction((t) => __awaiter(this, void 0, void 0, function* () { + await db.runTransaction(async (t) => { try { - const snap = yield t.get(partial.ref); + const snap = await t.get(partial.ref); if (snap.exists && isEmptyPartial(snap.data())) { t.delete(snap.ref); } @@ -264,14 +256,14 @@ class ShardedCounterWorker { } } catch (err) { - console.log("Partial cleanup failed: " + partial.ref.path); + firebase_functions_1.logger.log("Partial cleanup failed: " + partial.ref.path); } - })); + }); } catch (err) { - console.log("transaction to delete: " + partial.ref.path + " failed, skipping", err); + firebase_functions_1.logger.log("transaction to delete: " + partial.ref.path + " failed, skipping", err); } - })); + }); } } exports.ShardedCounterWorker = ShardedCounterWorker; diff --git a/firestore-counter/functions/package.json b/firestore-counter/functions/package.json index c3dabe563..3c71cadad 100644 --- a/firestore-counter/functions/package.json +++ b/firestore-counter/functions/package.json @@ -8,7 +8,7 @@ "@google-cloud/pubsub": "^1.1.5", "deep-equal": "^1.0.1", "firebase-admin": "^8.3.0", - "firebase-functions": "^3.3.0", + "firebase-functions": "^3.7.0", "uuid": "^3.3.2" }, "devDependencies": { diff --git a/firestore-counter/functions/src/controller.ts b/firestore-counter/functions/src/controller.ts index e800ce368..2cc6fc5c7 100644 --- a/firestore-counter/functions/src/controller.ts +++ b/firestore-counter/functions/src/controller.ts @@ -18,6 +18,7 @@ import { firestore } from "firebase-admin"; import { Slice, WorkerStats, queryRange } from "./common"; import { Planner } from "./planner"; import { Aggregator } from "./aggregator"; +import { logger } from "firebase-functions"; export interface WorkerShardingInfo { slice: Slice; // shard range a single worker is responsible for @@ -108,14 +109,14 @@ export class ShardedCounterController { }); const shutdown = async () => { - console.log( + logger.log( "Successfully ran " + rounds + " rounds. Aggregated " + shardsCount + " shards." ); - console.log( + logger.log( "Skipped " + skippedRoundsDueToWorkers + " rounds due to workers running." @@ -143,7 +144,7 @@ export class ShardedCounterController { try { controllerDoc = await t.get(this.controllerDocRef); } catch (err) { - console.log( + logger.log( "Failed to read controller doc: " + this.controllerDocRef.path ); throw Error("Failed to read controller doc."); @@ -166,7 +167,7 @@ export class ShardedCounterController { ) ); } catch (err) { - console.log("Query to find shards to aggregate failed.", err); + logger.log("Query to find shards to aggregate failed.", err); throw Error("Query to find shards to aggregate failed."); } if (shards.docs.length == 200) return ControllerStatus.TOO_MANY_SHARDS; @@ -183,7 +184,7 @@ export class ShardedCounterController { try { counter = await t.get(this.db.doc(plan.aggregate)); } catch (err) { - console.log("Failed to read document: " + plan.aggregate, err); + logger.log("Failed to read document: " + plan.aggregate, err); throw Error("Failed to read counter " + plan.aggregate); } // Calculate aggregated value and save to aggregate shard. @@ -197,7 +198,7 @@ export class ShardedCounterController { try { await Promise.all(promises); } catch (err) { - console.log("Some counter aggregation failed, bailing out."); + logger.log("Some counter aggregation failed, bailing out."); throw Error("Some counter aggregation failed, bailing out."); } t.set( @@ -205,12 +206,12 @@ export class ShardedCounterController { { timestamp: firestore.FieldValue.serverTimestamp() }, { merge: true } ); - console.log("Aggregated " + plans.length + " counters."); + logger.log("Aggregated " + plans.length + " counters."); return ControllerStatus.SUCCESS; }); return status; } catch (err) { - console.log("Transaction to aggregate shards failed.", err); + logger.log("Transaction to aggregate shards failed.", err); return ControllerStatus.FAILURE; } } @@ -226,7 +227,7 @@ export class ShardedCounterController { try { await t.get(this.controllerDocRef); } catch (err) { - console.log( + logger.log( "Failed to read controller doc " + this.controllerDocRef.path ); throw Error("Failed to read controller doc."); @@ -238,7 +239,7 @@ export class ShardedCounterController { this.workersRef.orderBy(firestore.FieldPath.documentId()) ); } catch (err) { - console.log("Failed to read worker docs.", err); + logger.log("Failed to read worker docs.", err); throw Error("Failed to read worker docs."); } let shardingInfo: WorkerShardingInfo[] = await Promise.all( @@ -274,7 +275,7 @@ export class ShardedCounterController { } } } catch (err) { - console.log( + logger.log( "Failed to calculate additional splits for worker: " + worker.id ); } @@ -286,7 +287,7 @@ export class ShardedCounterController { shardingInfo ); if (reshard) { - console.log( + logger.log( "Resharding workers, new workers: " + slices.length + " prev num workers: " + @@ -321,7 +322,7 @@ export class ShardedCounterController { failures++; } }); - console.log("Detected " + failures + " failed workers."); + logger.log("Detected " + failures + " failed workers."); t.set( this.controllerDocRef, { timestamp: firestore.FieldValue.serverTimestamp() }, diff --git a/firestore-counter/functions/src/worker.ts b/firestore-counter/functions/src/worker.ts index e42f64f25..2f7260f19 100644 --- a/firestore-counter/functions/src/worker.ts +++ b/firestore-counter/functions/src/worker.ts @@ -16,6 +16,7 @@ import { firestore } from "firebase-admin"; import * as deepEqual from "deep-equal"; +import { logger } from "firebase-functions"; import { Slice, @@ -115,11 +116,11 @@ export class ShardedCounterWorker { }); } } catch (err) { - console.log("Failed to save writer stats.", err); + logger.log("Failed to save writer stats.", err); } }); } catch (err) { - console.log("Failed to save writer stats.", err); + logger.log("Failed to save writer stats.", err); } }; @@ -139,7 +140,7 @@ export class ShardedCounterWorker { unsubscribeMetadataListener = this.metadoc.ref.onSnapshot((snap) => { // if something's changed in the worker metadata since we were called, abort. if (!snap.exists || !deepEqual(snap.data(), this.metadata)) { - console.log("Shutting down because metadoc changed."); + logger.log("Shutting down because metadoc changed."); shutdown() .then(resolve) .catch(reject); @@ -155,7 +156,7 @@ export class ShardedCounterWorker { ).onSnapshot((snap) => { this.shards = snap.docs; if (this.singleRun && this.shards.length === 0) { - console.log("Shutting down, single run mode."); + logger.log("Shutting down, single run mode."); shutdown() .then(writeStats) .then(resolve) @@ -217,7 +218,7 @@ export class ShardedCounterWorker { metadocPromise, ]); } catch (err) { - console.log( + logger.log( "Unable to read shards during aggregation round, skipping...", err ); @@ -226,7 +227,7 @@ export class ShardedCounterWorker { // Check that we still own the slice. if (!metadoc.exists || !deepEqual(metadoc.data(), this.metadata)) { - console.log("Metadata has changed, bailing out..."); + logger.log("Metadata has changed, bailing out..."); return []; } @@ -254,7 +255,7 @@ export class ShardedCounterWorker { }); this.allPaths.push(...paths); } catch (err) { - console.log( + logger.log( "transaction to: " + plan.aggregate + " failed, skipping...", err ); @@ -321,11 +322,11 @@ export class ShardedCounterWorker { t.set(snap.ref, update.toPartialShard(() => uuid.v4())); } } catch (err) { - console.log("Partial cleanup failed: " + partial.ref.path); + logger.log("Partial cleanup failed: " + partial.ref.path); } }); } catch (err) { - console.log( + logger.log( "transaction to delete: " + partial.ref.path + " failed, skipping", err ); diff --git a/firestore-counter/functions/tsconfig.json b/firestore-counter/functions/tsconfig.json index 3cbb0ac4d..e4c07e61c 100644 --- a/firestore-counter/functions/tsconfig.json +++ b/firestore-counter/functions/tsconfig.json @@ -1,6 +1,8 @@ { "extends": "../../tsconfig.json", "compilerOptions": { + "target": "es2018", + "lib": ["es2018"], "outDir": "lib", "types": ["node", "mocha", "chai"] }, From 8c6bd519917ac65f3894be8d7bd4ad3ab8ea01a0 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 24 Aug 2020 17:33:25 +0100 Subject: [PATCH 13/24] chore(firestore-send-email): migration to Node.js v10 (#428) --- firestore-send-email/PREINSTALL.md | 11 +- firestore-send-email/README.md | 75 ---- firestore-send-email/extension.yaml | 1 + firestore-send-email/functions/README.md | 11 +- .../functions/__tests__/functions.test.ts | 5 +- firestore-send-email/functions/lib/index.js | 383 +++++++++--------- firestore-send-email/functions/lib/logs.js | 19 +- .../functions/lib/templates.js | 38 +- firestore-send-email/functions/package.json | 2 +- firestore-send-email/functions/src/logs.ts | 19 +- .../functions/src/templates.ts | 3 +- firestore-send-email/functions/tsconfig.json | 3 +- 12 files changed, 236 insertions(+), 334 deletions(-) delete mode 100644 firestore-send-email/README.md diff --git a/firestore-send-email/PREINSTALL.md b/firestore-send-email/PREINSTALL.md index 4e239c781..2d88fab17 100644 --- a/firestore-send-email/PREINSTALL.md +++ b/firestore-send-email/PREINSTALL.md @@ -23,12 +23,11 @@ When you configure this extension, you'll need to supply your **SMTP credentials Before installing this extension, make sure that you've [set up a Cloud Firestore database](https://firebase.google.com/docs/firestore/quickstart) in your Firebase project. #### Billing +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Firestore + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) Usage of this extension also requires you to have SMTP credentials for mail delivery. You are responsible for any associated costs with your usage of your SMTP provider. diff --git a/firestore-send-email/README.md b/firestore-send-email/README.md deleted file mode 100644 index 5ecd26cf3..000000000 --- a/firestore-send-email/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Trigger Email - -**Description**: Composes and sends an email based on the contents of a document written to a specified Cloud Firestore collection. - - - -**Details**: Use this extension to render and send emails that contain the information from documents added to a specified Cloud Firestore collection. - -Adding a document triggers this extension to send an email built from the document's fields. The document's top-level fields specify the email sender and recipients, including `to`, `cc`, and `bcc` options (each supporting UIDs). The document's `message` field specifies the other email elements, like subject line and email body (either plaintext or HTML) - -Here's a basic example document write that would trigger this extension: - -```js -admin.firestore().collection('mail').add({ - to: 'someone@example.com', - message: { - subject: 'Hello from Firebase!', - html: 'This is an HTML email body.', - }, -}) -``` - -You can also optionally configure this extension to render emails using [Handlebar](https://handlebarsjs.com/) templates. Each template is a document stored in a Cloud Firestore collection. - -When you configure this extension, you'll need to supply your **SMTP credentials for mail delivery**. Note that this extension is for use with bulk email service providers, like SendGrid, Mailgun, etc. - -#### Additional setup - -Before installing this extension, make sure that you've [set up a Cloud Firestore database](https://firebase.google.com/docs/firestore/quickstart) in your Firebase project. - -#### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) - -Usage of this extension also requires you to have SMTP credentials for mail delivery. You are responsible for any associated costs with your usage of your SMTP provider. - - - - -**Configuration Parameters:** - -* Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). - -* SMTP connection URI: A URI representing an SMTP server that this extension can use to deliver email. - -* Email documents collection: What is the path to the collection that contains the documents used to build and send the emails? - -* Default FROM address: The email address to use as the sender's address (if it's not specified in the added email document). You can optionally include a name with the email address (`Friendly Firebaser `). - -* Default REPLY-TO address: The email address to use as the reply-to address (if it's not specified in the added email document). - -* Users collection: A collection of documents keyed by user UID. If the `toUids`, `ccUids`, and/or `bccUids` recipient options are used in the added email document, this extension delivers email to the `email` field based on lookups in this collection. - -* Templates collection: A collection of email templates keyed by name. This extension can render an email using a [Handlebar](https://handlebarsjs.com/) template, if the template is specified in the added email document. - - - -**Cloud Functions:** - -* **processQueue:** Processes document changes in the specified Cloud Firestore collection, delivers emails, and updates the document with delivery status information. - - - -**Access Required**: - - - -This extension will operate with the following project IAM roles: - -* datastore.user (Reason: Allows this extension to access Cloud Firestore to read and process added email documents.) diff --git a/firestore-send-email/extension.yaml b/firestore-send-email/extension.yaml index 77759bc3e..2b49f398e 100644 --- a/firestore-send-email/extension.yaml +++ b/firestore-send-email/extension.yaml @@ -50,6 +50,7 @@ resources: delivers emails, and updates the document with delivery status information. properties: location: ${LOCATION} + runtime: nodejs10 eventTrigger: eventType: providers/cloud.firestore/eventTypes/document.write resource: projects/${PROJECT_ID}/databases/(default)/documents/${MAIL_COLLECTION}/{id} diff --git a/firestore-send-email/functions/README.md b/firestore-send-email/functions/README.md index ec9dff2fd..856260d70 100644 --- a/firestore-send-email/functions/README.md +++ b/firestore-send-email/functions/README.md @@ -31,13 +31,12 @@ When you configure this extension, you'll need to supply your **SMTP credentials Before installing this extension, make sure that you've [set up a Cloud Firestore database](https://firebase.google.com/docs/firestore/quickstart) in your Firebase project. #### Billing +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Firestore + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) Usage of this extension also requires you to have SMTP credentials for mail delivery. You are responsible for any associated costs with your usage of your SMTP provider. diff --git a/firestore-send-email/functions/__tests__/functions.test.ts b/firestore-send-email/functions/__tests__/functions.test.ts index 1633208f6..fc56d3b86 100644 --- a/firestore-send-email/functions/__tests__/functions.test.ts +++ b/firestore-send-email/functions/__tests__/functions.test.ts @@ -1,5 +1,6 @@ -const consoleLogSpy = jest.spyOn(console, "log").mockImplementation(); -const consoleWarnSpy = jest.spyOn(console, "warn").mockImplementation(); +const { logger } = require("firebase-functions"); + +const consoleLogSpy = jest.spyOn(logger, "log").mockImplementation(); import functionsConfig from "../src/config"; import { obfuscatedConfig } from "../src/logs"; diff --git a/firestore-send-email/functions/lib/index.js b/firestore-send-email/functions/lib/index.js index 4d43eda38..cdf10516b 100644 --- a/firestore-send-email/functions/lib/index.js +++ b/firestore-send-email/functions/lib/index.js @@ -14,15 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); exports.processQueue = void 0; const admin = require("firebase-admin"); @@ -58,225 +49,217 @@ function validateFieldArray(field, array) { throw new Error(`Invalid field "${field}". Expected an array of strings.`); } } -function processCreate(snap) { - return __awaiter(this, void 0, void 0, function* () { - // Wrapping in transaction to allow for automatic retries (#48) - return admin.firestore().runTransaction((transaction) => { - transaction.update(snap.ref, { - delivery: { - startTime: admin.firestore.FieldValue.serverTimestamp(), - state: "PENDING", - attempts: 0, - error: null, - }, - }); - return Promise.resolve(); +async function processCreate(snap) { + // Wrapping in transaction to allow for automatic retries (#48) + return admin.firestore().runTransaction((transaction) => { + transaction.update(snap.ref, { + delivery: { + startTime: admin.firestore.FieldValue.serverTimestamp(), + state: "PENDING", + attempts: 0, + error: null, + }, }); + return Promise.resolve(); }); } -function preparePayload(payload) { - return __awaiter(this, void 0, void 0, function* () { - const { template } = payload; - if (templates && template) { - if (!template.name) { - throw new Error(`Template object is missing a 'name' parameter.`); - } - payload.message = Object.assign(payload.message || {}, yield templates.render(template.name, template.data)); - } - let to = []; - let cc = []; - let bcc = []; - if (typeof payload.to === "string") { - to = [payload.to]; - } - else if (payload.to) { - validateFieldArray("to", payload.to); - to = to.concat(payload.to); - } - if (typeof payload.cc === "string") { - cc = [payload.cc]; - } - else if (payload.cc) { - validateFieldArray("cc", payload.cc); - cc = cc.concat(payload.cc); - } - if (typeof payload.bcc === "string") { - bcc = [payload.bcc]; - } - else if (payload.bcc) { - validateFieldArray("bcc", payload.bcc); - bcc = bcc.concat(payload.bcc); - } - if (!payload.toUids && !payload.ccUids && !payload.bccUids) { - payload.to = to; - payload.cc = cc; - payload.bcc = bcc; - return payload; - } - if (!config_1.default.usersCollection) { - throw new Error("Must specify a users collection to send using uids."); - } - let uids = []; - if (payload.toUids) { - validateFieldArray("toUids", payload.toUids); - uids = uids.concat(payload.toUids); - } - if (payload.ccUids) { - validateFieldArray("ccUids", payload.ccUids); - uids = uids.concat(payload.ccUids); - } - if (payload.bccUids) { - validateFieldArray("bccUids", payload.bccUids); - uids = uids.concat(payload.bccUids); - } - const toFetch = {}; - uids.forEach((uid) => (toFetch[uid] = null)); - const documents = yield db.getAll(...Object.keys(toFetch).map((uid) => db.collection(config_1.default.usersCollection).doc(uid)), { - fieldMask: ["email"], - }); - const missingUids = []; - documents.forEach((documentSnapshot) => { - if (documentSnapshot.exists) { - const email = documentSnapshot.get("email"); - if (email) { - toFetch[documentSnapshot.id] = email; - } - else { - missingUids.push(documentSnapshot.id); - } - } - else { - missingUids.push(documentSnapshot.id); - } - }); - logs.missingUids(missingUids); - if (payload.toUids) { - payload.toUids.forEach((uid) => { - const email = toFetch[uid]; - if (email) { - to.push(email); - } - }); +async function preparePayload(payload) { + const { template } = payload; + if (templates && template) { + if (!template.name) { + throw new Error(`Template object is missing a 'name' parameter.`); } + payload.message = Object.assign(payload.message || {}, await templates.render(template.name, template.data)); + } + let to = []; + let cc = []; + let bcc = []; + if (typeof payload.to === "string") { + to = [payload.to]; + } + else if (payload.to) { + validateFieldArray("to", payload.to); + to = to.concat(payload.to); + } + if (typeof payload.cc === "string") { + cc = [payload.cc]; + } + else if (payload.cc) { + validateFieldArray("cc", payload.cc); + cc = cc.concat(payload.cc); + } + if (typeof payload.bcc === "string") { + bcc = [payload.bcc]; + } + else if (payload.bcc) { + validateFieldArray("bcc", payload.bcc); + bcc = bcc.concat(payload.bcc); + } + if (!payload.toUids && !payload.ccUids && !payload.bccUids) { payload.to = to; - if (payload.ccUids) { - payload.ccUids.forEach((uid) => { - const email = toFetch[uid]; - if (email) { - cc.push(email); - } - }); - } payload.cc = cc; - if (payload.bccUids) { - payload.bccUids.forEach((uid) => { - const email = toFetch[uid]; - if (email) { - bcc.push(email); - } - }); - } payload.bcc = bcc; return payload; + } + if (!config_1.default.usersCollection) { + throw new Error("Must specify a users collection to send using uids."); + } + let uids = []; + if (payload.toUids) { + validateFieldArray("toUids", payload.toUids); + uids = uids.concat(payload.toUids); + } + if (payload.ccUids) { + validateFieldArray("ccUids", payload.ccUids); + uids = uids.concat(payload.ccUids); + } + if (payload.bccUids) { + validateFieldArray("bccUids", payload.bccUids); + uids = uids.concat(payload.bccUids); + } + const toFetch = {}; + uids.forEach((uid) => (toFetch[uid] = null)); + const documents = await db.getAll(...Object.keys(toFetch).map((uid) => db.collection(config_1.default.usersCollection).doc(uid)), { + fieldMask: ["email"], }); -} -function deliver(payload, ref) { - return __awaiter(this, void 0, void 0, function* () { - logs.attemptingDelivery(ref); - const update = { - "delivery.attempts": admin.firestore.FieldValue.increment(1), - "delivery.endTime": admin.firestore.FieldValue.serverTimestamp(), - "delivery.error": null, - "delivery.leaseExpireTime": null, - }; - try { - payload = yield preparePayload(payload); - if (!payload.to.length && !payload.cc.length && !payload.bcc.length) { - throw new Error("Failed to deliver email. Expected at least 1 recipient."); + const missingUids = []; + documents.forEach((documentSnapshot) => { + if (documentSnapshot.exists) { + const email = documentSnapshot.get("email"); + if (email) { + toFetch[documentSnapshot.id] = email; + } + else { + missingUids.push(documentSnapshot.id); } - const result = yield transport.sendMail(Object.assign(payload.message, { - from: payload.from || config_1.default.defaultFrom, - replyTo: payload.replyTo || config_1.default.defaultReplyTo, - to: payload.to, - cc: payload.cc, - bcc: payload.bcc, - headers: payload.headers || {}, - })); - const info = { - messageId: result.messageId || null, - accepted: result.accepted || [], - rejected: result.rejected || [], - pending: result.pending || [], - response: result.response || null, - }; - update["delivery.state"] = "SUCCESS"; - update["delivery.info"] = info; - logs.delivered(ref, info); } - catch (e) { - update["delivery.state"] = "ERROR"; - update["delivery.error"] = e.toString(); - logs.deliveryError(ref, e); + else { + missingUids.push(documentSnapshot.id); } - // Wrapping in transaction to allow for automatic retries (#48) - return admin.firestore().runTransaction((transaction) => { - transaction.update(ref, update); - return Promise.resolve(); - }); }); + logs.missingUids(missingUids); + if (payload.toUids) { + payload.toUids.forEach((uid) => { + const email = toFetch[uid]; + if (email) { + to.push(email); + } + }); + } + payload.to = to; + if (payload.ccUids) { + payload.ccUids.forEach((uid) => { + const email = toFetch[uid]; + if (email) { + cc.push(email); + } + }); + } + payload.cc = cc; + if (payload.bccUids) { + payload.bccUids.forEach((uid) => { + const email = toFetch[uid]; + if (email) { + bcc.push(email); + } + }); + } + payload.bcc = bcc; + return payload; } -function processWrite(change) { - return __awaiter(this, void 0, void 0, function* () { - if (!change.after.exists) { - return null; - } - if (!change.before.exists && change.after.exists) { - return processCreate(change.after); +async function deliver(payload, ref) { + logs.attemptingDelivery(ref); + const update = { + "delivery.attempts": admin.firestore.FieldValue.increment(1), + "delivery.endTime": admin.firestore.FieldValue.serverTimestamp(), + "delivery.error": null, + "delivery.leaseExpireTime": null, + }; + try { + payload = await preparePayload(payload); + if (!payload.to.length && !payload.cc.length && !payload.bcc.length) { + throw new Error("Failed to deliver email. Expected at least 1 recipient."); } - const payload = change.after.data(); - if (!payload.delivery) { - logs.missingDeliveryField(change.after.ref); + const result = await transport.sendMail(Object.assign(payload.message, { + from: payload.from || config_1.default.defaultFrom, + replyTo: payload.replyTo || config_1.default.defaultReplyTo, + to: payload.to, + cc: payload.cc, + bcc: payload.bcc, + headers: payload.headers || {}, + })); + const info = { + messageId: result.messageId || null, + accepted: result.accepted || [], + rejected: result.rejected || [], + pending: result.pending || [], + response: result.response || null, + }; + update["delivery.state"] = "SUCCESS"; + update["delivery.info"] = info; + logs.delivered(ref, info); + } + catch (e) { + update["delivery.state"] = "ERROR"; + update["delivery.error"] = e.toString(); + logs.deliveryError(ref, e); + } + // Wrapping in transaction to allow for automatic retries (#48) + return admin.firestore().runTransaction((transaction) => { + transaction.update(ref, update); + return Promise.resolve(); + }); +} +async function processWrite(change) { + if (!change.after.exists) { + return null; + } + if (!change.before.exists && change.after.exists) { + return processCreate(change.after); + } + const payload = change.after.data(); + if (!payload.delivery) { + logs.missingDeliveryField(change.after.ref); + return null; + } + switch (payload.delivery.state) { + case "SUCCESS": + case "ERROR": return null; - } - switch (payload.delivery.state) { - case "SUCCESS": - case "ERROR": - return null; - case "PROCESSING": - if (payload.delivery.leaseExpireTime.toMillis() < Date.now()) { - // Wrapping in transaction to allow for automatic retries (#48) - return admin.firestore().runTransaction((transaction) => { - transaction.update(change.after.ref, { - "delivery.state": "ERROR", - error: "Message processing lease expired.", - }); - return Promise.resolve(); - }); - } - return null; - case "PENDING": - case "RETRY": + case "PROCESSING": + if (payload.delivery.leaseExpireTime.toMillis() < Date.now()) { // Wrapping in transaction to allow for automatic retries (#48) - yield admin.firestore().runTransaction((transaction) => { + return admin.firestore().runTransaction((transaction) => { transaction.update(change.after.ref, { - "delivery.state": "PROCESSING", - "delivery.leaseExpireTime": admin.firestore.Timestamp.fromMillis(Date.now() + 60000), + "delivery.state": "ERROR", + error: "Message processing lease expired.", }); return Promise.resolve(); }); - return deliver(payload, change.after.ref); - } - }); + } + return null; + case "PENDING": + case "RETRY": + // Wrapping in transaction to allow for automatic retries (#48) + await admin.firestore().runTransaction((transaction) => { + transaction.update(change.after.ref, { + "delivery.state": "PROCESSING", + "delivery.leaseExpireTime": admin.firestore.Timestamp.fromMillis(Date.now() + 60000), + }); + return Promise.resolve(); + }); + return deliver(payload, change.after.ref); + } } -exports.processQueue = functions.handler.firestore.document.onWrite((change) => __awaiter(void 0, void 0, void 0, function* () { +exports.processQueue = functions.handler.firestore.document.onWrite(async (change) => { initialize(); logs.start(); try { - yield processWrite(change); + await processWrite(change); } catch (err) { logs.error(err); return null; } logs.complete(); -})); +}); diff --git a/firestore-send-email/functions/lib/logs.js b/firestore-send-email/functions/lib/logs.js index 7dcffc48f..4c0b395dd 100644 --- a/firestore-send-email/functions/lib/logs.js +++ b/firestore-send-email/functions/lib/logs.js @@ -17,42 +17,43 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.missingUids = exports.missingDeliveryField = exports.deliveryError = exports.delivered = exports.attemptingDelivery = exports.complete = exports.error = exports.start = exports.init = exports.obfuscatedConfig = void 0; const config_1 = require("./config"); +const firebase_functions_1 = require("firebase-functions"); exports.obfuscatedConfig = Object.assign({}, config_1.default, { smtpConnectionUri: "", }); function init() { - console.log("Initializing extension with configuration", exports.obfuscatedConfig); + firebase_functions_1.logger.log("Initializing extension with configuration", exports.obfuscatedConfig); } exports.init = init; function start() { - console.log("Started execution of extension with configuration", exports.obfuscatedConfig); + firebase_functions_1.logger.log("Started execution of extension with configuration", exports.obfuscatedConfig); } exports.start = start; function error(err) { - console.log("Unhandled error occurred during processing:", err); + firebase_functions_1.logger.log("Unhandled error occurred during processing:", err); } exports.error = error; function complete() { - console.log("Completed execution of extension"); + firebase_functions_1.logger.log("Completed execution of extension"); } exports.complete = complete; function attemptingDelivery(ref) { - console.log(`Attempting delivery for message: ${ref.path}`); + firebase_functions_1.logger.log(`Attempting delivery for message: ${ref.path}`); } exports.attemptingDelivery = attemptingDelivery; function delivered(ref, info) { - console.log(`Delivered message: ${ref.path} successfully. messageId: ${info.messageId} accepted: ${info.accepted.length} rejected: ${info.rejected.length} pending: ${info.pending.length}`); + firebase_functions_1.logger.log(`Delivered message: ${ref.path} successfully. messageId: ${info.messageId} accepted: ${info.accepted.length} rejected: ${info.rejected.length} pending: ${info.pending.length}`); } exports.delivered = delivered; function deliveryError(ref, e) { - console.error(`Error when delivering message=${ref.path}: ${e.toString()}`); + firebase_functions_1.logger.error(`Error when delivering message=${ref.path}: ${e.toString()}`); } exports.deliveryError = deliveryError; function missingDeliveryField(ref) { - console.error(`message=${ref.path} is missing 'delivery' field`); + firebase_functions_1.logger.error(`message=${ref.path} is missing 'delivery' field`); } exports.missingDeliveryField = missingDeliveryField; function missingUids(uids) { - console.log(`The following uids were provided, however a document does not exist or has no 'email' field: ${uids.join(",")}`); + firebase_functions_1.logger.log(`The following uids were provided, however a document does not exist or has no 'email' field: ${uids.join(",")}`); } exports.missingUids = missingUids; diff --git a/firestore-send-email/functions/lib/templates.js b/firestore-send-email/functions/lib/templates.js index 66da67c48..4715375cf 100644 --- a/firestore-send-email/functions/lib/templates.js +++ b/firestore-send-email/functions/lib/templates.js @@ -14,17 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); const handlebars_1 = require("handlebars"); +const firebase_functions_1 = require("firebase-functions"); class Templates { constructor(collection) { this.collection = collection; @@ -58,25 +50,23 @@ class Templates { templates.amp = handlebars_1.compile(data.amp); } this.templateMap[doc.id] = templates; - console.log(`loaded template '${doc.id}'`); + firebase_functions_1.logger.log(`loaded template '${doc.id}'`); }); this.ready = true; this.waits.forEach((wait) => wait()); } - render(name, data) { - return __awaiter(this, void 0, void 0, function* () { - yield this.waitUntilReady(); - if (!this.templateMap[name]) { - return Promise.reject(new Error(`tried to render non-existent template '${name}'`)); - } - const t = this.templateMap[name]; - return { - subject: t.subject ? t.subject(data) : null, - html: t.html ? t.html(data) : null, - text: t.text ? t.text(data) : null, - amp: t.amp ? t.amp(data) : null, - }; - }); + async render(name, data) { + await this.waitUntilReady(); + if (!this.templateMap[name]) { + return Promise.reject(new Error(`tried to render non-existent template '${name}'`)); + } + const t = this.templateMap[name]; + return { + subject: t.subject ? t.subject(data) : null, + html: t.html ? t.html(data) : null, + text: t.text ? t.text(data) : null, + amp: t.amp ? t.amp(data) : null, + }; } } exports.default = Templates; diff --git a/firestore-send-email/functions/package.json b/firestore-send-email/functions/package.json index 9dcea1437..edeebd78c 100644 --- a/firestore-send-email/functions/package.json +++ b/firestore-send-email/functions/package.json @@ -16,7 +16,7 @@ "@types/node": "^12.6.9", "@types/nodemailer": "^6.2.1", "firebase-admin": "^8.3.0", - "firebase-functions": "^3.2.0", + "firebase-functions": "^3.7.0", "handlebars": "^4.5.3", "nodemailer": "^6.3.0" }, diff --git a/firestore-send-email/functions/src/logs.ts b/firestore-send-email/functions/src/logs.ts index ffa8afd2c..876e04c50 100644 --- a/firestore-send-email/functions/src/logs.ts +++ b/firestore-send-email/functions/src/logs.ts @@ -15,32 +15,33 @@ */ import config from "./config"; +import { logger } from "firebase-functions"; export const obfuscatedConfig = Object.assign({}, config, { smtpConnectionUri: "", }); export function init() { - console.log("Initializing extension with configuration", obfuscatedConfig); + logger.log("Initializing extension with configuration", obfuscatedConfig); } export function start() { - console.log( + logger.log( "Started execution of extension with configuration", obfuscatedConfig ); } export function error(err: Error) { - console.log("Unhandled error occurred during processing:", err); + logger.log("Unhandled error occurred during processing:", err); } export function complete() { - console.log("Completed execution of extension"); + logger.log("Completed execution of extension"); } export function attemptingDelivery(ref: FirebaseFirestore.DocumentReference) { - console.log(`Attempting delivery for message: ${ref.path}`); + logger.log(`Attempting delivery for message: ${ref.path}`); } export function delivered( @@ -52,7 +53,7 @@ export function delivered( pending: string[]; } ) { - console.log( + logger.log( `Delivered message: ${ref.path} successfully. messageId: ${ info.messageId } accepted: ${info.accepted.length} rejected: ${ @@ -65,15 +66,15 @@ export function deliveryError( ref: FirebaseFirestore.DocumentReference, e: Error ) { - console.error(`Error when delivering message=${ref.path}: ${e.toString()}`); + logger.error(`Error when delivering message=${ref.path}: ${e.toString()}`); } export function missingDeliveryField(ref: FirebaseFirestore.DocumentReference) { - console.error(`message=${ref.path} is missing 'delivery' field`); + logger.error(`message=${ref.path} is missing 'delivery' field`); } export function missingUids(uids: string[]) { - console.log( + logger.log( `The following uids were provided, however a document does not exist or has no 'email' field: ${uids.join( "," )}` diff --git a/firestore-send-email/functions/src/templates.ts b/firestore-send-email/functions/src/templates.ts index 8b7580fdf..6eef2a067 100644 --- a/firestore-send-email/functions/src/templates.ts +++ b/firestore-send-email/functions/src/templates.ts @@ -15,6 +15,7 @@ */ import { compile } from "handlebars"; +import { logger } from "firebase-functions"; interface TemplateGroup { subject?: HandlebarsTemplateDelegate; @@ -64,7 +65,7 @@ export default class Templates { templates.amp = compile(data.amp); } this.templateMap[doc.id] = templates; - console.log(`loaded template '${doc.id}'`); + logger.log(`loaded template '${doc.id}'`); }); this.ready = true; this.waits.forEach((wait) => wait()); diff --git a/firestore-send-email/functions/tsconfig.json b/firestore-send-email/functions/tsconfig.json index 1b551f4e9..9f7e58b1e 100644 --- a/firestore-send-email/functions/tsconfig.json +++ b/firestore-send-email/functions/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "outDir": "lib" + "outDir": "lib", + "target": "es2018" }, "include": ["src"] } From e9d5b801e7da8651f871517ab02f7f61af0723d1 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 24 Aug 2020 17:33:41 +0100 Subject: [PATCH 14/24] chore(auth-mailchimp-sync): migration to Node.js v10 (#430) --- auth-mailchimp-sync/PREINSTALL.md | 14 +++-- auth-mailchimp-sync/README.md | 52 ------------------- auth-mailchimp-sync/extension.yaml | 2 + auth-mailchimp-sync/functions/README.md | 14 +++-- .../functions/__tests__/functions.test.ts | 5 +- auth-mailchimp-sync/functions/lib/index.js | 21 +++----- auth-mailchimp-sync/functions/lib/logs.js | 30 ++++++----- auth-mailchimp-sync/functions/src/logs.ts | 25 ++++----- auth-mailchimp-sync/functions/tsconfig.json | 3 +- 9 files changed, 55 insertions(+), 111 deletions(-) delete mode 100644 auth-mailchimp-sync/README.md diff --git a/auth-mailchimp-sync/PREINSTALL.md b/auth-mailchimp-sync/PREINSTALL.md index d9f8fc62f..b94150e27 100644 --- a/auth-mailchimp-sync/PREINSTALL.md +++ b/auth-mailchimp-sync/PREINSTALL.md @@ -13,13 +13,11 @@ Make sure that you've set up [Firebase Authentication](https://firebase.google.c You must also have a Mailchimp account before installing this extension. #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Firebase Realtime Database -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Functions (Node.js 10+ runtime. See [FAQs](https://firebase.google.com/support/faq#expandable-24)) Usage of this extension also requires you to have a Mailchimp account. You are responsible for any associated costs with your usage of Mailchimp. - diff --git a/auth-mailchimp-sync/README.md b/auth-mailchimp-sync/README.md deleted file mode 100644 index 941dd5892..000000000 --- a/auth-mailchimp-sync/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Sync with Mailchimp - -**Description**: Adds new users from Firebase Authentication to a specified Mailchimp audience. - - - -**Details**: Use this extension to add new users to an existing [Mailchimp](https://mailchimp.com) audience. - -This extension adds the email address of each new user to your specified Mailchimp audience. Also, if the user deletes their user account for your app, this extension removes the user from the Mailchimp audience. - -**Note:** To use this extension, you need to manage your users with Firebase Authentication. - -This extension uses Mailchimp, so you'll need to supply your Mailchimp API Key and Audience ID when installing this extension. - -#### Additional setup - -Make sure that you've set up [Firebase Authentication](https://firebase.google.com/docs/auth) to manage your users. - -You must also have a Mailchimp account before installing this extension. - -#### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Firebase Realtime Database -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) - -Usage of this extension also requires you to have a Mailchimp account. You are responsible for any associated costs with your usage of Mailchimp. - - - - - -**Configuration Parameters:** - -* Cloud Functions location: Where do you want to deploy the functions created for this extension? - -* Mailchimp API key: What is your Mailchimp API key? To obtain a Mailchimp API key, go to your [Mailchimp account](https://admin.mailchimp.com/account/api/). - -* Audience ID: What is the Mailchimp Audience ID to which you want to subscribe new users? To find your Audience ID: visit https://admin.mailchimp.com/lists, click on the desired audience or create a new audience, then select **Settings**. Look for **Audience ID** (for example, `27735fc60a`). - -* Contact status: When the extension adds a new user to the Mailchimp audience, what is their initial status? This value can be `subscribed` or `pending`. `subscribed` means the user can receive campaigns; `pending` means the user still needs to opt-in to receive campaigns. - - - -**Cloud Functions:** - -* **addUserToList:** Listens for new user accounts (as managed by Firebase Authentication), then automatically adds the new user to your specified MailChimp audience. - -* **removeUserFromList:** Listens for existing user accounts to be deleted (as managed by Firebase Authentication), then automatically removes them from your specified MailChimp audience. diff --git a/auth-mailchimp-sync/extension.yaml b/auth-mailchimp-sync/extension.yaml index fec27d84c..bcac90e35 100644 --- a/auth-mailchimp-sync/extension.yaml +++ b/auth-mailchimp-sync/extension.yaml @@ -49,6 +49,7 @@ resources: then automatically adds the new user to your specified MailChimp audience. properties: location: ${LOCATION} + runtime: nodejs10 eventTrigger: eventType: providers/firebase.auth/eventTypes/user.create resource: projects/${PROJECT_ID} @@ -61,6 +62,7 @@ resources: MailChimp audience. properties: location: ${LOCATION} + runtime: nodejs10 eventTrigger: eventType: providers/firebase.auth/eventTypes/user.delete resource: projects/${PROJECT_ID} diff --git a/auth-mailchimp-sync/functions/README.md b/auth-mailchimp-sync/functions/README.md index e35eab874..a7862052c 100644 --- a/auth-mailchimp-sync/functions/README.md +++ b/auth-mailchimp-sync/functions/README.md @@ -21,20 +21,18 @@ Make sure that you've set up [Firebase Authentication](https://firebase.google.c You must also have a Mailchimp account before installing this extension. #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Firebase Realtime Database -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Functions (Node.js 10+ runtime. See [FAQs](https://firebase.google.com/support/faq#expandable-24)) Usage of this extension also requires you to have a Mailchimp account. You are responsible for any associated costs with your usage of Mailchimp. - **Configuration Parameters:** * Cloud Functions location: Where do you want to deploy the functions created for this extension? diff --git a/auth-mailchimp-sync/functions/__tests__/functions.test.ts b/auth-mailchimp-sync/functions/__tests__/functions.test.ts index 182922828..7a18d1d52 100644 --- a/auth-mailchimp-sync/functions/__tests__/functions.test.ts +++ b/auth-mailchimp-sync/functions/__tests__/functions.test.ts @@ -1,5 +1,6 @@ -const consoleLogSpy = jest.spyOn(console, "log").mockImplementation(); -const consoleWarnSpy = jest.spyOn(console, "warn").mockImplementation(); +const { logger } = require("firebase-functions"); + +const consoleLogSpy = jest.spyOn(logger, "log").mockImplementation(); import functionsConfig from "../src/config"; import { obfuscatedConfig } from "../src/logs"; diff --git a/auth-mailchimp-sync/functions/lib/index.js b/auth-mailchimp-sync/functions/lib/index.js index 4f0a72057..f4d8529fe 100644 --- a/auth-mailchimp-sync/functions/lib/index.js +++ b/auth-mailchimp-sync/functions/lib/index.js @@ -14,15 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); exports.removeUserFromList = exports.addUserToList = void 0; const crypto = require("crypto"); @@ -39,7 +30,7 @@ try { catch (err) { logs.initError(err); } -exports.addUserToList = functions.handler.auth.user.onCreate((user) => __awaiter(void 0, void 0, void 0, function* () { +exports.addUserToList = functions.handler.auth.user.onCreate(async (user) => { logs.start(); if (!mailchimp) { logs.mailchimpNotInitialized(); @@ -52,7 +43,7 @@ exports.addUserToList = functions.handler.auth.user.onCreate((user) => __awaiter } try { logs.userAdding(uid, config_1.default.mailchimpAudienceId); - const results = yield mailchimp.post(`/lists/${config_1.default.mailchimpAudienceId}/members`, { + const results = await mailchimp.post(`/lists/${config_1.default.mailchimpAudienceId}/members`, { email_address: email, status: config_1.default.mailchimpContactStatus, }); @@ -62,8 +53,8 @@ exports.addUserToList = functions.handler.auth.user.onCreate((user) => __awaiter catch (err) { logs.errorAddUser(err); } -})); -exports.removeUserFromList = functions.handler.auth.user.onDelete((user) => __awaiter(void 0, void 0, void 0, function* () { +}); +exports.removeUserFromList = functions.handler.auth.user.onDelete(async (user) => { logs.start(); if (!mailchimp) { logs.mailchimpNotInitialized(); @@ -80,11 +71,11 @@ exports.removeUserFromList = functions.handler.auth.user.onDelete((user) => __aw .update(email) .digest("hex"); logs.userRemoving(uid, hashed, config_1.default.mailchimpAudienceId); - yield mailchimp.delete(`/lists/${config_1.default.mailchimpAudienceId}/members/${hashed}`); + await mailchimp.delete(`/lists/${config_1.default.mailchimpAudienceId}/members/${hashed}`); logs.userRemoved(uid, hashed, config_1.default.mailchimpAudienceId); logs.complete(); } catch (err) { logs.errorRemoveUser(err); } -})); +}); diff --git a/auth-mailchimp-sync/functions/lib/logs.js b/auth-mailchimp-sync/functions/lib/logs.js index 73f585660..455df1c77 100644 --- a/auth-mailchimp-sync/functions/lib/logs.js +++ b/auth-mailchimp-sync/functions/lib/logs.js @@ -16,41 +16,45 @@ */ Object.defineProperty(exports, "__esModule", { value: true }); exports.userRemoving = exports.userRemoved = exports.userNoEmail = exports.userAdding = exports.userAdded = exports.start = exports.mailchimpNotInitialized = exports.initError = exports.init = exports.errorRemoveUser = exports.errorAddUser = exports.complete = exports.obfuscatedConfig = void 0; +const { logger } = require("firebase-functions"); const config_1 = require("./config"); -exports.obfuscatedConfig = Object.assign(Object.assign({}, config_1.default), { mailchimpApiKey: "" }); +exports.obfuscatedConfig = { + ...config_1.default, + mailchimpApiKey: "", +}; exports.complete = () => { - console.log("Completed execution of extension"); + logger.log("Completed execution of extension"); }; exports.errorAddUser = (err) => { - console.error("Error when adding user to Mailchimp audience", err); + logger.error("Error when adding user to Mailchimp audience", err); }; exports.errorRemoveUser = (err) => { - console.error("Error when removing user from Mailchimp audience", err); + logger.error("Error when removing user from Mailchimp audience", err); }; exports.init = () => { - console.log("Initializing extension with configuration", exports.obfuscatedConfig); + logger.log("Initializing extension with configuration", exports.obfuscatedConfig); }; exports.initError = (err) => { - console.error("Error when initializing extension", err); + logger.error("Error when initializing extension", err); }; exports.mailchimpNotInitialized = () => { - console.error("Mailchimp was not initialized correctly, check for errors in the logs"); + logger.error("Mailchimp was not initialized correctly, check for errors in the logs"); }; exports.start = () => { - console.log("Started execution of extension with configuration", exports.obfuscatedConfig); + logger.log("Started execution of extension with configuration", exports.obfuscatedConfig); }; exports.userAdded = (userId, audienceId, mailchimpId, status) => { - console.log(`Added user: ${userId} with status '${status}' to Mailchimp audience: ${audienceId} with Mailchimp ID: ${mailchimpId}`); + logger.log(`Added user: ${userId} with status '${status}' to Mailchimp audience: ${audienceId} with Mailchimp ID: ${mailchimpId}`); }; exports.userAdding = (userId, audienceId) => { - console.log(`Adding user: ${userId} to Mailchimp audience: ${audienceId}`); + logger.log(`Adding user: ${userId} to Mailchimp audience: ${audienceId}`); }; exports.userNoEmail = () => { - console.log("User does not have an email"); + logger.log("User does not have an email"); }; exports.userRemoved = (userId, hashedEmail, audienceId) => { - console.log(`Removed user: ${userId} with hashed email: ${hashedEmail} from Mailchimp audience: ${audienceId}`); + logger.log(`Removed user: ${userId} with hashed email: ${hashedEmail} from Mailchimp audience: ${audienceId}`); }; exports.userRemoving = (userId, hashedEmail, audienceId) => { - console.log(`Removing user: ${userId} with hashed email: ${hashedEmail} from Mailchimp audience: ${audienceId}`); + logger.log(`Removing user: ${userId} with hashed email: ${hashedEmail} from Mailchimp audience: ${audienceId}`); }; diff --git a/auth-mailchimp-sync/functions/src/logs.ts b/auth-mailchimp-sync/functions/src/logs.ts index ff5e29d13..94adbab2a 100644 --- a/auth-mailchimp-sync/functions/src/logs.ts +++ b/auth-mailchimp-sync/functions/src/logs.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { logger } from "firebase-functions"; import config from "./config"; export const obfuscatedConfig = { @@ -22,33 +23,33 @@ export const obfuscatedConfig = { }; export const complete = () => { - console.log("Completed execution of extension"); + logger.log("Completed execution of extension"); }; export const errorAddUser = (err: Error) => { - console.error("Error when adding user to Mailchimp audience", err); + logger.error("Error when adding user to Mailchimp audience", err); }; export const errorRemoveUser = (err: Error) => { - console.error("Error when removing user from Mailchimp audience", err); + logger.error("Error when removing user from Mailchimp audience", err); }; export const init = () => { - console.log("Initializing extension with configuration", obfuscatedConfig); + logger.log("Initializing extension with configuration", obfuscatedConfig); }; export const initError = (err: Error) => { - console.error("Error when initializing extension", err); + logger.error("Error when initializing extension", err); }; export const mailchimpNotInitialized = () => { - console.error( + logger.error( "Mailchimp was not initialized correctly, check for errors in the logs" ); }; export const start = () => { - console.log( + logger.log( "Started execution of extension with configuration", obfuscatedConfig ); @@ -60,17 +61,17 @@ export const userAdded = ( mailchimpId: string, status: string ) => { - console.log( + logger.log( `Added user: ${userId} with status '${status}' to Mailchimp audience: ${audienceId} with Mailchimp ID: ${mailchimpId}` ); }; export const userAdding = (userId: string, audienceId: string) => { - console.log(`Adding user: ${userId} to Mailchimp audience: ${audienceId}`); + logger.log(`Adding user: ${userId} to Mailchimp audience: ${audienceId}`); }; export const userNoEmail = () => { - console.log("User does not have an email"); + logger.log("User does not have an email"); }; export const userRemoved = ( @@ -78,7 +79,7 @@ export const userRemoved = ( hashedEmail: string, audienceId: string ) => { - console.log( + logger.log( `Removed user: ${userId} with hashed email: ${hashedEmail} from Mailchimp audience: ${audienceId}` ); }; @@ -88,7 +89,7 @@ export const userRemoving = ( hashedEmail: string, audienceId: string ) => { - console.log( + logger.log( `Removing user: ${userId} with hashed email: ${hashedEmail} from Mailchimp audience: ${audienceId}` ); }; diff --git a/auth-mailchimp-sync/functions/tsconfig.json b/auth-mailchimp-sync/functions/tsconfig.json index 1b551f4e9..9f7e58b1e 100644 --- a/auth-mailchimp-sync/functions/tsconfig.json +++ b/auth-mailchimp-sync/functions/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "outDir": "lib" + "outDir": "lib", + "target": "es2018" }, "include": ["src"] } From 90405802276a54de26072ae67019df1948c1f420 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 24 Aug 2020 17:33:57 +0100 Subject: [PATCH 15/24] chore(firestore-shorten-urls-bitly): migration to Node.js v10 (#432) --- firestore-shorten-urls-bitly/PREINSTALL.md | 18 +-- firestore-shorten-urls-bitly/extension.yaml | 1 + .../functions/README.md | 18 +-- .../functions/lib/abstract-shortener.js | 125 ++++++++---------- .../functions/lib/index.js | 43 +++--- .../functions/lib/logs.js | 38 +++--- .../functions/package.json | 2 +- .../functions/src/logs.ts | 33 ++--- .../functions/tsconfig.json | 3 +- 9 files changed, 130 insertions(+), 151 deletions(-) diff --git a/firestore-shorten-urls-bitly/PREINSTALL.md b/firestore-shorten-urls-bitly/PREINSTALL.md index b6ea59fee..1e4d7a206 100644 --- a/firestore-shorten-urls-bitly/PREINSTALL.md +++ b/firestore-shorten-urls-bitly/PREINSTALL.md @@ -16,12 +16,12 @@ Before installing this extension, make sure that you've [set up a Cloud Firestor You must also have a Bitly account and access token before installing this extension. #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) - -Usage of this extension also requires you to have a Bitly account. You are responsible for any associated costs with your usage of Bitly. + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Firestore + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) +- This extension also uses these services: + - [Bitly](https://bitly.com/). You must have a Bitly account and you're responsible for any associated charges. diff --git a/firestore-shorten-urls-bitly/extension.yaml b/firestore-shorten-urls-bitly/extension.yaml index f661f7468..60032debe 100644 --- a/firestore-shorten-urls-bitly/extension.yaml +++ b/firestore-shorten-urls-bitly/extension.yaml @@ -54,6 +54,7 @@ resources: then writes the shortened form back to the same document. properties: location: ${LOCATION} + runtime: nodejs10 eventTrigger: eventType: providers/cloud.firestore/eventTypes/document.write resource: projects/${PROJECT_ID}/databases/(default)/documents/${COLLECTION_PATH}/{documentId} diff --git a/firestore-shorten-urls-bitly/functions/README.md b/firestore-shorten-urls-bitly/functions/README.md index cf1b70abf..a400eb4b7 100644 --- a/firestore-shorten-urls-bitly/functions/README.md +++ b/firestore-shorten-urls-bitly/functions/README.md @@ -24,15 +24,15 @@ Before installing this extension, make sure that you've [set up a Cloud Firestor You must also have a Bitly account and access token before installing this extension. #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Firestore -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) - -Usage of this extension also requires you to have a Bitly account. You are responsible for any associated costs with your usage of Bitly. + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Firestore + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) +- This extension also uses these services: + - [Bitly](https://bitly.com/). You must have a Bitly account and you're responsible for any associated charges. diff --git a/firestore-shorten-urls-bitly/functions/lib/abstract-shortener.js b/firestore-shorten-urls-bitly/functions/lib/abstract-shortener.js index 5273e0ef2..9b6b1af47 100644 --- a/firestore-shorten-urls-bitly/functions/lib/abstract-shortener.js +++ b/firestore-shorten-urls-bitly/functions/lib/abstract-shortener.js @@ -14,15 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); exports.FirestoreUrlShortener = void 0; const admin = require("firebase-admin"); @@ -43,30 +34,28 @@ class FirestoreUrlShortener { // Initialize the Firebase Admin SDK admin.initializeApp(); } - onDocumentWrite(change) { - return __awaiter(this, void 0, void 0, function* () { - this.logs.start(); - if (this.urlFieldName === this.shortUrlFieldName) { - this.logs.fieldNamesNotDifferent(); - return; - } - const changeType = this.getChangeType(change); - switch (changeType) { - case ChangeType.CREATE: - yield this.handleCreateDocument(change.after); - break; - case ChangeType.DELETE: - this.handleDeleteDocument(); - break; - case ChangeType.UPDATE: - yield this.handleUpdateDocument(change.before, change.after); - break; - default: { - throw new Error(`Invalid change type: ${changeType}`); - } + async onDocumentWrite(change) { + this.logs.start(); + if (this.urlFieldName === this.shortUrlFieldName) { + this.logs.fieldNamesNotDifferent(); + return; + } + const changeType = this.getChangeType(change); + switch (changeType) { + case ChangeType.CREATE: + await this.handleCreateDocument(change.after); + break; + case ChangeType.DELETE: + this.handleDeleteDocument(); + break; + case ChangeType.UPDATE: + await this.handleUpdateDocument(change.before, change.after); + break; + default: { + throw new Error(`Invalid change type: ${changeType}`); } - this.logs.complete(); - }); + } + this.logs.complete(); } extractUrl(snapshot) { return snapshot.get(this.urlFieldName); @@ -80,51 +69,45 @@ class FirestoreUrlShortener { } return ChangeType.UPDATE; } - handleCreateDocument(snapshot) { - return __awaiter(this, void 0, void 0, function* () { - const url = this.extractUrl(snapshot); - if (url) { - this.logs.documentCreatedWithUrl(); - yield this.shortenUrl(snapshot); - } - else { - this.logs.documentCreatedNoUrl(); - } - }); + async handleCreateDocument(snapshot) { + const url = this.extractUrl(snapshot); + if (url) { + this.logs.documentCreatedWithUrl(); + await this.shortenUrl(snapshot); + } + else { + this.logs.documentCreatedNoUrl(); + } } handleDeleteDocument() { this.logs.documentDeleted(); } - handleUpdateDocument(before, after) { - return __awaiter(this, void 0, void 0, function* () { - const urlAfter = this.extractUrl(after); - const urlBefore = this.extractUrl(before); - if (urlAfter === urlBefore) { - this.logs.documentUpdatedUnchangedUrl(); - } - else if (urlAfter) { - this.logs.documentUpdatedChangedUrl(); - yield this.shortenUrl(after); - } - else if (urlBefore) { - this.logs.documentUpdatedDeletedUrl(); - yield this.updateShortUrl(after, admin.firestore.FieldValue.delete()); - } - else { - this.logs.documentUpdatedNoUrl(); - } - }); + async handleUpdateDocument(before, after) { + const urlAfter = this.extractUrl(after); + const urlBefore = this.extractUrl(before); + if (urlAfter === urlBefore) { + this.logs.documentUpdatedUnchangedUrl(); + } + else if (urlAfter) { + this.logs.documentUpdatedChangedUrl(); + await this.shortenUrl(after); + } + else if (urlBefore) { + this.logs.documentUpdatedDeletedUrl(); + await this.updateShortUrl(after, admin.firestore.FieldValue.delete()); + } + else { + this.logs.documentUpdatedNoUrl(); + } } - updateShortUrl(snapshot, url) { - return __awaiter(this, void 0, void 0, function* () { - this.logs.updateDocument(snapshot.ref.path); - // Wrapping in transaction to allow for automatic retries (#48) - yield admin.firestore().runTransaction((transaction) => { - transaction.update(snapshot.ref, this.shortUrlFieldName, url); - return Promise.resolve(); - }); - this.logs.updateDocumentComplete(snapshot.ref.path); + async updateShortUrl(snapshot, url) { + this.logs.updateDocument(snapshot.ref.path); + // Wrapping in transaction to allow for automatic retries (#48) + await admin.firestore().runTransaction((transaction) => { + transaction.update(snapshot.ref, this.shortUrlFieldName, url); + return Promise.resolve(); }); + this.logs.updateDocumentComplete(snapshot.ref.path); } } exports.FirestoreUrlShortener = FirestoreUrlShortener; diff --git a/firestore-shorten-urls-bitly/functions/lib/index.js b/firestore-shorten-urls-bitly/functions/lib/index.js index 3c0d95304..5dbd0e14d 100644 --- a/firestore-shorten-urls-bitly/functions/lib/index.js +++ b/firestore-shorten-urls-bitly/functions/lib/index.js @@ -14,15 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); exports.fsurlshortener = void 0; const functions = require("firebase-functions"); @@ -42,25 +33,23 @@ class FirestoreBitlyUrlShortener extends abstract_shortener_1.FirestoreUrlShorte }); logs.init(); } - shortenUrl(snapshot) { - return __awaiter(this, void 0, void 0, function* () { - const url = this.extractUrl(snapshot); - logs.shortenUrl(url); - try { - const response = yield this.instance.post("bitlinks", { - long_url: url, - }); - const { link } = response.data; - logs.shortenUrlComplete(link); - yield this.updateShortUrl(snapshot, link); - } - catch (err) { - logs.error(err); - } - }); + async shortenUrl(snapshot) { + const url = this.extractUrl(snapshot); + logs.shortenUrl(url); + try { + const response = await this.instance.post("bitlinks", { + long_url: url, + }); + const { link } = response.data; + logs.shortenUrlComplete(link); + await this.updateShortUrl(snapshot, link); + } + catch (err) { + logs.error(err); + } } } const urlShortener = new FirestoreBitlyUrlShortener(config_1.default.urlFieldName, config_1.default.shortUrlFieldName, config_1.default.bitlyAccessToken); -exports.fsurlshortener = functions.handler.firestore.document.onWrite((change) => __awaiter(void 0, void 0, void 0, function* () { +exports.fsurlshortener = functions.handler.firestore.document.onWrite(async (change) => { return urlShortener.onDocumentWrite(change); -})); +}); diff --git a/firestore-shorten-urls-bitly/functions/lib/logs.js b/firestore-shorten-urls-bitly/functions/lib/logs.js index 8737f6f9f..90dd05210 100644 --- a/firestore-shorten-urls-bitly/functions/lib/logs.js +++ b/firestore-shorten-urls-bitly/functions/lib/logs.js @@ -16,53 +16,57 @@ */ Object.defineProperty(exports, "__esModule", { value: true }); exports.updateDocumentComplete = exports.updateDocument = exports.start = exports.shortenUrlComplete = exports.shortenUrl = exports.init = exports.fieldNamesNotDifferent = exports.error = exports.documentUpdatedUnchangedUrl = exports.documentUpdatedNoUrl = exports.documentUpdatedDeletedUrl = exports.documentUpdatedChangedUrl = exports.documentDeleted = exports.documentCreatedWithUrl = exports.documentCreatedNoUrl = exports.complete = void 0; +const firebase_functions_1 = require("firebase-functions"); const config_1 = require("./config"); -const obfuscatedConfig = Object.assign(Object.assign({}, config_1.default), { bitlyAccessToken: "********" }); +const obfuscatedConfig = { + ...config_1.default, + bitlyAccessToken: "********", +}; exports.complete = () => { - console.log("Completed execution of extension"); + firebase_functions_1.logger.log("Completed execution of extension"); }; exports.documentCreatedNoUrl = () => { - console.log("Document was created without a URL, no processing is required"); + firebase_functions_1.logger.log("Document was created without a URL, no processing is required"); }; exports.documentCreatedWithUrl = () => { - console.log("Document was created with a URL"); + firebase_functions_1.logger.log("Document was created with a URL"); }; exports.documentDeleted = () => { - console.log("Document was deleted, no processing is required"); + firebase_functions_1.logger.log("Document was deleted, no processing is required"); }; exports.documentUpdatedChangedUrl = () => { - console.log("Document was updated, URL has changed"); + firebase_functions_1.logger.log("Document was updated, URL has changed"); }; exports.documentUpdatedDeletedUrl = () => { - console.log("Document was updated, URL was deleted"); + firebase_functions_1.logger.log("Document was updated, URL was deleted"); }; exports.documentUpdatedNoUrl = () => { - console.log("Document was updated, no URL exists, no processing is required"); + firebase_functions_1.logger.log("Document was updated, no URL exists, no processing is required"); }; exports.documentUpdatedUnchangedUrl = () => { - console.log("Document was updated, URL has not changed, no processing is required"); + firebase_functions_1.logger.log("Document was updated, URL has not changed, no processing is required"); }; exports.error = (err) => { - console.error("Error when shortening URL", err); + firebase_functions_1.logger.error("Error when shortening URL", err); }; exports.fieldNamesNotDifferent = () => { - console.error("The `URL` and `Short URL` field names must be different for this extension to function correctly"); + firebase_functions_1.logger.error("The `URL` and `Short URL` field names must be different for this extension to function correctly"); }; exports.init = () => { - console.log("Initializing extension with configuration", obfuscatedConfig); + firebase_functions_1.logger.log("Initializing extension with configuration", obfuscatedConfig); }; exports.shortenUrl = (url) => { - console.log(`Shortening URL: '${url}'`); + firebase_functions_1.logger.log(`Shortening URL: '${url}'`); }; exports.shortenUrlComplete = (shortUrl) => { - console.log(`Finished shortening URL to: '${shortUrl}'`); + firebase_functions_1.logger.log(`Finished shortening URL to: '${shortUrl}'`); }; exports.start = () => { - console.log("Started execution of extension with configuration", obfuscatedConfig); + firebase_functions_1.logger.log("Started execution of extension with configuration", obfuscatedConfig); }; exports.updateDocument = (path) => { - console.log(`Updating Cloud Firestore document: '${path}'`); + firebase_functions_1.logger.log(`Updating Cloud Firestore document: '${path}'`); }; exports.updateDocumentComplete = (path) => { - console.log(`Finished updating Cloud Firestore document: '${path}'`); + firebase_functions_1.logger.log(`Finished updating Cloud Firestore document: '${path}'`); }; diff --git a/firestore-shorten-urls-bitly/functions/package.json b/firestore-shorten-urls-bitly/functions/package.json index f7177f3f7..3ada4e185 100644 --- a/firestore-shorten-urls-bitly/functions/package.json +++ b/firestore-shorten-urls-bitly/functions/package.json @@ -14,7 +14,7 @@ "dependencies": { "axios": "^0.19.1", "firebase-admin": "^8.0.0", - "firebase-functions": "^3.3.0" + "firebase-functions": "^3.7.0" }, "devDependencies": { "rimraf": "^2.6.3", diff --git a/firestore-shorten-urls-bitly/functions/src/logs.ts b/firestore-shorten-urls-bitly/functions/src/logs.ts index 2705e99a1..04e2940ee 100644 --- a/firestore-shorten-urls-bitly/functions/src/logs.ts +++ b/firestore-shorten-urls-bitly/functions/src/logs.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { logger } from "firebase-functions"; import config from "./config"; const obfuscatedConfig = { @@ -22,72 +23,72 @@ const obfuscatedConfig = { }; export const complete = () => { - console.log("Completed execution of extension"); + logger.log("Completed execution of extension"); }; export const documentCreatedNoUrl = () => { - console.log("Document was created without a URL, no processing is required"); + logger.log("Document was created without a URL, no processing is required"); }; export const documentCreatedWithUrl = () => { - console.log("Document was created with a URL"); + logger.log("Document was created with a URL"); }; export const documentDeleted = () => { - console.log("Document was deleted, no processing is required"); + logger.log("Document was deleted, no processing is required"); }; export const documentUpdatedChangedUrl = () => { - console.log("Document was updated, URL has changed"); + logger.log("Document was updated, URL has changed"); }; export const documentUpdatedDeletedUrl = () => { - console.log("Document was updated, URL was deleted"); + logger.log("Document was updated, URL was deleted"); }; export const documentUpdatedNoUrl = () => { - console.log("Document was updated, no URL exists, no processing is required"); + logger.log("Document was updated, no URL exists, no processing is required"); }; export const documentUpdatedUnchangedUrl = () => { - console.log( + logger.log( "Document was updated, URL has not changed, no processing is required" ); }; export const error = (err: Error) => { - console.error("Error when shortening URL", err); + logger.error("Error when shortening URL", err); }; export const fieldNamesNotDifferent = () => { - console.error( + logger.error( "The `URL` and `Short URL` field names must be different for this extension to function correctly" ); }; export const init = () => { - console.log("Initializing extension with configuration", obfuscatedConfig); + logger.log("Initializing extension with configuration", obfuscatedConfig); }; export const shortenUrl = (url: string) => { - console.log(`Shortening URL: '${url}'`); + logger.log(`Shortening URL: '${url}'`); }; export const shortenUrlComplete = (shortUrl: string) => { - console.log(`Finished shortening URL to: '${shortUrl}'`); + logger.log(`Finished shortening URL to: '${shortUrl}'`); }; export const start = () => { - console.log( + logger.log( "Started execution of extension with configuration", obfuscatedConfig ); }; export const updateDocument = (path: string) => { - console.log(`Updating Cloud Firestore document: '${path}'`); + logger.log(`Updating Cloud Firestore document: '${path}'`); }; export const updateDocumentComplete = (path: string) => { - console.log(`Finished updating Cloud Firestore document: '${path}'`); + logger.log(`Finished updating Cloud Firestore document: '${path}'`); }; diff --git a/firestore-shorten-urls-bitly/functions/tsconfig.json b/firestore-shorten-urls-bitly/functions/tsconfig.json index 1b551f4e9..9f7e58b1e 100644 --- a/firestore-shorten-urls-bitly/functions/tsconfig.json +++ b/firestore-shorten-urls-bitly/functions/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "outDir": "lib" + "outDir": "lib", + "target": "es2018" }, "include": ["src"] } From f4846245f69e7e1a7380b4c1124800e05018c953 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 24 Aug 2020 17:34:09 +0100 Subject: [PATCH 16/24] chore(storage-resize-images): migration to Node.js v10 (#433) --- storage-resize-images/PREINSTALL.md | 14 +++--- storage-resize-images/README.md | 14 +++--- storage-resize-images/extension.yaml | 3 +- storage-resize-images/functions/lib/index.js | 29 +++++-------- storage-resize-images/functions/lib/logs.js | 45 ++++++++++---------- storage-resize-images/functions/src/logs.ts | 45 ++++++++++---------- storage-resize-images/package.json | 2 +- storage-resize-images/tsconfig.json | 3 +- 8 files changed, 75 insertions(+), 80 deletions(-) diff --git a/storage-resize-images/PREINSTALL.md b/storage-resize-images/PREINSTALL.md index 6f796cd29..fe37ed41e 100644 --- a/storage-resize-images/PREINSTALL.md +++ b/storage-resize-images/PREINSTALL.md @@ -21,10 +21,10 @@ For example, say that you specify a max width of 200px and a max height of 100px Before installing this extension, make sure that you've [set up a Cloud Storage bucket](https://firebase.google.com/docs/storage) in your Firebase project. #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Storage -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Storage + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/storage-resize-images/README.md b/storage-resize-images/README.md index 193205f7d..e5af29756 100644 --- a/storage-resize-images/README.md +++ b/storage-resize-images/README.md @@ -29,13 +29,13 @@ For example, say that you specify a max width of 200px and a max height of 100px Before installing this extension, make sure that you've [set up a Cloud Storage bucket](https://firebase.google.com/docs/storage) in your Firebase project. #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Storage -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Storage + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/storage-resize-images/extension.yaml b/storage-resize-images/extension.yaml index 1c2014139..ed4844646 100644 --- a/storage-resize-images/extension.yaml +++ b/storage-resize-images/extension.yaml @@ -39,7 +39,7 @@ contributors: email: mike@invertase.io url: https://github.com/salakar -billingRequired: false +billingRequired: true apis: - apiName: storage-component.googleapis.com @@ -58,6 +58,7 @@ resources: properties: sourceDirectory: . location: ${LOCATION} + runtime: nodejs10 availableMemoryMb: 1024 eventTrigger: eventType: google.storage.object.finalize diff --git a/storage-resize-images/functions/lib/index.js b/storage-resize-images/functions/lib/index.js index e58a98dd9..5ed9a86b0 100644 --- a/storage-resize-images/functions/lib/index.js +++ b/storage-resize-images/functions/lib/index.js @@ -14,15 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); exports.generateResizedImage = void 0; const admin = require("firebase-admin"); @@ -53,7 +44,7 @@ const supportedContentTypes = [ * When an image is uploaded in the Storage bucket, we generate a resized image automatically using * the Sharp image converting library. */ -exports.generateResizedImage = functions.storage.object().onFinalize((object) => __awaiter(void 0, void 0, void 0, function* () { +exports.generateResizedImage = functions.storage.object().onFinalize(async (object) => { logs.start(); const { contentType } = object; // This is the image MIME type if (!contentType) { @@ -85,12 +76,12 @@ exports.generateResizedImage = functions.storage.object().onFinalize((object) => const tempLocalDir = path.dirname(originalFile); // Create the temp directory where the storage file will be downloaded. logs.tempDirectoryCreating(tempLocalDir); - yield mkdirp(tempLocalDir); + await mkdirp(tempLocalDir); logs.tempDirectoryCreated(tempLocalDir); // Download file from bucket. remoteFile = bucket.file(filePath); logs.imageDownloading(filePath); - yield remoteFile.download({ destination: originalFile }); + await remoteFile.download({ destination: originalFile }); logs.imageDownloaded(filePath, originalFile); // Convert to a set to remove any duplicate sizes const imageSizes = new Set(config_1.default.imageSizes); @@ -107,7 +98,7 @@ exports.generateResizedImage = functions.storage.object().onFinalize((object) => objectMetadata: objectMetadata, })); }); - const results = yield Promise.all(tasks); + const results = await Promise.all(tasks); const failed = results.some((result) => result.success === false); if (failed) { logs.failed(); @@ -129,7 +120,7 @@ exports.generateResizedImage = functions.storage.object().onFinalize((object) => if (remoteFile) { try { logs.remoteFileDeleting(filePath); - yield remoteFile.delete(); + await remoteFile.delete(); logs.remoteFileDeleted(filePath); } catch (err) { @@ -138,7 +129,7 @@ exports.generateResizedImage = functions.storage.object().onFinalize((object) => } } } -})); +}); function resize(originalFile, resizedFile, size) { let height, width; if (size.indexOf(",") !== -1) { @@ -158,7 +149,7 @@ function resize(originalFile, resizedFile, size) { }) .toFile(resizedFile); } -const resizeImage = ({ bucket, originalFile, fileDir, fileNameWithoutExtension, fileExtension, contentType, size, objectMetadata, }) => __awaiter(void 0, void 0, void 0, function* () { +const resizeImage = async ({ bucket, originalFile, fileDir, fileNameWithoutExtension, fileExtension, contentType, size, objectMetadata, }) => { const resizedFileName = `${fileNameWithoutExtension}_${size}${fileExtension}`; // Path where resized image will be uploaded to in Storage. const resizedFilePath = path.normalize(config_1.default.resizedImagesPath @@ -189,11 +180,11 @@ const resizeImage = ({ bucket, originalFile, fileDir, fileNameWithoutExtension, } // Generate a resized image using Sharp. logs.imageResizing(resizedFile, size); - yield resize(originalFile, resizedFile, size); + await resize(originalFile, resizedFile, size); logs.imageResized(resizedFile); // Uploading the resized image. logs.imageUploading(resizedFilePath); - yield bucket.upload(resizedFile, { + await bucket.upload(resizedFile, { destination: resizedFilePath, metadata, }); @@ -217,4 +208,4 @@ const resizeImage = ({ bucket, originalFile, fileDir, fileNameWithoutExtension, logs.errorDeleting(err); } } -}); +}; diff --git a/storage-resize-images/functions/lib/logs.js b/storage-resize-images/functions/lib/logs.js index be76cdc8f..03f2417da 100644 --- a/storage-resize-images/functions/lib/logs.js +++ b/storage-resize-images/functions/lib/logs.js @@ -16,18 +16,19 @@ */ Object.defineProperty(exports, "__esModule", { value: true }); exports.remoteFileDeleting = exports.remoteFileDeleted = exports.tempResizedFileDeleting = exports.tempResizedFileDeleted = exports.tempOriginalFileDeleting = exports.tempOriginalFileDeleted = exports.tempDirectoryCreating = exports.tempDirectoryCreated = exports.start = exports.init = exports.imageUploading = exports.imageUploaded = exports.imageResizing = exports.imageResized = exports.imageDownloading = exports.imageDownloaded = exports.imageAlreadyResized = exports.failed = exports.errorDeleting = exports.error = exports.unsupportedType = exports.contentTypeInvalid = exports.noContentType = exports.complete = void 0; +const firebase_functions_1 = require("firebase-functions"); const config_1 = require("./config"); exports.complete = () => { - console.log("Completed execution of extension"); + firebase_functions_1.logger.log("Completed execution of extension"); }; exports.noContentType = () => { - console.log(`File has no Content-Type, no processing is required`); + firebase_functions_1.logger.log(`File has no Content-Type, no processing is required`); }; exports.contentTypeInvalid = (contentType) => { - console.log(`File of type '${contentType}' is not an image, no processing is required`); + firebase_functions_1.logger.log(`File of type '${contentType}' is not an image, no processing is required`); }; exports.unsupportedType = (unsupportedTypes, contentType) => { - console.log(`Image type '${contentType}' is not supported, here are the supported file types: ${unsupportedTypes.join(", ")}`); + firebase_functions_1.logger.log(`Image type '${contentType}' is not supported, here are the supported file types: ${unsupportedTypes.join(", ")}`); }; exports.error = (err) => { console.error("Error when resizing image", err); @@ -36,56 +37,56 @@ exports.errorDeleting = (err) => { console.warn("Error when deleting temporary files", err); }; exports.failed = () => { - console.log("Failed execution of extension"); + firebase_functions_1.logger.log("Failed execution of extension"); }; exports.imageAlreadyResized = () => { - console.log("File is already a resized image, no processing is required"); + firebase_functions_1.logger.log("File is already a resized image, no processing is required"); }; exports.imageDownloaded = (remotePath, localPath) => { - console.log(`Downloaded image file: '${remotePath}' to '${localPath}'`); + firebase_functions_1.logger.log(`Downloaded image file: '${remotePath}' to '${localPath}'`); }; exports.imageDownloading = (path) => { - console.log(`Downloading image file: '${path}'`); + firebase_functions_1.logger.log(`Downloading image file: '${path}'`); }; exports.imageResized = (path) => { - console.log(`Resized image created at '${path}'`); + firebase_functions_1.logger.log(`Resized image created at '${path}'`); }; exports.imageResizing = (path, size) => { - console.log(`Resizing image at path '${path}' to size: ${size}`); + firebase_functions_1.logger.log(`Resizing image at path '${path}' to size: ${size}`); }; exports.imageUploaded = (path) => { - console.log(`Uploaded resized image to '${path}'`); + firebase_functions_1.logger.log(`Uploaded resized image to '${path}'`); }; exports.imageUploading = (path) => { - console.log(`Uploading resized image to '${path}'`); + firebase_functions_1.logger.log(`Uploading resized image to '${path}'`); }; exports.init = () => { - console.log("Initializing extension with configuration", config_1.default); + firebase_functions_1.logger.log("Initializing extension with configuration", config_1.default); }; exports.start = () => { - console.log("Started execution of extension with configuration", config_1.default); + firebase_functions_1.logger.log("Started execution of extension with configuration", config_1.default); }; exports.tempDirectoryCreated = (directory) => { - console.log(`Created temporary directory: '${directory}'`); + firebase_functions_1.logger.log(`Created temporary directory: '${directory}'`); }; exports.tempDirectoryCreating = (directory) => { - console.log(`Creating temporary directory: '${directory}'`); + firebase_functions_1.logger.log(`Creating temporary directory: '${directory}'`); }; exports.tempOriginalFileDeleted = (path) => { - console.log(`Deleted temporary original file: '${path}'`); + firebase_functions_1.logger.log(`Deleted temporary original file: '${path}'`); }; exports.tempOriginalFileDeleting = (path) => { - console.log(`Deleting temporary original file: '${path}'`); + firebase_functions_1.logger.log(`Deleting temporary original file: '${path}'`); }; exports.tempResizedFileDeleted = (path) => { - console.log(`Deleted temporary resized file: '${path}'`); + firebase_functions_1.logger.log(`Deleted temporary resized file: '${path}'`); }; exports.tempResizedFileDeleting = (path) => { - console.log(`Deleting temporary resized file: '${path}'`); + firebase_functions_1.logger.log(`Deleting temporary resized file: '${path}'`); }; exports.remoteFileDeleted = (path) => { - console.log(`Deleted original file from storage bucket: '${path}'`); + firebase_functions_1.logger.log(`Deleted original file from storage bucket: '${path}'`); }; exports.remoteFileDeleting = (path) => { - console.log(`Deleting original file from storage bucket: '${path}'`); + firebase_functions_1.logger.log(`Deleting original file from storage bucket: '${path}'`); }; diff --git a/storage-resize-images/functions/src/logs.ts b/storage-resize-images/functions/src/logs.ts index 9c0b0ce45..2edd2b36f 100644 --- a/storage-resize-images/functions/src/logs.ts +++ b/storage-resize-images/functions/src/logs.ts @@ -14,18 +14,19 @@ * limitations under the License. */ +import { logger } from "firebase-functions"; import config from "./config"; export const complete = () => { - console.log("Completed execution of extension"); + logger.log("Completed execution of extension"); }; export const noContentType = () => { - console.log(`File has no Content-Type, no processing is required`); + logger.log(`File has no Content-Type, no processing is required`); }; export const contentTypeInvalid = (contentType: string) => { - console.log( + logger.log( `File of type '${contentType}' is not an image, no processing is required` ); }; @@ -34,7 +35,7 @@ export const unsupportedType = ( unsupportedTypes: string[], contentType: string ) => { - console.log( + logger.log( `Image type '${contentType}' is not supported, here are the supported file types: ${unsupportedTypes.join( ", " )}` @@ -50,73 +51,73 @@ export const errorDeleting = (err: Error) => { }; export const failed = () => { - console.log("Failed execution of extension"); + logger.log("Failed execution of extension"); }; export const imageAlreadyResized = () => { - console.log("File is already a resized image, no processing is required"); + logger.log("File is already a resized image, no processing is required"); }; export const imageDownloaded = (remotePath: string, localPath: string) => { - console.log(`Downloaded image file: '${remotePath}' to '${localPath}'`); + logger.log(`Downloaded image file: '${remotePath}' to '${localPath}'`); }; export const imageDownloading = (path: string) => { - console.log(`Downloading image file: '${path}'`); + logger.log(`Downloading image file: '${path}'`); }; export const imageResized = (path: string) => { - console.log(`Resized image created at '${path}'`); + logger.log(`Resized image created at '${path}'`); }; export const imageResizing = (path: string, size: string) => { - console.log(`Resizing image at path '${path}' to size: ${size}`); + logger.log(`Resizing image at path '${path}' to size: ${size}`); }; export const imageUploaded = (path: string) => { - console.log(`Uploaded resized image to '${path}'`); + logger.log(`Uploaded resized image to '${path}'`); }; export const imageUploading = (path: string) => { - console.log(`Uploading resized image to '${path}'`); + logger.log(`Uploading resized image to '${path}'`); }; export const init = () => { - console.log("Initializing extension with configuration", config); + logger.log("Initializing extension with configuration", config); }; export const start = () => { - console.log("Started execution of extension with configuration", config); + logger.log("Started execution of extension with configuration", config); }; export const tempDirectoryCreated = (directory: string) => { - console.log(`Created temporary directory: '${directory}'`); + logger.log(`Created temporary directory: '${directory}'`); }; export const tempDirectoryCreating = (directory: string) => { - console.log(`Creating temporary directory: '${directory}'`); + logger.log(`Creating temporary directory: '${directory}'`); }; export const tempOriginalFileDeleted = (path: string) => { - console.log(`Deleted temporary original file: '${path}'`); + logger.log(`Deleted temporary original file: '${path}'`); }; export const tempOriginalFileDeleting = (path: string) => { - console.log(`Deleting temporary original file: '${path}'`); + logger.log(`Deleting temporary original file: '${path}'`); }; export const tempResizedFileDeleted = (path: string) => { - console.log(`Deleted temporary resized file: '${path}'`); + logger.log(`Deleted temporary resized file: '${path}'`); }; export const tempResizedFileDeleting = (path: string) => { - console.log(`Deleting temporary resized file: '${path}'`); + logger.log(`Deleting temporary resized file: '${path}'`); }; export const remoteFileDeleted = (path: string) => { - console.log(`Deleted original file from storage bucket: '${path}'`); + logger.log(`Deleted original file from storage bucket: '${path}'`); }; export const remoteFileDeleting = (path: string) => { - console.log(`Deleting original file from storage bucket: '${path}'`); + logger.log(`Deleting original file from storage bucket: '${path}'`); }; diff --git a/storage-resize-images/package.json b/storage-resize-images/package.json index 6bb5cbee9..4738a1e94 100644 --- a/storage-resize-images/package.json +++ b/storage-resize-images/package.json @@ -13,7 +13,7 @@ }, "dependencies": { "firebase-admin": "^8.0.0", - "firebase-functions": "^3.3.0", + "firebase-functions": "^3.7.0", "mkdirp": "^1.0.4", "sharp": "0.23.4", "uuidv4": "^6.1.0" diff --git a/storage-resize-images/tsconfig.json b/storage-resize-images/tsconfig.json index bc30a346e..cdfe29ea4 100644 --- a/storage-resize-images/tsconfig.json +++ b/storage-resize-images/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../tsconfig.json", "compilerOptions": { - "outDir": "functions/lib" + "outDir": "lib", + "target": "es2018" }, "include": ["functions/src"] } From ed5cd4e08e2e09884442bab2db2fcb060da9c16f Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 24 Aug 2020 17:34:21 +0100 Subject: [PATCH 17/24] chore(rtdb-limit-child-nodes): migration to Node.js v10 (#434) --- rtdb-limit-child-nodes/PREINSTALL.md | 16 ++++++++-------- rtdb-limit-child-nodes/extension.yaml | 3 ++- rtdb-limit-child-nodes/functions/README.md | 15 ++++++++------- rtdb-limit-child-nodes/functions/lib/index.js | 17 ++++------------- rtdb-limit-child-nodes/functions/lib/logs.js | 17 +++++++++-------- rtdb-limit-child-nodes/functions/package.json | 2 +- rtdb-limit-child-nodes/functions/src/logs.ts | 17 +++++++++-------- rtdb-limit-child-nodes/functions/tsconfig.json | 3 ++- 8 files changed, 43 insertions(+), 47 deletions(-) diff --git a/rtdb-limit-child-nodes/PREINSTALL.md b/rtdb-limit-child-nodes/PREINSTALL.md index 13fb69077..40945edbe 100644 --- a/rtdb-limit-child-nodes/PREINSTALL.md +++ b/rtdb-limit-child-nodes/PREINSTALL.md @@ -6,11 +6,11 @@ If the number of nodes in your specified Realtime Database path exceeds the spec Before installing this extension, make sure that you've [set up a Realtime Database instance](https://firebase.google.com/docs/database) in your Firebase project. -#### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Firebase Realtime Database -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) +### Billing + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) + - Firebase Realtime Database diff --git a/rtdb-limit-child-nodes/extension.yaml b/rtdb-limit-child-nodes/extension.yaml index 89152a43d..902df29e7 100644 --- a/rtdb-limit-child-nodes/extension.yaml +++ b/rtdb-limit-child-nodes/extension.yaml @@ -37,7 +37,7 @@ contributors: email: oss@invertase.io url: https://github.com/invertase -billingRequired: false +billingRequired: true roles: - role: firebasedatabase.admin @@ -52,6 +52,7 @@ resources: then deletes the oldest nodes first, as needed to maintain the max count. properties: location: ${LOCATION} + runtime: nodejs10 eventTrigger: eventType: providers/google.firebase.database/eventTypes/ref.create resource: projects/_/instances/${DATABASE_INSTANCE}/refs/${NODE_PATH}/{messageid} diff --git a/rtdb-limit-child-nodes/functions/README.md b/rtdb-limit-child-nodes/functions/README.md index e5fc12fca..a90a9aae4 100644 --- a/rtdb-limit-child-nodes/functions/README.md +++ b/rtdb-limit-child-nodes/functions/README.md @@ -14,14 +14,15 @@ If the number of nodes in your specified Realtime Database path exceeds the spec Before installing this extension, make sure that you've [set up a Realtime Database instance](https://firebase.google.com/docs/database) in your Firebase project. -#### Billing +### Billing + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) + - Firebase Realtime Database -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Firebase Realtime Database -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) diff --git a/rtdb-limit-child-nodes/functions/lib/index.js b/rtdb-limit-child-nodes/functions/lib/index.js index 05590d816..b25ce12b9 100644 --- a/rtdb-limit-child-nodes/functions/lib/index.js +++ b/rtdb-limit-child-nodes/functions/lib/index.js @@ -14,26 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); exports.rtdblimit = void 0; const functions = require("firebase-functions"); const config_1 = require("./config"); const logs = require("./logs"); logs.init(); -exports.rtdblimit = functions.handler.database.ref.onCreate((snapshot) => __awaiter(void 0, void 0, void 0, function* () { +exports.rtdblimit = functions.handler.database.ref.onCreate(async (snapshot) => { logs.start(); try { const parentRef = snapshot.ref.parent; - const parentSnapshot = yield parentRef.once("value"); + const parentSnapshot = await parentRef.once("value"); logs.childCount(parentRef.path, parentSnapshot.numChildren()); if (parentSnapshot.numChildren() > config_1.default.maxCount) { let childCount = 0; @@ -44,7 +35,7 @@ exports.rtdblimit = functions.handler.database.ref.onCreate((snapshot) => __awai } }); logs.pathTruncating(parentRef.path, config_1.default.maxCount); - yield parentRef.update(updates); + await parentRef.update(updates); logs.pathTruncated(parentRef.path, config_1.default.maxCount); } else { @@ -55,4 +46,4 @@ exports.rtdblimit = functions.handler.database.ref.onCreate((snapshot) => __awai catch (err) { logs.error(err); } -})); +}); diff --git a/rtdb-limit-child-nodes/functions/lib/logs.js b/rtdb-limit-child-nodes/functions/lib/logs.js index 0247d8930..bfb2fe0f3 100644 --- a/rtdb-limit-child-nodes/functions/lib/logs.js +++ b/rtdb-limit-child-nodes/functions/lib/logs.js @@ -16,28 +16,29 @@ */ Object.defineProperty(exports, "__esModule", { value: true }); exports.start = exports.pathTruncating = exports.pathTruncated = exports.pathSkipped = exports.init = exports.error = exports.complete = exports.childCount = void 0; +const firebase_functions_1 = require("firebase-functions"); const config_1 = require("./config"); exports.childCount = (path, childCount) => { - console.log(`Node: '${path}' has: ${childCount} children`); + firebase_functions_1.logger.log(`Node: '${path}' has: ${childCount} children`); }; exports.complete = () => { - console.log("Completed execution of extension"); + firebase_functions_1.logger.log("Completed execution of extension"); }; exports.error = (err) => { - console.error("Error when truncating the database node", err); + firebase_functions_1.logger.error("Error when truncating the database node", err); }; exports.init = () => { - console.log("Initializing extension with configuration", config_1.default); + firebase_functions_1.logger.log("Initializing extension with configuration", config_1.default); }; exports.pathSkipped = (path) => { - console.log(`Path: '${path}' does not need to be truncated`); + firebase_functions_1.logger.log(`Path: '${path}' does not need to be truncated`); }; exports.pathTruncated = (path, count) => { - console.log(`Truncated path: '${path}' to ${count} items`); + firebase_functions_1.logger.log(`Truncated path: '${path}' to ${count} items`); }; exports.pathTruncating = (path, count) => { - console.log(`Truncating path: '${path}' to ${count} items`); + firebase_functions_1.logger.log(`Truncating path: '${path}' to ${count} items`); }; exports.start = () => { - console.log("Started execution of extension with configuration", config_1.default); + firebase_functions_1.logger.log("Started execution of extension with configuration", config_1.default); }; diff --git a/rtdb-limit-child-nodes/functions/package.json b/rtdb-limit-child-nodes/functions/package.json index f78e2efb0..cff87a6c5 100644 --- a/rtdb-limit-child-nodes/functions/package.json +++ b/rtdb-limit-child-nodes/functions/package.json @@ -12,7 +12,7 @@ "license": "Apache-2.0", "dependencies": { "firebase-admin": "^8.0.0", - "firebase-functions": "^3.3.0" + "firebase-functions": "^3.7.0" }, "devDependencies": { "rimraf": "^2.6.3", diff --git a/rtdb-limit-child-nodes/functions/src/logs.ts b/rtdb-limit-child-nodes/functions/src/logs.ts index ec1566fbd..12ec4ae5d 100644 --- a/rtdb-limit-child-nodes/functions/src/logs.ts +++ b/rtdb-limit-child-nodes/functions/src/logs.ts @@ -14,36 +14,37 @@ * limitations under the License. */ +import { logger } from "firebase-functions"; import config from "./config"; export const childCount = (path: string, childCount: number) => { - console.log(`Node: '${path}' has: ${childCount} children`); + logger.log(`Node: '${path}' has: ${childCount} children`); }; export const complete = () => { - console.log("Completed execution of extension"); + logger.log("Completed execution of extension"); }; export const error = (err: Error) => { - console.error("Error when truncating the database node", err); + logger.error("Error when truncating the database node", err); }; export const init = () => { - console.log("Initializing extension with configuration", config); + logger.log("Initializing extension with configuration", config); }; export const pathSkipped = (path: string) => { - console.log(`Path: '${path}' does not need to be truncated`); + logger.log(`Path: '${path}' does not need to be truncated`); }; export const pathTruncated = (path: string, count: number) => { - console.log(`Truncated path: '${path}' to ${count} items`); + logger.log(`Truncated path: '${path}' to ${count} items`); }; export const pathTruncating = (path: string, count: number) => { - console.log(`Truncating path: '${path}' to ${count} items`); + logger.log(`Truncating path: '${path}' to ${count} items`); }; export const start = () => { - console.log("Started execution of extension with configuration", config); + logger.log("Started execution of extension with configuration", config); }; diff --git a/rtdb-limit-child-nodes/functions/tsconfig.json b/rtdb-limit-child-nodes/functions/tsconfig.json index 1b551f4e9..9f7e58b1e 100644 --- a/rtdb-limit-child-nodes/functions/tsconfig.json +++ b/rtdb-limit-child-nodes/functions/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "outDir": "lib" + "outDir": "lib", + "target": "es2018" }, "include": ["src"] } From d0fe2a25b47f84386bd80dbf40b54555c454f1f6 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 24 Aug 2020 17:34:35 +0100 Subject: [PATCH 18/24] chore(delete-user-data): migration to Node.js v10 (#431) --- delete-user-data/PREINSTALL.md | 18 +++--- delete-user-data/README.md | 24 ++++---- delete-user-data/extension.yaml | 4 +- delete-user-data/functions/README.md | 71 ++++++++++++++++++++++++ delete-user-data/functions/lib/index.js | 53 ++++++++---------- delete-user-data/functions/lib/logs.js | 45 +++++++-------- delete-user-data/functions/package.json | 2 +- delete-user-data/functions/src/logs.ts | 45 +++++++-------- delete-user-data/functions/tsconfig.json | 3 +- 9 files changed, 165 insertions(+), 100 deletions(-) create mode 100644 delete-user-data/functions/README.md diff --git a/delete-user-data/PREINSTALL.md b/delete-user-data/PREINSTALL.md index b7bea54d2..b0047eb7f 100644 --- a/delete-user-data/PREINSTALL.md +++ b/delete-user-data/PREINSTALL.md @@ -13,12 +13,12 @@ Depending on where you'd like to delete user data from, make sure that you've se Also, make sure that you've set up [Firebase Authentication](https://firebase.google.com/docs/auth) to manage your users. #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Firestore -- Firebase Realtime Database -- Cloud Storage -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Firestore + - Firebase Realtime Database + - Cloud Storage + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/delete-user-data/README.md b/delete-user-data/README.md index feba5aec6..90e803f60 100644 --- a/delete-user-data/README.md +++ b/delete-user-data/README.md @@ -19,22 +19,22 @@ Depending on where you'd like to delete user data from, make sure that you've se Also, make sure that you've set up [Firebase Authentication](https://firebase.google.com/docs/auth) to manage your users. #### Billing - -This extension uses other Firebase or Google Cloud Platform services which may have associated charges: - -- Cloud Firestore -- Firebase Realtime Database -- Cloud Storage -- Cloud Functions - -When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Firestore + - Firebase Realtime Database + - Cloud Storage + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) **Configuration Parameters:** -* Deployment location: Where should the extension be deployed? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). +* Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database or Storage bucket. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). * Cloud Firestore paths: Which paths in your Cloud Firestore instance contain user data? Leave empty if you don't use Cloud Firestore. Enter the full paths, separated by commas. You can represent the User ID of the deleted user with `{UID}`. @@ -47,8 +47,8 @@ Enter the full paths, separated by commas. You can represent the User ID of the For example: `users/{UID},admins/{UID}`. * Cloud Storage paths: Where in Google Cloud Storage do you store user data? Leave empty if you don't use Cloud Storage. -Enter the full paths, separated by commas. You can represent the User ID of the deleted user with `{UID}`. You can use `{DEFAULT}` to represent your default bucket. -For example, if you are using your default bucket, and the bucket has files with the naming scheme `{UID}-pic.png`, then you can enter `{DEFAULT}/{UID}-pic.png`. If you also have files in another bucket called `my-awesome-app-logs`, and that bucket has files with the naming scheme `{UID}-logs.txt`, then you can enter `{DEFAULT}/{UID}-pic.png,my-awesome-app-logs/{UID}-logs.txt`. +Enter the full paths to files or directories in your Storage buckets, separated by commas. Use `{UID}` to represent the User ID of the deleted user, and use `{DEFAULT}` to represent your default Storage bucket. +Here's a series of examples. To delete all the files in your default bucket with the file naming scheme `{UID}-pic.png`, enter `{DEFAULT}/{UID}-pic.png`. To also delete all the files in another bucket called my-app-logs with the file naming scheme `{UID}-logs.txt`, enter `{DEFAULT}/{UID}-pic.png,my-app-logs/{UID}-logs.txt`. To *also* delete a User ID-labeled directory and all its files (like `media/{UID}`), enter `{DEFAULT}/{UID}-pic.png,my-app-logs/{UID}-logs.txt,{DEFAULT}/media/{UID}`. diff --git a/delete-user-data/extension.yaml b/delete-user-data/extension.yaml index ae0eff466..cdd93ba83 100644 --- a/delete-user-data/extension.yaml +++ b/delete-user-data/extension.yaml @@ -40,7 +40,7 @@ contributors: email: oss@invertase.io url: https://github.com/invertase -billingRequired: false +billingRequired: true roles: - role: datastore.owner @@ -60,10 +60,10 @@ resources: Cloud Storage. properties: location: ${LOCATION} + runtime: nodejs10 eventTrigger: eventType: providers/firebase.auth/eventTypes/user.delete resource: projects/${PROJECT_ID} - runtime: nodejs10 params: - param: LOCATION diff --git a/delete-user-data/functions/README.md b/delete-user-data/functions/README.md new file mode 100644 index 000000000..90e803f60 --- /dev/null +++ b/delete-user-data/functions/README.md @@ -0,0 +1,71 @@ +# Delete User Data + +**Description**: Deletes data keyed on a userId from Cloud Firestore, Realtime Database, and/or Cloud Storage when a user deletes their account. + + + +**Details**: Use this extension to automatically delete a user's data if the user is deleted from your authenticated users. + +You can configure this extension to delete user data from any or all of the following: Cloud Firestore, Realtime Database, or Cloud Storage. Each trigger of the extension to delete data is keyed to the user's UserId. + +**Note:** To use this extension, you need to manage your users with Firebase Authentication. + +This extension is useful in respecting user privacy and fulfilling compliance requirements. However, using this extension does not guarantee compliance with government and industry regulations. + +#### Additional setup + +Depending on where you'd like to delete user data from, make sure that you've set up [Cloud Firestore](https://firebase.google.com/docs/firestore), [Realtime Database](https://firebase.google.com/docs/database), or [Cloud Storage](https://firebase.google.com/docs/storage) in your Firebase project before installing this extension. + +Also, make sure that you've set up [Firebase Authentication](https://firebase.google.com/docs/auth) to manage your users. + +#### Billing + +To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) + +- You will be charged around $0.01/month for each instance of this extension you install. +- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: + - Cloud Firestore + - Firebase Realtime Database + - Cloud Storage + - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) + + + + +**Configuration Parameters:** + +* Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database or Storage bucket. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). + +* Cloud Firestore paths: Which paths in your Cloud Firestore instance contain user data? Leave empty if you don't use Cloud Firestore. +Enter the full paths, separated by commas. You can represent the User ID of the deleted user with `{UID}`. +For example, if you have the collections `users` and `admins`, and each collection has documents with User ID as document IDs, then you can enter `users/{UID},admins/{UID}`. + +* Cloud Firestore delete mode: (Only applicable if you use the `Cloud Firestore paths` parameter.) How do you want to delete Cloud Firestore documents? To also delete documents in subcollections, set this parameter to `recursive`. + +* Realtime Database paths: Which paths in your Realtime Database instance contain user data? Leave empty if you don't use Realtime Database. +Enter the full paths, separated by commas. You can represent the User ID of the deleted user with `{UID}`. +For example: `users/{UID},admins/{UID}`. + +* Cloud Storage paths: Where in Google Cloud Storage do you store user data? Leave empty if you don't use Cloud Storage. +Enter the full paths to files or directories in your Storage buckets, separated by commas. Use `{UID}` to represent the User ID of the deleted user, and use `{DEFAULT}` to represent your default Storage bucket. +Here's a series of examples. To delete all the files in your default bucket with the file naming scheme `{UID}-pic.png`, enter `{DEFAULT}/{UID}-pic.png`. To also delete all the files in another bucket called my-app-logs with the file naming scheme `{UID}-logs.txt`, enter `{DEFAULT}/{UID}-pic.png,my-app-logs/{UID}-logs.txt`. To *also* delete a User ID-labeled directory and all its files (like `media/{UID}`), enter `{DEFAULT}/{UID}-pic.png,my-app-logs/{UID}-logs.txt,{DEFAULT}/media/{UID}`. + + + +**Cloud Functions:** + +* **clearData:** Listens for user accounts to be deleted from your project's authenticated users, then removes any associated user data (based on Firebase Authentication's User ID) from Realtime Database, Cloud Firestore, and/or Cloud Storage. + + + +**Access Required**: + + + +This extension will operate with the following project IAM roles: + +* datastore.owner (Reason: Allows the extension to delete (user) data from Cloud Firestore.) + +* firebasedatabase.admin (Reason: Allows the extension to delete (user) data from Realtime Database.) + +* storage.admin (Reason: Allows the extension to delete (user) data from Cloud Storage.) diff --git a/delete-user-data/functions/lib/index.js b/delete-user-data/functions/lib/index.js index f8ed5ba8c..fccf027d1 100644 --- a/delete-user-data/functions/lib/index.js +++ b/delete-user-data/functions/lib/index.js @@ -14,15 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; Object.defineProperty(exports, "__esModule", { value: true }); exports.clearData = void 0; const admin = require("firebase-admin"); @@ -38,7 +29,7 @@ logs.init(); * Storage, and Firestore. It waits for all deletions to complete, and then * returns a success message. */ -exports.clearData = functions.auth.user().onDelete((user) => __awaiter(void 0, void 0, void 0, function* () { +exports.clearData = functions.auth.user().onDelete(async (user) => { logs.start(); const { firestorePaths, rtdbPaths, storagePaths } = config_1.default; const { uid } = user; @@ -61,16 +52,16 @@ exports.clearData = functions.auth.user().onDelete((user) => __awaiter(void 0, v else { logs.storageNotConfigured(); } - yield Promise.all(promises); + await Promise.all(promises); logs.complete(uid); -})); -const clearDatabaseData = (databasePaths, uid) => __awaiter(void 0, void 0, void 0, function* () { +}); +const clearDatabaseData = async (databasePaths, uid) => { logs.rtdbDeleting(); const paths = extractUserPaths(databasePaths, uid); - const promises = paths.map((path) => __awaiter(void 0, void 0, void 0, function* () { + const promises = paths.map(async (path) => { try { logs.rtdbPathDeleting(path); - yield admin + await admin .database() .ref(path) .remove(); @@ -79,14 +70,14 @@ const clearDatabaseData = (databasePaths, uid) => __awaiter(void 0, void 0, void catch (err) { logs.rtdbPathError(path, err); } - })); - yield Promise.all(promises); + }); + await Promise.all(promises); logs.rtdbDeleted(); -}); -const clearStorageData = (storagePaths, uid) => __awaiter(void 0, void 0, void 0, function* () { +}; +const clearStorageData = async (storagePaths, uid) => { logs.storageDeleting(); const paths = extractUserPaths(storagePaths, uid); - const promises = paths.map((path) => __awaiter(void 0, void 0, void 0, function* () { + const promises = paths.map(async (path) => { const parts = path.split("/"); const bucketName = parts[0]; const bucket = bucketName === "{DEFAULT}" @@ -95,7 +86,7 @@ const clearStorageData = (storagePaths, uid) => __awaiter(void 0, void 0, void 0 const prefix = parts.slice(1).join("/"); try { logs.storagePathDeleting(prefix); - yield bucket.deleteFiles({ + await bucket.deleteFiles({ prefix, }); logs.storagePathDeleted(prefix); @@ -108,21 +99,21 @@ const clearStorageData = (storagePaths, uid) => __awaiter(void 0, void 0, void 0 logs.storagePathError(prefix, err); } } - })); - yield Promise.all(promises); + }); + await Promise.all(promises); logs.storageDeleted(); -}); -const clearFirestoreData = (firestorePaths, uid) => __awaiter(void 0, void 0, void 0, function* () { +}; +const clearFirestoreData = async (firestorePaths, uid) => { logs.firestoreDeleting(); const paths = extractUserPaths(firestorePaths, uid); - const promises = paths.map((path) => __awaiter(void 0, void 0, void 0, function* () { + const promises = paths.map(async (path) => { try { const isRecursive = config_1.default.firestoreDeleteMode === "recursive"; if (!isRecursive) { const firestore = admin.firestore(); logs.firestorePathDeleting(path, false); // Wrapping in transaction to allow for automatic retries (#48) - yield firestore.runTransaction((transaction) => { + await firestore.runTransaction((transaction) => { transaction.delete(firestore.doc(path)); return Promise.resolve(); }); @@ -130,7 +121,7 @@ const clearFirestoreData = (firestorePaths, uid) => __awaiter(void 0, void 0, vo } else { logs.firestorePathDeleting(path, true); - yield firebase_tools.firestore.delete(path, { + await firebase_tools.firestore.delete(path, { project: process.env.PROJECT_ID, recursive: true, yes: true, @@ -141,10 +132,10 @@ const clearFirestoreData = (firestorePaths, uid) => __awaiter(void 0, void 0, vo catch (err) { logs.firestorePathError(path, err); } - })); - yield Promise.all(promises); + }); + await Promise.all(promises); logs.firestoreDeleted(); -}); +}; const extractUserPaths = (paths, uid) => { return paths.split(",").map((path) => replaceUID(path, uid)); }; diff --git a/delete-user-data/functions/lib/logs.js b/delete-user-data/functions/lib/logs.js index 2329883cb..c70578b0a 100644 --- a/delete-user-data/functions/lib/logs.js +++ b/delete-user-data/functions/lib/logs.js @@ -16,70 +16,71 @@ */ Object.defineProperty(exports, "__esModule", { value: true }); exports.storagePathError = exports.storagePath404 = exports.storagePathDeleting = exports.storagePathDeleted = exports.storageNotConfigured = exports.storageDeleting = exports.storageDeleted = exports.start = exports.rtdbPathError = exports.rtdbPathDeleting = exports.rtdbNotConfigured = exports.rtdbPathDeleted = exports.rtdbDeleting = exports.rtdbDeleted = exports.init = exports.firestorePathError = exports.firestorePathDeleting = exports.firestorePathDeleted = exports.firestoreNotConfigured = exports.firestoreDeleting = exports.firestoreDeleted = exports.complete = void 0; +const firebase_functions_1 = require("firebase-functions"); const config_1 = require("./config"); exports.complete = (uid) => { - console.log(`Successfully removed data for user: ${uid}`); + firebase_functions_1.logger.log(`Successfully removed data for user: ${uid}`); }; exports.firestoreDeleted = () => { - console.log("Finished deleting user data from Cloud Firestore"); + firebase_functions_1.logger.log("Finished deleting user data from Cloud Firestore"); }; exports.firestoreDeleting = () => { - console.log("Deleting user data from Cloud Firestore"); + firebase_functions_1.logger.log("Deleting user data from Cloud Firestore"); }; exports.firestoreNotConfigured = () => { - console.log("Cloud Firestore paths are not configured, skipping"); + firebase_functions_1.logger.log("Cloud Firestore paths are not configured, skipping"); }; exports.firestorePathDeleted = (path, recursive) => { - console.log(`Deleted: '${path}' from Cloud Firestore ${recursive ? "with recursive delete" : ""}`); + firebase_functions_1.logger.log(`Deleted: '${path}' from Cloud Firestore ${recursive ? "with recursive delete" : ""}`); }; exports.firestorePathDeleting = (path, recursive) => { - console.log(`Deleting: '${path}' from Cloud Firestore ${recursive ? "with recursive delete" : ""}`); + firebase_functions_1.logger.log(`Deleting: '${path}' from Cloud Firestore ${recursive ? "with recursive delete" : ""}`); }; exports.firestorePathError = (path, err) => { - console.error(`Error when deleting: '${path}' from Cloud Firestore`, err); + firebase_functions_1.logger.error(`Error when deleting: '${path}' from Cloud Firestore`, err); }; exports.init = () => { - console.log("Initializing extension with configuration", config_1.default); + firebase_functions_1.logger.log("Initializing extension with configuration", config_1.default); }; exports.rtdbDeleted = () => { - console.log("Finished deleting user data from the Realtime Database"); + firebase_functions_1.logger.log("Finished deleting user data from the Realtime Database"); }; exports.rtdbDeleting = () => { - console.log("Deleting user data from the Realtime Database"); + firebase_functions_1.logger.log("Deleting user data from the Realtime Database"); }; exports.rtdbPathDeleted = (path) => { - console.log(`Deleted: '${path}' from the Realtime Database`); + firebase_functions_1.logger.log(`Deleted: '${path}' from the Realtime Database`); }; exports.rtdbNotConfigured = () => { - console.log("Realtime Database paths are not configured, skipping"); + firebase_functions_1.logger.log("Realtime Database paths are not configured, skipping"); }; exports.rtdbPathDeleting = (path) => { - console.log(`Deleting: '${path}' from the Realtime Database`); + firebase_functions_1.logger.log(`Deleting: '${path}' from the Realtime Database`); }; exports.rtdbPathError = (path, err) => { - console.error(`Error when deleting: '${path}' from the Realtime Database`, err); + firebase_functions_1.logger.error(`Error when deleting: '${path}' from the Realtime Database`, err); }; exports.start = () => { - console.log("Started extension execution with configuration", config_1.default); + firebase_functions_1.logger.log("Started extension execution with configuration", config_1.default); }; exports.storageDeleted = () => { - console.log("Finished deleting user data from Cloud Storage"); + firebase_functions_1.logger.log("Finished deleting user data from Cloud Storage"); }; exports.storageDeleting = () => { - console.log("Deleting user data from Cloud Storage"); + firebase_functions_1.logger.log("Deleting user data from Cloud Storage"); }; exports.storageNotConfigured = () => { - console.log("Cloud Storage paths are not configured, skipping"); + firebase_functions_1.logger.log("Cloud Storage paths are not configured, skipping"); }; exports.storagePathDeleted = (path) => { - console.log(`Deleted: '${path}' from Cloud Storage`); + firebase_functions_1.logger.log(`Deleted: '${path}' from Cloud Storage`); }; exports.storagePathDeleting = (path) => { - console.log(`Deleting: '${path}' from Cloud Storage`); + firebase_functions_1.logger.log(`Deleting: '${path}' from Cloud Storage`); }; exports.storagePath404 = (path) => { - console.log(`File: '${path}' does not exist in Cloud Storage, skipping`); + firebase_functions_1.logger.log(`File: '${path}' does not exist in Cloud Storage, skipping`); }; exports.storagePathError = (path, err) => { - console.error(`Error deleting: '${path}' from Cloud Storage`, err); + firebase_functions_1.logger.error(`Error deleting: '${path}' from Cloud Storage`, err); }; diff --git a/delete-user-data/functions/package.json b/delete-user-data/functions/package.json index b0b55f371..33eafd9cc 100644 --- a/delete-user-data/functions/package.json +++ b/delete-user-data/functions/package.json @@ -13,7 +13,7 @@ "license": "Apache-2.0", "dependencies": { "firebase-admin": "^8.0.0", - "firebase-functions": "^3.3.0", + "firebase-functions": "^3.7.0", "firebase-tools": "^7.14.0" }, "devDependencies": { diff --git a/delete-user-data/functions/src/logs.ts b/delete-user-data/functions/src/logs.ts index 6de9a13c9..93bf16f57 100644 --- a/delete-user-data/functions/src/logs.ts +++ b/delete-user-data/functions/src/logs.ts @@ -14,26 +14,27 @@ * limitations under the License. */ +import { logger } from "firebase-functions"; import config from "./config"; export const complete = (uid: string) => { - console.log(`Successfully removed data for user: ${uid}`); + logger.log(`Successfully removed data for user: ${uid}`); }; export const firestoreDeleted = () => { - console.log("Finished deleting user data from Cloud Firestore"); + logger.log("Finished deleting user data from Cloud Firestore"); }; export const firestoreDeleting = () => { - console.log("Deleting user data from Cloud Firestore"); + logger.log("Deleting user data from Cloud Firestore"); }; export const firestoreNotConfigured = () => { - console.log("Cloud Firestore paths are not configured, skipping"); + logger.log("Cloud Firestore paths are not configured, skipping"); }; export const firestorePathDeleted = (path: string, recursive: boolean) => { - console.log( + logger.log( `Deleted: '${path}' from Cloud Firestore ${ recursive ? "with recursive delete" : "" }` @@ -41,7 +42,7 @@ export const firestorePathDeleted = (path: string, recursive: boolean) => { }; export const firestorePathDeleting = (path: string, recursive: boolean) => { - console.log( + logger.log( `Deleting: '${path}' from Cloud Firestore ${ recursive ? "with recursive delete" : "" }` @@ -49,68 +50,68 @@ export const firestorePathDeleting = (path: string, recursive: boolean) => { }; export const firestorePathError = (path: string, err: Error) => { - console.error(`Error when deleting: '${path}' from Cloud Firestore`, err); + logger.error(`Error when deleting: '${path}' from Cloud Firestore`, err); }; export const init = () => { - console.log("Initializing extension with configuration", config); + logger.log("Initializing extension with configuration", config); }; export const rtdbDeleted = () => { - console.log("Finished deleting user data from the Realtime Database"); + logger.log("Finished deleting user data from the Realtime Database"); }; export const rtdbDeleting = () => { - console.log("Deleting user data from the Realtime Database"); + logger.log("Deleting user data from the Realtime Database"); }; export const rtdbPathDeleted = (path: string) => { - console.log(`Deleted: '${path}' from the Realtime Database`); + logger.log(`Deleted: '${path}' from the Realtime Database`); }; export const rtdbNotConfigured = () => { - console.log("Realtime Database paths are not configured, skipping"); + logger.log("Realtime Database paths are not configured, skipping"); }; export const rtdbPathDeleting = (path: string) => { - console.log(`Deleting: '${path}' from the Realtime Database`); + logger.log(`Deleting: '${path}' from the Realtime Database`); }; export const rtdbPathError = (path: string, err: Error) => { - console.error( + logger.error( `Error when deleting: '${path}' from the Realtime Database`, err ); }; export const start = () => { - console.log("Started extension execution with configuration", config); + logger.log("Started extension execution with configuration", config); }; export const storageDeleted = () => { - console.log("Finished deleting user data from Cloud Storage"); + logger.log("Finished deleting user data from Cloud Storage"); }; export const storageDeleting = () => { - console.log("Deleting user data from Cloud Storage"); + logger.log("Deleting user data from Cloud Storage"); }; export const storageNotConfigured = () => { - console.log("Cloud Storage paths are not configured, skipping"); + logger.log("Cloud Storage paths are not configured, skipping"); }; export const storagePathDeleted = (path: string) => { - console.log(`Deleted: '${path}' from Cloud Storage`); + logger.log(`Deleted: '${path}' from Cloud Storage`); }; export const storagePathDeleting = (path: string) => { - console.log(`Deleting: '${path}' from Cloud Storage`); + logger.log(`Deleting: '${path}' from Cloud Storage`); }; export const storagePath404 = (path: string) => { - console.log(`File: '${path}' does not exist in Cloud Storage, skipping`); + logger.log(`File: '${path}' does not exist in Cloud Storage, skipping`); }; export const storagePathError = (path: string, err: Error) => { - console.error(`Error deleting: '${path}' from Cloud Storage`, err); + logger.error(`Error deleting: '${path}' from Cloud Storage`, err); }; diff --git a/delete-user-data/functions/tsconfig.json b/delete-user-data/functions/tsconfig.json index 1b551f4e9..9f7e58b1e 100644 --- a/delete-user-data/functions/tsconfig.json +++ b/delete-user-data/functions/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "outDir": "lib" + "outDir": "lib", + "target": "es2018" }, "include": ["src"] } From a5100ee6cc9236494a16fc96935af3a23827108b Mon Sep 17 00:00:00 2001 From: Jeff Date: Mon, 24 Aug 2020 11:59:51 -0700 Subject: [PATCH 19/24] update with latest recommended wording (#438) --- auth-mailchimp-sync/PREINSTALL.md | 2 +- auth-mailchimp-sync/functions/README.md | 2 +- delete-user-data/PREINSTALL.md | 2 +- delete-user-data/README.md | 2 +- delete-user-data/functions/README.md | 2 +- firestore-bigquery-export/PREINSTALL.md | 2 +- firestore-bigquery-export/README.md | 2 +- firestore-counter/PREINSTALL.md | 2 +- firestore-counter/functions/README.md | 2 +- firestore-send-email/PREINSTALL.md | 2 +- firestore-send-email/functions/README.md | 2 +- firestore-shorten-urls-bitly/PREINSTALL.md | 2 +- firestore-shorten-urls-bitly/functions/README.md | 2 +- firestore-translate-text/PREINSTALL.md | 2 +- firestore-translate-text/README.md | 2 +- rtdb-limit-child-nodes/PREINSTALL.md | 2 +- rtdb-limit-child-nodes/functions/README.md | 2 +- storage-resize-images/PREINSTALL.md | 2 +- storage-resize-images/README.md | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/auth-mailchimp-sync/PREINSTALL.md b/auth-mailchimp-sync/PREINSTALL.md index b94150e27..a14b01af0 100644 --- a/auth-mailchimp-sync/PREINSTALL.md +++ b/auth-mailchimp-sync/PREINSTALL.md @@ -16,7 +16,7 @@ You must also have a Mailchimp account before installing this extension. To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Functions (Node.js 10+ runtime. See [FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/auth-mailchimp-sync/functions/README.md b/auth-mailchimp-sync/functions/README.md index a7862052c..fa4d44e74 100644 --- a/auth-mailchimp-sync/functions/README.md +++ b/auth-mailchimp-sync/functions/README.md @@ -24,7 +24,7 @@ You must also have a Mailchimp account before installing this extension. To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Functions (Node.js 10+ runtime. See [FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/delete-user-data/PREINSTALL.md b/delete-user-data/PREINSTALL.md index b0047eb7f..374c29e0c 100644 --- a/delete-user-data/PREINSTALL.md +++ b/delete-user-data/PREINSTALL.md @@ -16,7 +16,7 @@ Also, make sure that you've set up [Firebase Authentication](https://firebase.go To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Firestore - Firebase Realtime Database diff --git a/delete-user-data/README.md b/delete-user-data/README.md index 90e803f60..a5bdcd5fc 100644 --- a/delete-user-data/README.md +++ b/delete-user-data/README.md @@ -22,7 +22,7 @@ Also, make sure that you've set up [Firebase Authentication](https://firebase.go To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Firestore - Firebase Realtime Database diff --git a/delete-user-data/functions/README.md b/delete-user-data/functions/README.md index 90e803f60..a5bdcd5fc 100644 --- a/delete-user-data/functions/README.md +++ b/delete-user-data/functions/README.md @@ -22,7 +22,7 @@ Also, make sure that you've set up [Firebase Authentication](https://firebase.go To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Firestore - Firebase Realtime Database diff --git a/firestore-bigquery-export/PREINSTALL.md b/firestore-bigquery-export/PREINSTALL.md index 23fe30dc9..6985437f1 100644 --- a/firestore-bigquery-export/PREINSTALL.md +++ b/firestore-bigquery-export/PREINSTALL.md @@ -30,7 +30,7 @@ After your data is in BigQuery, you can run the [schema-views script](https://gi To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - BigQuery (this extension writes to BigQuery with [streaming inserts](https://cloud.google.com/bigquery/pricing#streaming_pricing)) - Cloud Firestore diff --git a/firestore-bigquery-export/README.md b/firestore-bigquery-export/README.md index 82baf34af..241bb92d6 100644 --- a/firestore-bigquery-export/README.md +++ b/firestore-bigquery-export/README.md @@ -38,7 +38,7 @@ After your data is in BigQuery, you can run the [schema-views script](https://gi To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - BigQuery (this extension writes to BigQuery with [streaming inserts](https://cloud.google.com/bigquery/pricing#streaming_pricing)) - Cloud Firestore diff --git a/firestore-counter/PREINSTALL.md b/firestore-counter/PREINSTALL.md index 5b6f17d0a..f003e1c08 100644 --- a/firestore-counter/PREINSTALL.md +++ b/firestore-counter/PREINSTALL.md @@ -28,7 +28,7 @@ Detailed information for these post-installation tasks are provided after you in To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Firestore - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) \ No newline at end of file diff --git a/firestore-counter/functions/README.md b/firestore-counter/functions/README.md index 3b2f98ff0..ab15ff43a 100644 --- a/firestore-counter/functions/README.md +++ b/firestore-counter/functions/README.md @@ -36,7 +36,7 @@ Detailed information for these post-installation tasks are provided after you in To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Firestore - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/firestore-send-email/PREINSTALL.md b/firestore-send-email/PREINSTALL.md index 2d88fab17..cbe9ff7aa 100644 --- a/firestore-send-email/PREINSTALL.md +++ b/firestore-send-email/PREINSTALL.md @@ -25,7 +25,7 @@ Before installing this extension, make sure that you've [set up a Cloud Firestor #### Billing To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Firestore - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/firestore-send-email/functions/README.md b/firestore-send-email/functions/README.md index 856260d70..af74d4daa 100644 --- a/firestore-send-email/functions/README.md +++ b/firestore-send-email/functions/README.md @@ -33,7 +33,7 @@ Before installing this extension, make sure that you've [set up a Cloud Firestor #### Billing To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Firestore - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/firestore-shorten-urls-bitly/PREINSTALL.md b/firestore-shorten-urls-bitly/PREINSTALL.md index 1e4d7a206..a0f24aa8c 100644 --- a/firestore-shorten-urls-bitly/PREINSTALL.md +++ b/firestore-shorten-urls-bitly/PREINSTALL.md @@ -19,7 +19,7 @@ You must also have a Bitly account and access token before installing this exten To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Firestore - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/firestore-shorten-urls-bitly/functions/README.md b/firestore-shorten-urls-bitly/functions/README.md index a400eb4b7..e160b9bd8 100644 --- a/firestore-shorten-urls-bitly/functions/README.md +++ b/firestore-shorten-urls-bitly/functions/README.md @@ -27,7 +27,7 @@ You must also have a Bitly account and access token before installing this exten To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Firestore - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/firestore-translate-text/PREINSTALL.md b/firestore-translate-text/PREINSTALL.md index 0b127b240..12d4c3f1b 100644 --- a/firestore-translate-text/PREINSTALL.md +++ b/firestore-translate-text/PREINSTALL.md @@ -16,7 +16,7 @@ Before installing this extension, make sure that you've [set up a Cloud Firestor #### Billing To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Translation API - Cloud Firestore diff --git a/firestore-translate-text/README.md b/firestore-translate-text/README.md index 4ef1ae19b..0f87915f0 100644 --- a/firestore-translate-text/README.md +++ b/firestore-translate-text/README.md @@ -24,7 +24,7 @@ Before installing this extension, make sure that you've [set up a Cloud Firestor #### Billing To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Translation API - Cloud Firestore diff --git a/rtdb-limit-child-nodes/PREINSTALL.md b/rtdb-limit-child-nodes/PREINSTALL.md index 40945edbe..1666c6075 100644 --- a/rtdb-limit-child-nodes/PREINSTALL.md +++ b/rtdb-limit-child-nodes/PREINSTALL.md @@ -10,7 +10,7 @@ Before installing this extension, make sure that you've [set up a Realtime Datab To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) - Firebase Realtime Database diff --git a/rtdb-limit-child-nodes/functions/README.md b/rtdb-limit-child-nodes/functions/README.md index a90a9aae4..d8d995de9 100644 --- a/rtdb-limit-child-nodes/functions/README.md +++ b/rtdb-limit-child-nodes/functions/README.md @@ -18,7 +18,7 @@ Before installing this extension, make sure that you've [set up a Realtime Datab To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) - Firebase Realtime Database diff --git a/storage-resize-images/PREINSTALL.md b/storage-resize-images/PREINSTALL.md index fe37ed41e..62244e8d3 100644 --- a/storage-resize-images/PREINSTALL.md +++ b/storage-resize-images/PREINSTALL.md @@ -24,7 +24,7 @@ Before installing this extension, make sure that you've [set up a Cloud Storage To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Storage - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) diff --git a/storage-resize-images/README.md b/storage-resize-images/README.md index e5af29756..445e4417b 100644 --- a/storage-resize-images/README.md +++ b/storage-resize-images/README.md @@ -32,7 +32,7 @@ Before installing this extension, make sure that you've [set up a Cloud Storage To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) -- You will be charged around $0.01/month for each instance of this extension you install. +- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). - This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - Cloud Storage - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) From d495e0570d87a8a57d0615b4171437d4c4e65e0d Mon Sep 17 00:00:00 2001 From: Lauren Long Date: Mon, 24 Aug 2020 13:55:03 -0700 Subject: [PATCH 20/24] chore(*) Regenerate JS files --- auth-mailchimp-sync/functions/lib/logs.js | 26 +-- .../functions/lib/logs.js | 1 - .../functions/lib/util.js | 1 - .../functions/lib/index.js | 1 + .../functions/lib/logs/messages.js | 1 + .../functions/lib/validators.js | 1 + storage-resize-images/functions/lib/config.js | 24 -- storage-resize-images/functions/lib/index.js | 211 ------------------ storage-resize-images/functions/lib/logs.js | 92 -------- storage-resize-images/functions/lib/util.js | 7 - 10 files changed, 16 insertions(+), 349 deletions(-) delete mode 100644 storage-resize-images/functions/lib/config.js delete mode 100644 storage-resize-images/functions/lib/index.js delete mode 100644 storage-resize-images/functions/lib/logs.js delete mode 100644 storage-resize-images/functions/lib/util.js diff --git a/auth-mailchimp-sync/functions/lib/logs.js b/auth-mailchimp-sync/functions/lib/logs.js index 455df1c77..beeb0bbf8 100644 --- a/auth-mailchimp-sync/functions/lib/logs.js +++ b/auth-mailchimp-sync/functions/lib/logs.js @@ -16,45 +16,45 @@ */ Object.defineProperty(exports, "__esModule", { value: true }); exports.userRemoving = exports.userRemoved = exports.userNoEmail = exports.userAdding = exports.userAdded = exports.start = exports.mailchimpNotInitialized = exports.initError = exports.init = exports.errorRemoveUser = exports.errorAddUser = exports.complete = exports.obfuscatedConfig = void 0; -const { logger } = require("firebase-functions"); +const firebase_functions_1 = require("firebase-functions"); const config_1 = require("./config"); exports.obfuscatedConfig = { ...config_1.default, mailchimpApiKey: "", }; exports.complete = () => { - logger.log("Completed execution of extension"); + firebase_functions_1.logger.log("Completed execution of extension"); }; exports.errorAddUser = (err) => { - logger.error("Error when adding user to Mailchimp audience", err); + firebase_functions_1.logger.error("Error when adding user to Mailchimp audience", err); }; exports.errorRemoveUser = (err) => { - logger.error("Error when removing user from Mailchimp audience", err); + firebase_functions_1.logger.error("Error when removing user from Mailchimp audience", err); }; exports.init = () => { - logger.log("Initializing extension with configuration", exports.obfuscatedConfig); + firebase_functions_1.logger.log("Initializing extension with configuration", exports.obfuscatedConfig); }; exports.initError = (err) => { - logger.error("Error when initializing extension", err); + firebase_functions_1.logger.error("Error when initializing extension", err); }; exports.mailchimpNotInitialized = () => { - logger.error("Mailchimp was not initialized correctly, check for errors in the logs"); + firebase_functions_1.logger.error("Mailchimp was not initialized correctly, check for errors in the logs"); }; exports.start = () => { - logger.log("Started execution of extension with configuration", exports.obfuscatedConfig); + firebase_functions_1.logger.log("Started execution of extension with configuration", exports.obfuscatedConfig); }; exports.userAdded = (userId, audienceId, mailchimpId, status) => { - logger.log(`Added user: ${userId} with status '${status}' to Mailchimp audience: ${audienceId} with Mailchimp ID: ${mailchimpId}`); + firebase_functions_1.logger.log(`Added user: ${userId} with status '${status}' to Mailchimp audience: ${audienceId} with Mailchimp ID: ${mailchimpId}`); }; exports.userAdding = (userId, audienceId) => { - logger.log(`Adding user: ${userId} to Mailchimp audience: ${audienceId}`); + firebase_functions_1.logger.log(`Adding user: ${userId} to Mailchimp audience: ${audienceId}`); }; exports.userNoEmail = () => { - logger.log("User does not have an email"); + firebase_functions_1.logger.log("User does not have an email"); }; exports.userRemoved = (userId, hashedEmail, audienceId) => { - logger.log(`Removed user: ${userId} with hashed email: ${hashedEmail} from Mailchimp audience: ${audienceId}`); + firebase_functions_1.logger.log(`Removed user: ${userId} with hashed email: ${hashedEmail} from Mailchimp audience: ${audienceId}`); }; exports.userRemoving = (userId, hashedEmail, audienceId) => { - logger.log(`Removing user: ${userId} with hashed email: ${hashedEmail} from Mailchimp audience: ${audienceId}`); + firebase_functions_1.logger.log(`Removing user: ${userId} with hashed email: ${hashedEmail} from Mailchimp audience: ${audienceId}`); }; diff --git a/firestore-bigquery-export/functions/lib/logs.js b/firestore-bigquery-export/functions/lib/logs.js index 090409138..10abf8917 100644 --- a/firestore-bigquery-export/functions/lib/logs.js +++ b/firestore-bigquery-export/functions/lib/logs.js @@ -15,7 +15,6 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); -exports.timestampMissingValue = exports.start = exports.init = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0; const config_1 = require("./config"); exports.arrayFieldInvalid = (fieldName) => { console.warn(`Array field '${fieldName}' does not contain an array, skipping`); diff --git a/firestore-bigquery-export/functions/lib/util.js b/firestore-bigquery-export/functions/lib/util.js index fc29c0d8b..440e1709d 100644 --- a/firestore-bigquery-export/functions/lib/util.js +++ b/firestore-bigquery-export/functions/lib/util.js @@ -15,7 +15,6 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); -exports.getDocumentId = exports.getChangeType = void 0; const firestore_bigquery_change_tracker_1 = require("@firebaseextensions/firestore-bigquery-change-tracker"); function getChangeType(change) { if (!change.after.exists) { diff --git a/firestore-translate-text/functions/lib/index.js b/firestore-translate-text/functions/lib/index.js index aaceec6bf..cad8cacf6 100644 --- a/firestore-translate-text/functions/lib/index.js +++ b/firestore-translate-text/functions/lib/index.js @@ -15,6 +15,7 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); +exports.fstranslate = void 0; const admin = require("firebase-admin"); const functions = require("firebase-functions"); const translate_1 = require("@google-cloud/translate"); diff --git a/firestore-translate-text/functions/lib/logs/messages.js b/firestore-translate-text/functions/lib/logs/messages.js index 0cee416f5..5b25303c7 100644 --- a/firestore-translate-text/functions/lib/logs/messages.js +++ b/firestore-translate-text/functions/lib/logs/messages.js @@ -1,5 +1,6 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); +exports.messages = void 0; exports.messages = { complete: () => "Completed execution of extension", documentCreatedNoInput: () => "Document was created without an input string, no processing is required", diff --git a/firestore-translate-text/functions/lib/validators.js b/firestore-translate-text/functions/lib/validators.js index ddc5469d2..e900c3520 100644 --- a/firestore-translate-text/functions/lib/validators.js +++ b/firestore-translate-text/functions/lib/validators.js @@ -15,6 +15,7 @@ * limitations under the License. */ Object.defineProperty(exports, "__esModule", { value: true }); +exports.fieldNameIsTranslationPath = exports.fieldNamesMatch = void 0; exports.fieldNamesMatch = (field1, field2) => field1 === field2; exports.fieldNameIsTranslationPath = (inputFieldName, outputFieldName, languages) => { for (const language of languages) { diff --git a/storage-resize-images/functions/lib/config.js b/storage-resize-images/functions/lib/config.js deleted file mode 100644 index 9d7f650ee..000000000 --- a/storage-resize-images/functions/lib/config.js +++ /dev/null @@ -1,24 +0,0 @@ -"use strict"; -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -Object.defineProperty(exports, "__esModule", { value: true }); -exports.default = { - bucket: process.env.IMG_BUCKET, - cacheControlHeader: process.env.CACHE_CONTROL_HEADER, - imageSizes: process.env.IMG_SIZES.split(","), - resizedImagesPath: process.env.RESIZED_IMAGES_PATH, - deleteOriginalFile: process.env.DELETE_ORIGINAL_FILE === "true", -}; diff --git a/storage-resize-images/functions/lib/index.js b/storage-resize-images/functions/lib/index.js deleted file mode 100644 index 5ed9a86b0..000000000 --- a/storage-resize-images/functions/lib/index.js +++ /dev/null @@ -1,211 +0,0 @@ -"use strict"; -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -Object.defineProperty(exports, "__esModule", { value: true }); -exports.generateResizedImage = void 0; -const admin = require("firebase-admin"); -const fs = require("fs"); -const functions = require("firebase-functions"); -const mkdirp = require("mkdirp"); -const os = require("os"); -const path = require("path"); -const sharp = require("sharp"); -const uuidv4_1 = require("uuidv4"); -const config_1 = require("./config"); -const logs = require("./logs"); -const util_1 = require("./util"); -sharp.cache(false); -// Initialize the Firebase Admin SDK -admin.initializeApp(); -logs.init(); -/** - * Supported file types - */ -const supportedContentTypes = [ - "image/jpeg", - "image/png", - "image/tiff", - "image/webp", -]; -/** - * When an image is uploaded in the Storage bucket, we generate a resized image automatically using - * the Sharp image converting library. - */ -exports.generateResizedImage = functions.storage.object().onFinalize(async (object) => { - logs.start(); - const { contentType } = object; // This is the image MIME type - if (!contentType) { - logs.noContentType(); - return; - } - if (!contentType.startsWith("image/")) { - logs.contentTypeInvalid(contentType); - return; - } - if (!supportedContentTypes.includes(contentType)) { - logs.unsupportedType(supportedContentTypes, contentType); - return; - } - if (object.metadata && object.metadata.resizedImage === "true") { - logs.imageAlreadyResized(); - return; - } - const bucket = admin.storage().bucket(object.bucket); - const filePath = object.name; // File path in the bucket. - const fileDir = path.dirname(filePath); - const fileExtension = path.extname(filePath); - const fileNameWithoutExtension = util_1.extractFileNameWithoutExtension(filePath, fileExtension); - const objectMetadata = object; - let originalFile; - let remoteFile; - try { - originalFile = path.join(os.tmpdir(), filePath); - const tempLocalDir = path.dirname(originalFile); - // Create the temp directory where the storage file will be downloaded. - logs.tempDirectoryCreating(tempLocalDir); - await mkdirp(tempLocalDir); - logs.tempDirectoryCreated(tempLocalDir); - // Download file from bucket. - remoteFile = bucket.file(filePath); - logs.imageDownloading(filePath); - await remoteFile.download({ destination: originalFile }); - logs.imageDownloaded(filePath, originalFile); - // Convert to a set to remove any duplicate sizes - const imageSizes = new Set(config_1.default.imageSizes); - const tasks = []; - imageSizes.forEach((size) => { - tasks.push(resizeImage({ - bucket, - originalFile, - fileDir, - fileNameWithoutExtension, - fileExtension, - contentType, - size, - objectMetadata: objectMetadata, - })); - }); - const results = await Promise.all(tasks); - const failed = results.some((result) => result.success === false); - if (failed) { - logs.failed(); - return; - } - logs.complete(); - } - catch (err) { - logs.error(err); - } - finally { - if (originalFile) { - logs.tempOriginalFileDeleting(filePath); - fs.unlinkSync(originalFile); - logs.tempOriginalFileDeleted(filePath); - } - if (config_1.default.deleteOriginalFile) { - // Delete the original file - if (remoteFile) { - try { - logs.remoteFileDeleting(filePath); - await remoteFile.delete(); - logs.remoteFileDeleted(filePath); - } - catch (err) { - logs.errorDeleting(err); - } - } - } - } -}); -function resize(originalFile, resizedFile, size) { - let height, width; - if (size.indexOf(",") !== -1) { - [width, height] = size.split(","); - } - else if (size.indexOf("x") !== -1) { - [width, height] = size.split("x"); - } - else { - throw new Error("height and width are not delimited by a ',' or a 'x'"); - } - return sharp(originalFile) - .rotate() - .resize(parseInt(width, 10), parseInt(height, 10), { - fit: "inside", - withoutEnlargement: true, - }) - .toFile(resizedFile); -} -const resizeImage = async ({ bucket, originalFile, fileDir, fileNameWithoutExtension, fileExtension, contentType, size, objectMetadata, }) => { - const resizedFileName = `${fileNameWithoutExtension}_${size}${fileExtension}`; - // Path where resized image will be uploaded to in Storage. - const resizedFilePath = path.normalize(config_1.default.resizedImagesPath - ? path.join(fileDir, config_1.default.resizedImagesPath, resizedFileName) - : path.join(fileDir, resizedFileName)); - let resizedFile; - try { - resizedFile = path.join(os.tmpdir(), resizedFileName); - // Cloud Storage files. - const metadata = { - contentDisposition: objectMetadata.contentDisposition, - contentEncoding: objectMetadata.contentEncoding, - contentLanguage: objectMetadata.contentLanguage, - contentType: contentType, - metadata: objectMetadata.metadata || {}, - }; - metadata.metadata.resizedImage = true; - if (config_1.default.cacheControlHeader) { - metadata.cacheControl = config_1.default.cacheControlHeader; - } - else { - metadata.cacheControl = objectMetadata.cacheControl; - } - // If the original image has a download token, add a - // new token to the image being resized #323 - if (metadata.metadata.firebaseStorageDownloadTokens) { - metadata.metadata.firebaseStorageDownloadTokens = uuidv4_1.uuid(); - } - // Generate a resized image using Sharp. - logs.imageResizing(resizedFile, size); - await resize(originalFile, resizedFile, size); - logs.imageResized(resizedFile); - // Uploading the resized image. - logs.imageUploading(resizedFilePath); - await bucket.upload(resizedFile, { - destination: resizedFilePath, - metadata, - }); - logs.imageUploaded(resizedFilePath); - return { size, success: true }; - } - catch (err) { - logs.error(err); - return { size, success: false }; - } - finally { - try { - // Make sure the local resized file is cleaned up to free up disk space. - if (resizedFile) { - logs.tempResizedFileDeleting(resizedFilePath); - fs.unlinkSync(resizedFile); - logs.tempResizedFileDeleted(resizedFilePath); - } - } - catch (err) { - logs.errorDeleting(err); - } - } -}; diff --git a/storage-resize-images/functions/lib/logs.js b/storage-resize-images/functions/lib/logs.js deleted file mode 100644 index 03f2417da..000000000 --- a/storage-resize-images/functions/lib/logs.js +++ /dev/null @@ -1,92 +0,0 @@ -"use strict"; -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -Object.defineProperty(exports, "__esModule", { value: true }); -exports.remoteFileDeleting = exports.remoteFileDeleted = exports.tempResizedFileDeleting = exports.tempResizedFileDeleted = exports.tempOriginalFileDeleting = exports.tempOriginalFileDeleted = exports.tempDirectoryCreating = exports.tempDirectoryCreated = exports.start = exports.init = exports.imageUploading = exports.imageUploaded = exports.imageResizing = exports.imageResized = exports.imageDownloading = exports.imageDownloaded = exports.imageAlreadyResized = exports.failed = exports.errorDeleting = exports.error = exports.unsupportedType = exports.contentTypeInvalid = exports.noContentType = exports.complete = void 0; -const firebase_functions_1 = require("firebase-functions"); -const config_1 = require("./config"); -exports.complete = () => { - firebase_functions_1.logger.log("Completed execution of extension"); -}; -exports.noContentType = () => { - firebase_functions_1.logger.log(`File has no Content-Type, no processing is required`); -}; -exports.contentTypeInvalid = (contentType) => { - firebase_functions_1.logger.log(`File of type '${contentType}' is not an image, no processing is required`); -}; -exports.unsupportedType = (unsupportedTypes, contentType) => { - firebase_functions_1.logger.log(`Image type '${contentType}' is not supported, here are the supported file types: ${unsupportedTypes.join(", ")}`); -}; -exports.error = (err) => { - console.error("Error when resizing image", err); -}; -exports.errorDeleting = (err) => { - console.warn("Error when deleting temporary files", err); -}; -exports.failed = () => { - firebase_functions_1.logger.log("Failed execution of extension"); -}; -exports.imageAlreadyResized = () => { - firebase_functions_1.logger.log("File is already a resized image, no processing is required"); -}; -exports.imageDownloaded = (remotePath, localPath) => { - firebase_functions_1.logger.log(`Downloaded image file: '${remotePath}' to '${localPath}'`); -}; -exports.imageDownloading = (path) => { - firebase_functions_1.logger.log(`Downloading image file: '${path}'`); -}; -exports.imageResized = (path) => { - firebase_functions_1.logger.log(`Resized image created at '${path}'`); -}; -exports.imageResizing = (path, size) => { - firebase_functions_1.logger.log(`Resizing image at path '${path}' to size: ${size}`); -}; -exports.imageUploaded = (path) => { - firebase_functions_1.logger.log(`Uploaded resized image to '${path}'`); -}; -exports.imageUploading = (path) => { - firebase_functions_1.logger.log(`Uploading resized image to '${path}'`); -}; -exports.init = () => { - firebase_functions_1.logger.log("Initializing extension with configuration", config_1.default); -}; -exports.start = () => { - firebase_functions_1.logger.log("Started execution of extension with configuration", config_1.default); -}; -exports.tempDirectoryCreated = (directory) => { - firebase_functions_1.logger.log(`Created temporary directory: '${directory}'`); -}; -exports.tempDirectoryCreating = (directory) => { - firebase_functions_1.logger.log(`Creating temporary directory: '${directory}'`); -}; -exports.tempOriginalFileDeleted = (path) => { - firebase_functions_1.logger.log(`Deleted temporary original file: '${path}'`); -}; -exports.tempOriginalFileDeleting = (path) => { - firebase_functions_1.logger.log(`Deleting temporary original file: '${path}'`); -}; -exports.tempResizedFileDeleted = (path) => { - firebase_functions_1.logger.log(`Deleted temporary resized file: '${path}'`); -}; -exports.tempResizedFileDeleting = (path) => { - firebase_functions_1.logger.log(`Deleting temporary resized file: '${path}'`); -}; -exports.remoteFileDeleted = (path) => { - firebase_functions_1.logger.log(`Deleted original file from storage bucket: '${path}'`); -}; -exports.remoteFileDeleting = (path) => { - firebase_functions_1.logger.log(`Deleting original file from storage bucket: '${path}'`); -}; diff --git a/storage-resize-images/functions/lib/util.js b/storage-resize-images/functions/lib/util.js deleted file mode 100644 index 660859c9e..000000000 --- a/storage-resize-images/functions/lib/util.js +++ /dev/null @@ -1,7 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.extractFileNameWithoutExtension = void 0; -const path = require("path"); -exports.extractFileNameWithoutExtension = (filePath, ext) => { - return path.basename(filePath, ext); -}; From 48557c2e2bf80c30c66db268d1740e13f7689221 Mon Sep 17 00:00:00 2001 From: Lauren Long Date: Mon, 24 Aug 2020 13:57:41 -0700 Subject: [PATCH 21/24] chore(*) Regenerate READMEs --- auth-mailchimp-sync/functions/README.md | 52 --------------- firestore-bigquery-export/README.md | 79 ----------------------- firestore-send-email/functions/README.md | 76 ---------------------- storage-resize-images/README.md | 81 ------------------------ 4 files changed, 288 deletions(-) diff --git a/auth-mailchimp-sync/functions/README.md b/auth-mailchimp-sync/functions/README.md index fa4d44e74..e69de29bb 100644 --- a/auth-mailchimp-sync/functions/README.md +++ b/auth-mailchimp-sync/functions/README.md @@ -1,52 +0,0 @@ -# Sync with Mailchimp - -**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) - -**Description**: Adds new users from Firebase Authentication to a specified Mailchimp audience. - - - -**Details**: Use this extension to add new users to an existing [Mailchimp](https://mailchimp.com) audience. - -This extension adds the email address of each new user to your specified Mailchimp audience. Also, if the user deletes their user account for your app, this extension removes the user from the Mailchimp audience. - -**Note:** To use this extension, you need to manage your users with Firebase Authentication. - -This extension uses Mailchimp, so you'll need to supply your Mailchimp API Key and Audience ID when installing this extension. - -#### Additional setup - -Make sure that you've set up [Firebase Authentication](https://firebase.google.com/docs/auth) to manage your users. - -You must also have a Mailchimp account before installing this extension. - -#### Billing - -To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) - -- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). -- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - - Cloud Functions (Node.js 10+ runtime. See [FAQs](https://firebase.google.com/support/faq#expandable-24)) - -Usage of this extension also requires you to have a Mailchimp account. You are responsible for any associated costs with your usage of Mailchimp. - - - - -**Configuration Parameters:** - -* Cloud Functions location: Where do you want to deploy the functions created for this extension? - -* Mailchimp API key: What is your Mailchimp API key? To obtain a Mailchimp API key, go to your [Mailchimp account](https://admin.mailchimp.com/account/api/). - -* Audience ID: What is the Mailchimp Audience ID to which you want to subscribe new users? To find your Audience ID: visit https://admin.mailchimp.com/lists, click on the desired audience or create a new audience, then select **Settings**. Look for **Audience ID** (for example, `27735fc60a`). - -* Contact status: When the extension adds a new user to the Mailchimp audience, what is their initial status? This value can be `subscribed` or `pending`. `subscribed` means the user can receive campaigns; `pending` means the user still needs to opt-in to receive campaigns. - - - -**Cloud Functions:** - -* **addUserToList:** Listens for new user accounts (as managed by Firebase Authentication), then automatically adds the new user to your specified MailChimp audience. - -* **removeUserFromList:** Listens for existing user accounts to be deleted (as managed by Firebase Authentication), then automatically removes them from your specified MailChimp audience. diff --git a/firestore-bigquery-export/README.md b/firestore-bigquery-export/README.md index 241bb92d6..e69de29bb 100644 --- a/firestore-bigquery-export/README.md +++ b/firestore-bigquery-export/README.md @@ -1,79 +0,0 @@ -# Export Collections to BigQuery - -**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) - -**Description**: Sends realtime, incremental updates from a specified Cloud Firestore collection to BigQuery. - - - -**Details**: Use this extension to export the documents in a Cloud Firestore collection to BigQuery. Exports are realtime and incremental, so the data in BigQuery is a mirror of your content in Cloud Firestore. - -The extension creates and updates a [dataset](https://cloud.google.com/bigquery/docs/datasets-intro) containing the following two BigQuery resources: - -- A [table](https://cloud.google.com/bigquery/docs/tables-intro) of raw data that stores a full change history of the documents within your collection. This table includes a number of metadata fields so that BigQuery can display the current state of your data. The principle metadata fields are `timestamp`, `document_name`, and the `operation` for the document change. -- A [view](https://cloud.google.com/bigquery/docs/views-intro) which represents the current state of the data within your collection. It also shows a log of the latest `operation` for each document (`CREATE`, `UPDATE`, or `IMPORT`). - -If you create, update, delete, or import a document in the specified collection, this extension sends that update to BigQuery. You can then run queries on this mirrored dataset. - -Note that this extension only listens for _document_ changes in the collection, but not changes in any _subcollection_. You can, though, install additional instances of this extension to specifically listen to a subcollection or other collections in your database. Or if you have the same subcollection across documents in a given collection, you can use `{wildcard}` notation to listen to all those subcollections (for example: `chats/{chatid}/posts`). - -#### Additional setup - -Before installing this extension, you'll need to: - -- [Set up Cloud Firestore in your Firebase project.](https://firebase.google.com/docs/firestore/quickstart) -- [Link your Firebase project to BigQuery.](https://support.google.com/firebase/answer/6318765) - -#### Backfill your BigQuery dataset - -This extension only sends the content of documents that have been changed -- it does not export your full dataset of existing documents into BigQuery. So, to backfill your BigQuery dataset with all the documents in your collection, you can run the [import script](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/IMPORT_EXISTING_DOCUMENTS.md) provided by this extension. - -**Important:** Run the import script over the entire collection _after_ installing this extension, otherwise all writes to your database during the import might be lost. - -#### Generate schema views - -After your data is in BigQuery, you can run the [schema-views script](https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/guides/GENERATE_SCHEMA_VIEWS.md) (provided by this extension) to create views that make it easier to query relevant data. You only need to provide a JSON schema file that describes your data structure, and the schema-views script will create the views. - -#### Billing - -To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) - -- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). -- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - - BigQuery (this extension writes to BigQuery with [streaming inserts](https://cloud.google.com/bigquery/pricing#streaming_pricing)) - - Cloud Firestore - - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) - - - -**Configuration Parameters:** - -* Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). Note that this extension locates your BigQuery dataset in `us-central1`. - -* Collection path: What is the path of the collection that you would like to export? You may use `{wildcard}` notation to match a subcollection of all documents in a collection (for example: `chatrooms/{chatid}/posts`). - -* Dataset ID: What ID would you like to use for your BigQuery dataset? This extension will create the dataset, if it doesn't already exist. - -* Table ID: What identifying prefix would you like to use for your table and view inside your BigQuery dataset? This extension will create the table and view, if they don't already exist. - - - -**Cloud Functions:** - -* **fsexportbigquery:** Listens for document changes in your specified Cloud Firestore collection, then exports the changes into BigQuery. - - - -**APIs Used**: - -* bigquery-json.googleapis.com (Reason: Mirrors data from your Cloud Firestore collection in BigQuery.) - - - -**Access Required**: - - - -This extension will operate with the following project IAM roles: - -* bigquery.dataEditor (Reason: Allows the extension to configure and export data into BigQuery.) diff --git a/firestore-send-email/functions/README.md b/firestore-send-email/functions/README.md index af74d4daa..e69de29bb 100644 --- a/firestore-send-email/functions/README.md +++ b/firestore-send-email/functions/README.md @@ -1,76 +0,0 @@ -# Trigger Email - -**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) - -**Description**: Composes and sends an email based on the contents of a document written to a specified Cloud Firestore collection. - - - -**Details**: Use this extension to render and send emails that contain the information from documents added to a specified Cloud Firestore collection. - -Adding a document triggers this extension to send an email built from the document's fields. The document's top-level fields specify the email sender and recipients, including `to`, `cc`, and `bcc` options (each supporting UIDs). The document's `message` field specifies the other email elements, like subject line and email body (either plaintext or HTML) - -Here's a basic example document write that would trigger this extension: - -```js -admin.firestore().collection('mail').add({ - to: 'someone@example.com', - message: { - subject: 'Hello from Firebase!', - html: 'This is an HTML email body.', - }, -}) -``` - -You can also optionally configure this extension to render emails using [Handlebar](https://handlebarsjs.com/) templates. Each template is a document stored in a Cloud Firestore collection. - -When you configure this extension, you'll need to supply your **SMTP credentials for mail delivery**. Note that this extension is for use with bulk email service providers, like SendGrid, Mailgun, etc. - -#### Additional setup - -Before installing this extension, make sure that you've [set up a Cloud Firestore database](https://firebase.google.com/docs/firestore/quickstart) in your Firebase project. - -#### Billing -To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) - -- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). -- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - - Cloud Firestore - - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) - -Usage of this extension also requires you to have SMTP credentials for mail delivery. You are responsible for any associated costs with your usage of your SMTP provider. - - - - -**Configuration Parameters:** - -* Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). - -* SMTP connection URI: A URI representing an SMTP server that this extension can use to deliver email. - -* Email documents collection: What is the path to the collection that contains the documents used to build and send the emails? - -* Default FROM address: The email address to use as the sender's address (if it's not specified in the added email document). You can optionally include a name with the email address (`Friendly Firebaser `). - -* Default REPLY-TO address: The email address to use as the reply-to address (if it's not specified in the added email document). - -* Users collection: A collection of documents keyed by user UID. If the `toUids`, `ccUids`, and/or `bccUids` recipient options are used in the added email document, this extension delivers email to the `email` field based on lookups in this collection. - -* Templates collection: A collection of email templates keyed by name. This extension can render an email using a [Handlebar](https://handlebarsjs.com/) template, if the template is specified in the added email document. - - - -**Cloud Functions:** - -* **processQueue:** Processes document changes in the specified Cloud Firestore collection, delivers emails, and updates the document with delivery status information. - - - -**Access Required**: - - - -This extension will operate with the following project IAM roles: - -* datastore.user (Reason: Allows this extension to access Cloud Firestore to read and process added email documents.) diff --git a/storage-resize-images/README.md b/storage-resize-images/README.md index 445e4417b..e69de29bb 100644 --- a/storage-resize-images/README.md +++ b/storage-resize-images/README.md @@ -1,81 +0,0 @@ -# Resize Images - -**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) - -**Description**: Resizes images uploaded to Cloud Storage to a specified size, and optionally keeps or deletes the original image. - - - -**Details**: Use this extension to create resized versions of an image uploaded to a Cloud Storage bucket. - -When you upload an image file to your specified Cloud Storage bucket, this extension: - -- Creates a resized image with your specified dimensions. -- Names the resized image using the same name as the original uploaded image, but suffixed with your specified width and height. -- Stores the resized image in the same Storage bucket as the original uploaded image. - -You can even configure the extension to create resized images of different dimensions for each original image upload. For example, you might want images that are 200x200, 400x400, and 680x680 - this extension can create these three resized images then store them in your bucket. - -The extension automatically copies the following metadata, if present, from the original image to the resized image(s): `Cache-Control`, `Content-Disposition`, `Content-Encoding`, `Content-Language`, `Content-Type`, and user-provided metadata (a new Firebase storage download token will be generated on the resized image(s) if the original metadata contains a token). Note that you can optionally configure the extension to overwrite the [`Cache-Control`](https://developer.mozilla.org/docs/Web/HTTP/Headers/Cache-Control) value for the resized image(s). - -#### Detailed configuration information - -To configure this extension, you specify a maximum width and a maximum height (in pixels, px). This extension keeps the aspect ratio of uploaded images constant and shrinks the image until the resized image's dimensions are at or under your specified max width and height. - -For example, say that you specify a max width of 200px and a max height of 100px. You upload an image that is 480px wide by 640px high, which means a 0.75 aspect ratio. The final resized image will be 75px wide by 100px high to maintain the aspect ratio while also being at or under both of your maximum specified dimensions. - -#### Additional setup - -Before installing this extension, make sure that you've [set up a Cloud Storage bucket](https://firebase.google.com/docs/storage) in your Firebase project. - -#### Billing - -To install an extension, your project must be on the [Blaze (pay as you go) plan](https://firebase.google.com/pricing) - -- You will be charged a small amount (typically around $0.01/month) for the Firebase resources required by this extension (even if it is not used). -- This extension uses other Firebase and Google Cloud Platform services, which have associated charges if you exceed the service’s free tier: - - Cloud Storage - - Cloud Functions (Node.js 10+ runtime. [See FAQs](https://firebase.google.com/support/faq#expandable-24)) - - - - -**Configuration Parameters:** - -* Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your Storage bucket. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). - -* Cloud Storage bucket for images: To which Cloud Storage bucket will you upload images that you want to resize? Resized images will be stored in this bucket. Depending on your extension configuration, original images are either kept or deleted. - - -* Sizes of resized images: What sizes of images would you like (in pixels)? Enter the sizes as a comma-separated list of WIDTHxHEIGHT values. Learn more about [how this parameter works](https://firebase.google.com/products/extensions/storage-resize-images). - - -* Deletion of original file: Do you want to automatically delete the original file from the Cloud Storage bucket? Note that these deletions cannot be undone. - -* Cloud Storage path for resized images: A relative path in which to store resized images. For example, if you specify a path here of `thumbs` and you upload an image to `/images/original.jpg`, then the resized image is stored at `/images/thumbs/original_200x200.jpg`. If you prefer to store resized images at the root of your bucket, leave this field empty. - - -* Cache-Control header for resized images: This extension automatically copies any `Cache-Control` metadata from the original image to the resized images. For the resized images, do you want to overwrite this copied `Cache-Control` metadata or add `Cache-Control` metadata? Learn more about [`Cache-Control` headers](https://developer.mozilla.org/docs/Web/HTTP/Headers/Cache-Control). If you prefer not to overwrite or add `Cache-Control` metadata, leave this field empty. - - - - -**Cloud Functions:** - -* **generateResizedImage:** Listens for new images uploaded to your specified Cloud Storage bucket, resizes the images, then stores the resized images in the same bucket. Optionally keeps or deletes the original images. - - - -**APIs Used**: - -* storage-component.googleapis.com (Reason: Needed to use Cloud Storage) - - - -**Access Required**: - - - -This extension will operate with the following project IAM roles: - -* storage.admin (Reason: Allows the extension to store resized images in Cloud Storage) From 81e209ea2e3b27492205492c73f22240f9d3e5a9 Mon Sep 17 00:00:00 2001 From: Lauren Long Date: Mon, 24 Aug 2020 14:00:02 -0700 Subject: [PATCH 22/24] chore(storage-resize-images) Regenerate JS files --- storage-resize-images/functions/lib/config.js | 24 ++ storage-resize-images/functions/lib/index.js | 211 ++++++++++++++++++ storage-resize-images/functions/lib/logs.js | 92 ++++++++ storage-resize-images/functions/lib/util.js | 7 + storage-resize-images/tsconfig.json | 2 +- 5 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 storage-resize-images/functions/lib/config.js create mode 100644 storage-resize-images/functions/lib/index.js create mode 100644 storage-resize-images/functions/lib/logs.js create mode 100644 storage-resize-images/functions/lib/util.js diff --git a/storage-resize-images/functions/lib/config.js b/storage-resize-images/functions/lib/config.js new file mode 100644 index 000000000..9d7f650ee --- /dev/null +++ b/storage-resize-images/functions/lib/config.js @@ -0,0 +1,24 @@ +"use strict"; +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.default = { + bucket: process.env.IMG_BUCKET, + cacheControlHeader: process.env.CACHE_CONTROL_HEADER, + imageSizes: process.env.IMG_SIZES.split(","), + resizedImagesPath: process.env.RESIZED_IMAGES_PATH, + deleteOriginalFile: process.env.DELETE_ORIGINAL_FILE === "true", +}; diff --git a/storage-resize-images/functions/lib/index.js b/storage-resize-images/functions/lib/index.js new file mode 100644 index 000000000..5ed9a86b0 --- /dev/null +++ b/storage-resize-images/functions/lib/index.js @@ -0,0 +1,211 @@ +"use strict"; +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.generateResizedImage = void 0; +const admin = require("firebase-admin"); +const fs = require("fs"); +const functions = require("firebase-functions"); +const mkdirp = require("mkdirp"); +const os = require("os"); +const path = require("path"); +const sharp = require("sharp"); +const uuidv4_1 = require("uuidv4"); +const config_1 = require("./config"); +const logs = require("./logs"); +const util_1 = require("./util"); +sharp.cache(false); +// Initialize the Firebase Admin SDK +admin.initializeApp(); +logs.init(); +/** + * Supported file types + */ +const supportedContentTypes = [ + "image/jpeg", + "image/png", + "image/tiff", + "image/webp", +]; +/** + * When an image is uploaded in the Storage bucket, we generate a resized image automatically using + * the Sharp image converting library. + */ +exports.generateResizedImage = functions.storage.object().onFinalize(async (object) => { + logs.start(); + const { contentType } = object; // This is the image MIME type + if (!contentType) { + logs.noContentType(); + return; + } + if (!contentType.startsWith("image/")) { + logs.contentTypeInvalid(contentType); + return; + } + if (!supportedContentTypes.includes(contentType)) { + logs.unsupportedType(supportedContentTypes, contentType); + return; + } + if (object.metadata && object.metadata.resizedImage === "true") { + logs.imageAlreadyResized(); + return; + } + const bucket = admin.storage().bucket(object.bucket); + const filePath = object.name; // File path in the bucket. + const fileDir = path.dirname(filePath); + const fileExtension = path.extname(filePath); + const fileNameWithoutExtension = util_1.extractFileNameWithoutExtension(filePath, fileExtension); + const objectMetadata = object; + let originalFile; + let remoteFile; + try { + originalFile = path.join(os.tmpdir(), filePath); + const tempLocalDir = path.dirname(originalFile); + // Create the temp directory where the storage file will be downloaded. + logs.tempDirectoryCreating(tempLocalDir); + await mkdirp(tempLocalDir); + logs.tempDirectoryCreated(tempLocalDir); + // Download file from bucket. + remoteFile = bucket.file(filePath); + logs.imageDownloading(filePath); + await remoteFile.download({ destination: originalFile }); + logs.imageDownloaded(filePath, originalFile); + // Convert to a set to remove any duplicate sizes + const imageSizes = new Set(config_1.default.imageSizes); + const tasks = []; + imageSizes.forEach((size) => { + tasks.push(resizeImage({ + bucket, + originalFile, + fileDir, + fileNameWithoutExtension, + fileExtension, + contentType, + size, + objectMetadata: objectMetadata, + })); + }); + const results = await Promise.all(tasks); + const failed = results.some((result) => result.success === false); + if (failed) { + logs.failed(); + return; + } + logs.complete(); + } + catch (err) { + logs.error(err); + } + finally { + if (originalFile) { + logs.tempOriginalFileDeleting(filePath); + fs.unlinkSync(originalFile); + logs.tempOriginalFileDeleted(filePath); + } + if (config_1.default.deleteOriginalFile) { + // Delete the original file + if (remoteFile) { + try { + logs.remoteFileDeleting(filePath); + await remoteFile.delete(); + logs.remoteFileDeleted(filePath); + } + catch (err) { + logs.errorDeleting(err); + } + } + } + } +}); +function resize(originalFile, resizedFile, size) { + let height, width; + if (size.indexOf(",") !== -1) { + [width, height] = size.split(","); + } + else if (size.indexOf("x") !== -1) { + [width, height] = size.split("x"); + } + else { + throw new Error("height and width are not delimited by a ',' or a 'x'"); + } + return sharp(originalFile) + .rotate() + .resize(parseInt(width, 10), parseInt(height, 10), { + fit: "inside", + withoutEnlargement: true, + }) + .toFile(resizedFile); +} +const resizeImage = async ({ bucket, originalFile, fileDir, fileNameWithoutExtension, fileExtension, contentType, size, objectMetadata, }) => { + const resizedFileName = `${fileNameWithoutExtension}_${size}${fileExtension}`; + // Path where resized image will be uploaded to in Storage. + const resizedFilePath = path.normalize(config_1.default.resizedImagesPath + ? path.join(fileDir, config_1.default.resizedImagesPath, resizedFileName) + : path.join(fileDir, resizedFileName)); + let resizedFile; + try { + resizedFile = path.join(os.tmpdir(), resizedFileName); + // Cloud Storage files. + const metadata = { + contentDisposition: objectMetadata.contentDisposition, + contentEncoding: objectMetadata.contentEncoding, + contentLanguage: objectMetadata.contentLanguage, + contentType: contentType, + metadata: objectMetadata.metadata || {}, + }; + metadata.metadata.resizedImage = true; + if (config_1.default.cacheControlHeader) { + metadata.cacheControl = config_1.default.cacheControlHeader; + } + else { + metadata.cacheControl = objectMetadata.cacheControl; + } + // If the original image has a download token, add a + // new token to the image being resized #323 + if (metadata.metadata.firebaseStorageDownloadTokens) { + metadata.metadata.firebaseStorageDownloadTokens = uuidv4_1.uuid(); + } + // Generate a resized image using Sharp. + logs.imageResizing(resizedFile, size); + await resize(originalFile, resizedFile, size); + logs.imageResized(resizedFile); + // Uploading the resized image. + logs.imageUploading(resizedFilePath); + await bucket.upload(resizedFile, { + destination: resizedFilePath, + metadata, + }); + logs.imageUploaded(resizedFilePath); + return { size, success: true }; + } + catch (err) { + logs.error(err); + return { size, success: false }; + } + finally { + try { + // Make sure the local resized file is cleaned up to free up disk space. + if (resizedFile) { + logs.tempResizedFileDeleting(resizedFilePath); + fs.unlinkSync(resizedFile); + logs.tempResizedFileDeleted(resizedFilePath); + } + } + catch (err) { + logs.errorDeleting(err); + } + } +}; diff --git a/storage-resize-images/functions/lib/logs.js b/storage-resize-images/functions/lib/logs.js new file mode 100644 index 000000000..03f2417da --- /dev/null +++ b/storage-resize-images/functions/lib/logs.js @@ -0,0 +1,92 @@ +"use strict"; +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.remoteFileDeleting = exports.remoteFileDeleted = exports.tempResizedFileDeleting = exports.tempResizedFileDeleted = exports.tempOriginalFileDeleting = exports.tempOriginalFileDeleted = exports.tempDirectoryCreating = exports.tempDirectoryCreated = exports.start = exports.init = exports.imageUploading = exports.imageUploaded = exports.imageResizing = exports.imageResized = exports.imageDownloading = exports.imageDownloaded = exports.imageAlreadyResized = exports.failed = exports.errorDeleting = exports.error = exports.unsupportedType = exports.contentTypeInvalid = exports.noContentType = exports.complete = void 0; +const firebase_functions_1 = require("firebase-functions"); +const config_1 = require("./config"); +exports.complete = () => { + firebase_functions_1.logger.log("Completed execution of extension"); +}; +exports.noContentType = () => { + firebase_functions_1.logger.log(`File has no Content-Type, no processing is required`); +}; +exports.contentTypeInvalid = (contentType) => { + firebase_functions_1.logger.log(`File of type '${contentType}' is not an image, no processing is required`); +}; +exports.unsupportedType = (unsupportedTypes, contentType) => { + firebase_functions_1.logger.log(`Image type '${contentType}' is not supported, here are the supported file types: ${unsupportedTypes.join(", ")}`); +}; +exports.error = (err) => { + console.error("Error when resizing image", err); +}; +exports.errorDeleting = (err) => { + console.warn("Error when deleting temporary files", err); +}; +exports.failed = () => { + firebase_functions_1.logger.log("Failed execution of extension"); +}; +exports.imageAlreadyResized = () => { + firebase_functions_1.logger.log("File is already a resized image, no processing is required"); +}; +exports.imageDownloaded = (remotePath, localPath) => { + firebase_functions_1.logger.log(`Downloaded image file: '${remotePath}' to '${localPath}'`); +}; +exports.imageDownloading = (path) => { + firebase_functions_1.logger.log(`Downloading image file: '${path}'`); +}; +exports.imageResized = (path) => { + firebase_functions_1.logger.log(`Resized image created at '${path}'`); +}; +exports.imageResizing = (path, size) => { + firebase_functions_1.logger.log(`Resizing image at path '${path}' to size: ${size}`); +}; +exports.imageUploaded = (path) => { + firebase_functions_1.logger.log(`Uploaded resized image to '${path}'`); +}; +exports.imageUploading = (path) => { + firebase_functions_1.logger.log(`Uploading resized image to '${path}'`); +}; +exports.init = () => { + firebase_functions_1.logger.log("Initializing extension with configuration", config_1.default); +}; +exports.start = () => { + firebase_functions_1.logger.log("Started execution of extension with configuration", config_1.default); +}; +exports.tempDirectoryCreated = (directory) => { + firebase_functions_1.logger.log(`Created temporary directory: '${directory}'`); +}; +exports.tempDirectoryCreating = (directory) => { + firebase_functions_1.logger.log(`Creating temporary directory: '${directory}'`); +}; +exports.tempOriginalFileDeleted = (path) => { + firebase_functions_1.logger.log(`Deleted temporary original file: '${path}'`); +}; +exports.tempOriginalFileDeleting = (path) => { + firebase_functions_1.logger.log(`Deleting temporary original file: '${path}'`); +}; +exports.tempResizedFileDeleted = (path) => { + firebase_functions_1.logger.log(`Deleted temporary resized file: '${path}'`); +}; +exports.tempResizedFileDeleting = (path) => { + firebase_functions_1.logger.log(`Deleting temporary resized file: '${path}'`); +}; +exports.remoteFileDeleted = (path) => { + firebase_functions_1.logger.log(`Deleted original file from storage bucket: '${path}'`); +}; +exports.remoteFileDeleting = (path) => { + firebase_functions_1.logger.log(`Deleting original file from storage bucket: '${path}'`); +}; diff --git a/storage-resize-images/functions/lib/util.js b/storage-resize-images/functions/lib/util.js new file mode 100644 index 000000000..660859c9e --- /dev/null +++ b/storage-resize-images/functions/lib/util.js @@ -0,0 +1,7 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.extractFileNameWithoutExtension = void 0; +const path = require("path"); +exports.extractFileNameWithoutExtension = (filePath, ext) => { + return path.basename(filePath, ext); +}; diff --git a/storage-resize-images/tsconfig.json b/storage-resize-images/tsconfig.json index cdfe29ea4..06eefbe92 100644 --- a/storage-resize-images/tsconfig.json +++ b/storage-resize-images/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../tsconfig.json", "compilerOptions": { - "outDir": "lib", + "outDir": "functions/lib", "target": "es2018" }, "include": ["functions/src"] From 52f82e83b08cec11eeba1a41e617bd407122e4b2 Mon Sep 17 00:00:00 2001 From: Lauren Long Date: Mon, 24 Aug 2020 14:35:53 -0700 Subject: [PATCH 23/24] chore(*) format files --- firestore-bigquery-export/CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/firestore-bigquery-export/CHANGELOG.md b/firestore-bigquery-export/CHANGELOG.md index a9f6377da..67fb56074 100644 --- a/firestore-bigquery-export/CHANGELOG.md +++ b/firestore-bigquery-export/CHANGELOG.md @@ -2,7 +2,6 @@ fixed - Updated `@firebaseextensions/firestore-bigquery-change-tracker` dependency (fixes issues #235). - ## Version 0.1.6 fixed - Fixed issue with timestamp values not showing up in the latest view (#357) From 80965a620896e046d40c83172a22e16a838aafdd Mon Sep 17 00:00:00 2001 From: Lauren Long Date: Thu, 27 Aug 2020 14:31:18 -0700 Subject: [PATCH 24/24] chore(*) Bump versions and add release notes for Node.js 10 release (#440) --- auth-mailchimp-sync/CHANGELOG.md | 4 ++++ auth-mailchimp-sync/extension.yaml | 2 +- delete-user-data/CHANGELOG.md | 7 ++++++- delete-user-data/extension.yaml | 2 +- firestore-bigquery-export/CHANGELOG.md | 12 ++++++++++++ firestore-bigquery-export/extension.yaml | 2 +- firestore-counter/CHANGELOG.md | 4 ++++ firestore-counter/extension.yaml | 2 +- firestore-send-email/CHANGELOG.md | 4 ++++ firestore-send-email/extension.yaml | 2 +- firestore-shorten-urls-bitly/CHANGELOG.md | 4 ++++ firestore-shorten-urls-bitly/extension.yaml | 2 +- firestore-translate-text/CHANGELOG.md | 5 +++++ firestore-translate-text/extension.yaml | 2 +- rtdb-limit-child-nodes/CHANGELOG.md | 4 ++++ rtdb-limit-child-nodes/extension.yaml | 2 +- storage-resize-images/CHANGELOG.md | 9 +++++++++ storage-resize-images/extension.yaml | 2 +- 18 files changed, 61 insertions(+), 10 deletions(-) diff --git a/auth-mailchimp-sync/CHANGELOG.md b/auth-mailchimp-sync/CHANGELOG.md index 9ba62b0be..ea14904ff 100644 --- a/auth-mailchimp-sync/CHANGELOG.md +++ b/auth-mailchimp-sync/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.1 + +feature - Update Cloud Functions runtime to Node.js 10. + ## Version 0.1.0 Initial release of the _Sync with Mailchimp_ extension. diff --git a/auth-mailchimp-sync/extension.yaml b/auth-mailchimp-sync/extension.yaml index bcac90e35..81373f555 100644 --- a/auth-mailchimp-sync/extension.yaml +++ b/auth-mailchimp-sync/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: auth-mailchimp-sync -version: 0.1.0 +version: 0.1.1 specVersion: v1beta displayName: Sync with Mailchimp diff --git a/delete-user-data/CHANGELOG.md b/delete-user-data/CHANGELOG.md index 65bc1afa2..e2171695c 100644 --- a/delete-user-data/CHANGELOG.md +++ b/delete-user-data/CHANGELOG.md @@ -1,6 +1,10 @@ +## Version 0.1.5 + +feature - Update Cloud Functions runtime to Node.js 10. + ## Version 0.1.4 -fixed - updated `firebase-tools` dependency to avoid using deprecated `gcp-metadata` API (Issue #206). +fixed - Updated `firebase-tools` dependency to avoid using deprecated `gcp-metadata` API (Issue #206). **Important:** If you use this extension to delete user data from Cloud Firestore, you must update your extension to at minimum v0.1.4 before April 30, 2020. Otherwise, your installed extension will stop working. No further action is required. @@ -11,6 +15,7 @@ feature - Support deletion of directories (issue #148). ## Version 0.1.2 feature - Add a new param for recursively deleting subcollections in Cloud Firestore (issue #14). + fixed - Fixed "cold start" errors experienced when the extension runs after a period of inactivity (issue #48). ## Version 0.1.1 diff --git a/delete-user-data/extension.yaml b/delete-user-data/extension.yaml index cdd93ba83..9a55a0274 100644 --- a/delete-user-data/extension.yaml +++ b/delete-user-data/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: delete-user-data -version: 0.1.4 +version: 0.1.5 specVersion: v1beta displayName: Delete User Data diff --git a/firestore-bigquery-export/CHANGELOG.md b/firestore-bigquery-export/CHANGELOG.md index 67fb56074..1d961bd49 100644 --- a/firestore-bigquery-export/CHANGELOG.md +++ b/firestore-bigquery-export/CHANGELOG.md @@ -1,3 +1,9 @@ +## Version 0.1.8 + +feature - Update Cloud Functions runtime to Node.js 10. + +feature - Add validation regex for collection path parameter. (#418) + ## Version 0.1.7 fixed - Updated `@firebaseextensions/firestore-bigquery-change-tracker` dependency (fixes issues #235). @@ -5,19 +11,24 @@ fixed - Updated `@firebaseextensions/firestore-bigquery-change-tracker` dependen ## Version 0.1.6 fixed - Fixed issue with timestamp values not showing up in the latest view (#357) + feature - Record document ID of changes tracked by firestore-bigquery-change-tracker package (#374) + feature - Add document ID column to changelog table and snapshot view (#376) ## Version 0.1.5 fixed - TypeError: Cannot read property 'constructor' of null. (Issue #284) + fixed - Filtered out blob (buffer) data types from being stored as strings in BigQuery. ## Version 0.1.4 fixed - Converted circular structure to JSON error. (Issue #236) + fixed - Fixed bug where modules were not sharing the same Cloud Firestore DocumentReference. (Issue #265) + fixed - Updated @firebaseextensions/firestore-bigquery-change-tracker dependency. (Issues #250 and #196) ## Version 0.1.3 @@ -32,6 +43,7 @@ fixed - Updated `@firebaseextensions/firestore-bigquery-change-tracker` dependen ## Version 0.1.2 fixed - Added "IF NOT EXISTS" to safely run `fs-bq-schema-views` script multiple times (PR #193). + fixed - Updated BigQuery dependency in `package.json` for the `fs-bq-import-collection` script (issue #192 and PR #197). ## Version 0.1.1 diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index 3211e610a..d375a9b0e 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-bigquery-export -version: 0.1.7 +version: 0.1.8 specVersion: v1beta displayName: Export Collections to BigQuery diff --git a/firestore-counter/CHANGELOG.md b/firestore-counter/CHANGELOG.md index a9500113b..4b5db1872 100644 --- a/firestore-counter/CHANGELOG.md +++ b/firestore-counter/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.4 + +feature - Update Cloud Functions runtime to Node.js 10. + ## Version 0.1.3 build - Updates the firebase-admin and firebase-functions packages to the latest versions (issue #181). diff --git a/firestore-counter/extension.yaml b/firestore-counter/extension.yaml index 969138371..e22b5a2c4 100644 --- a/firestore-counter/extension.yaml +++ b/firestore-counter/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-counter -version: 0.1.3 +version: 0.1.4 specVersion: v1beta displayName: Distributed Counter diff --git a/firestore-send-email/CHANGELOG.md b/firestore-send-email/CHANGELOG.md index 34ac047a1..3dece18f2 100644 --- a/firestore-send-email/CHANGELOG.md +++ b/firestore-send-email/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.5 + +feature - Update Cloud Functions runtime to Node.js 10. + ## Version 0.1.4 feature - Updated "Default FROM address" parameter to accept either an email address (`foobar@example.com`) _or_ a name plus email address (`Friendly Firebaser `). (issue #167) diff --git a/firestore-send-email/extension.yaml b/firestore-send-email/extension.yaml index 2b49f398e..416e5f668 100644 --- a/firestore-send-email/extension.yaml +++ b/firestore-send-email/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-send-email -version: 0.1.4 +version: 0.1.5 specVersion: v1beta displayName: Trigger Email diff --git a/firestore-shorten-urls-bitly/CHANGELOG.md b/firestore-shorten-urls-bitly/CHANGELOG.md index 39f99f7fc..22c73c50d 100644 --- a/firestore-shorten-urls-bitly/CHANGELOG.md +++ b/firestore-shorten-urls-bitly/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.4 + +feature - Update Cloud Functions runtime to Node.js 10. + ## Version 0.1.3 fixed - Fixed 406 HTTP error code from Bitly API due to `Content-Type` header not being set (#202). diff --git a/firestore-shorten-urls-bitly/extension.yaml b/firestore-shorten-urls-bitly/extension.yaml index 60032debe..4d0a126f9 100644 --- a/firestore-shorten-urls-bitly/extension.yaml +++ b/firestore-shorten-urls-bitly/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-shorten-urls-bitly -version: 0.1.3 +version: 0.1.4 specVersion: v1beta displayName: Shorten URLs diff --git a/firestore-translate-text/CHANGELOG.md b/firestore-translate-text/CHANGELOG.md index 1febf8116..99cec9ad1 100644 --- a/firestore-translate-text/CHANGELOG.md +++ b/firestore-translate-text/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.3 + +feature - Update Cloud Functions runtime to Node.js 10. + ## Version 0.1.2 fixed - Fixed bug where target languages could not be reconfigured. @@ -5,6 +9,7 @@ fixed - Fixed bug where target languages could not be reconfigured. ## Version 0.1.1 fixed - Fixed bug where param validation failed when a single language was entered. + fixed - Fixed "cold start" errors experienced when the extension runs after a period of inactivity (issue #48). ## Version 0.1.0 diff --git a/firestore-translate-text/extension.yaml b/firestore-translate-text/extension.yaml index eb3ab2594..ebde4ff97 100644 --- a/firestore-translate-text/extension.yaml +++ b/firestore-translate-text/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-translate-text -version: 0.1.2 +version: 0.1.3 specVersion: v1beta displayName: Translate Text diff --git a/rtdb-limit-child-nodes/CHANGELOG.md b/rtdb-limit-child-nodes/CHANGELOG.md index f0c421517..06be63c6c 100644 --- a/rtdb-limit-child-nodes/CHANGELOG.md +++ b/rtdb-limit-child-nodes/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.1 + +feature - Update Cloud Functions runtime to Node.js 10. + ## Version 0.1.0 Initial release of the _Limit Child Nodes_ extension. diff --git a/rtdb-limit-child-nodes/extension.yaml b/rtdb-limit-child-nodes/extension.yaml index 902df29e7..dc48bbf1a 100644 --- a/rtdb-limit-child-nodes/extension.yaml +++ b/rtdb-limit-child-nodes/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: rtdb-limit-child-nodes -version: 0.1.0 +version: 0.1.1 specVersion: v1beta displayName: Limit Child Nodes diff --git a/storage-resize-images/CHANGELOG.md b/storage-resize-images/CHANGELOG.md index 792fec7c8..adea986b7 100644 --- a/storage-resize-images/CHANGELOG.md +++ b/storage-resize-images/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.11 + +feature - Update Cloud Functions runtime to Node.js 10. + ## Version 0.1.10 fixed - A fresh token is now generated for each resized image. (Issue #323, PR #351) @@ -5,7 +9,9 @@ fixed - A fresh token is now generated for each resized image. (Issue #323, PR # ## Version 0.1.9 changed - If the original image is a vector image, the extension does not resize it. (Issue #326, PR #329) + fixed - Replaced `mkdirp-promise` with `mkdirp` because `mkdirp-promise` is deprecated. (PR #266) + fixed - If the original image is smaller than the specified max width and height, the extension does not enlarge it or resize it. (Issue #337, PR #338) ## Version 0.1.8 @@ -15,6 +21,7 @@ fixed - Resized images now maintain the same orientation as the original image. ## Version 0.1.7 fixed - Resized images now render in the Firebase console. (Issue #140) + fixed - The Sharp cache is now cleared so that the latest image with a given file name is retrieved from the Storage bucket. (Issue #286) @@ -29,7 +36,9 @@ fixed - The original, uploaded image's MIME type must now always be specified in ## Version 0.1.4 fixed - Fixed bug where name of resized file was missing original name if there was no file extension. (issue #20) + fixed - Fixed "TypeError: Cannot set property 'resizedImage' of undefined". (issue #130) + fixed - Fixed bug where some valid bucket names were rejected during configuration. (issue #27) ## Version 0.1.3 diff --git a/storage-resize-images/extension.yaml b/storage-resize-images/extension.yaml index ed4844646..b4fc97240 100644 --- a/storage-resize-images/extension.yaml +++ b/storage-resize-images/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: storage-resize-images -version: 0.1.10 +version: 0.1.11 specVersion: v1beta displayName: Resize Images