extension.yaml

# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: firestore-bigquery-export
version: 0.1.56
specVersion: v1beta

displayName: Stream Firestore to BigQuery
description:
  Sends realtime, incremental updates from a specified Cloud Firestore
  collection to BigQuery.

license: Apache-2.0

sourceUrl: https://github.com/firebase/extensions/tree/master/firestore-bigquery-export
releaseNotesUrl: https://github.com/firebase/extensions/blob/master/firestore-bigquery-export/CHANGELOG.md

author:
  authorName: Firebase
  url: https://firebase.google.com

contributors:
  - authorName: Jan Wyszynski
    email: wyszynski@google.com
    url: https://github.com/IanWyszynski

billingRequired: true

apis:
  - apiName: bigquery.googleapis.com
    reason: Mirrors data from your Cloud Firestore collection in BigQuery.

roles:
  - role: bigquery.dataEditor
    reason: Allows the extension to configure and export data into BigQuery.

  - role: datastore.user
    reason: Allows the extension to write updates to the database.

resources:
  - name: fsexportbigquery
    type: firebaseextensions.v1beta.function
    description:
      Listens for document changes in your specified Cloud Firestore collection,
      then exports the changes into BigQuery.
    properties:
      runtime: nodejs18
      eventTrigger:
        eventType: providers/cloud.firestore/eventTypes/document.write
        resource: projects/${param:PROJECT_ID}/databases/(default)/documents/${param:COLLECTION_PATH}/{documentId}

  - name: syncBigQuery
    type: firebaseextensions.v1beta.function
    description: >-
      A task-triggered function that gets called on BigQuery sync
    properties:
      runtime: nodejs18
      taskQueueTrigger:
        rateLimits:
          maxConcurrentDispatches: 500
          maxDispatchesPerSecond: ${param:MAX_DISPATCHES_PER_SECOND}
        retryConfig:
          maxAttempts: 5
          minBackoffSeconds: 60

  - name: initBigQuerySync
    type: firebaseextensions.v1beta.function
    description: >-
      Runs configuration for sycning with BigQuery
    properties:
      runtime: nodejs18
      taskQueueTrigger:
        retryConfig:
          maxAttempts: 15
          minBackoffSeconds: 60

  - name: setupBigQuerySync
    type: firebaseextensions.v1beta.function
    description: >-
      Runs configuration for sycning with BigQuery
    properties:
      runtime: nodejs18
      taskQueueTrigger:
        retryConfig:
          maxAttempts: 15
          minBackoffSeconds: 60

params:
  - param: DATASET_LOCATION
    label: BigQuery Dataset location
    description: >-
      Where do you want to deploy the BigQuery dataset created for this
      extension? For help selecting a location, refer to the [location selection
      guide](https://cloud.google.com/bigquery/docs/locations).
    type: select
    options:
      - label: Iowa (us-central1)
        value: us-central1
      - label: Las Vegas (us-west4)
        value: us-west4
      - label: Warsaw (europe-central2)
        value: europe-central2
      - label: Los Angeles (us-west2)
        value: us-west2
      - label: Montreal (northamerica-northeast1)
        value: northamerica-northeast1
      - label: Northern Virginia (us-east4)
        value: us-east4
      - label: Oregon (us-west1)
        value: us-west1
      - label: Salt Lake City (us-west3)
        value: us-west3
      - label: Sao Paulo (southamerica-east1)
        value: southamerica-east1
      - label: South Carolina (us-east1)
        value: us-east1
      - label: Belgium (europe-west1)
        value: europe-west1
      - label: Finland (europe-north1)
        value: europe-north1
      - label: Frankfurt (europe-west3)
        value: europe-west3
      - label: London (europe-west2)
        value: europe-west2
      - label: Netherlands (europe-west4)
        value: europe-west4
      - label: Zurich (europe-west6)
        value: europe-west6
      - label: Taiwan (asia-east1)
        value: asia-east1
      - label: Hong Kong (asia-east2)
        value: asia-east2
      - label: Jakarta (asia-southeast2)
        value: asia-southeast2
      - label: Mumbai (asia-south1)
        value: asia-south1
      - label: Singapore (asia-southeast1)
        value: asia-southeast1
      - label: Osaka (asia-northeast2)
        value: asia-northeast2
      - label: Seoul (asia-northeast3)
        value: asia-northeast3
      - label: Singapore (asia-southeast1)
        value: asia-southeast1
      - label: Sydney (australia-southeast1)
        value: australia-southeast1
      - label: Taiwan (asia-east1)
        value: asia-east1
      - label: Tokyo (asia-northeast1)
        value: asia-northeast1
      - label: United States (multi-regional)
        value: us
      - label: Europe (multi-regional)
        value: eu
    default: us
    required: true
    immutable: true

  - param: BIGQUERY_PROJECT_ID
    label: BigQuery Project ID
    description: >-
      Override the default project for BigQuery instance. This can allow updates
      to be directed to to a BigQuery instance on another GCP project.
    type: string
    default: ${PROJECT_ID}
    required: true

  - param: COLLECTION_PATH
    label: Collection path
    description: >-
      What is the path of the collection that you would like to export? You may
      use `{wildcard}` notation to match a subcollection of all documents in a
      collection (for example: `chatrooms/{chatid}/posts`). Parent Firestore
      Document IDs from `{wildcards}`  can be returned in `path_params` as a
      JSON formatted string.
    type: string
    example: posts
    validationRegex: "^[^/]+(/[^/]+/[^/]+)*$"
    validationErrorMessage:
      Firestore collection paths must be an odd number of segments separated by
      slashes, e.g. "path/to/collection".
    default: posts
    required: true

  - param: WILDCARD_IDS
    label: Enable Wildcard Column field with Parent Firestore Document IDs
    description: >-
      If enabled, creates a column containing a JSON object of all wildcard ids
      from a documents path.
    type: select
    options:
      - label: No
        value: false
      - label: Yes
        value: true
    default: false
    required: false

  - param: DATASET_ID
    label: Dataset ID
    description: >-
      What ID would you like to use for your BigQuery dataset? This extension
      will create the dataset, if it doesn't already exist.
    type: string
    example: firestore_export
    validationRegex: "^[a-zA-Z0-9_]+$"
    validationErrorMessage: >
      BigQuery dataset IDs must be alphanumeric (plus underscores) and must be
      no more than 1024 characters.
    default: firestore_export
    required: true

  - param: TABLE_ID
    label: Table ID
    description: >-
      What identifying prefix would you like to use for your table and view
      inside your BigQuery dataset? This extension will create the table and
      view, if they don't already exist.
    type: string
    example: posts
    validationRegex: "^[a-zA-Z0-9_]+$"
    validationErrorMessage: >
      BigQuery table IDs must be alphanumeric (plus underscores) and must be no
      more than 1024 characters.
    default: posts
    required: true

  - param: TABLE_PARTITIONING
    label: BigQuery SQL table Time Partitioning option type
    description: >-
      This parameter will allow you to partition the BigQuery table and BigQuery
      view  created by the extension based on data ingestion time. You may
      select the granularity of partitioning based upon one of: HOUR, DAY,
      MONTH, YEAR. This will generate one partition per day, hour, month or
      year, respectively.
    type: select
    options:
      - label: hour
        value: HOUR
      - label: day
        value: DAY
      - label: month
        value: MONTH
      - label: year
        value: YEAR
      - label: none
        value: NONE
    default: NONE
    required: false

  - param: TIME_PARTITIONING_FIELD
    label: BigQuery Time Partitioning column name
    description: >-
      BigQuery table column/schema field name for TimePartitioning. You can
      choose schema available as `timestamp` OR a new custom defined column that
      will be assigned to the selected Firestore Document field below. Defaults
      to pseudo column _PARTITIONTIME if unspecified. Cannot be changed if Table
      is already partitioned.
    type: string
    required: false

  - param: TIME_PARTITIONING_FIRESTORE_FIELD
    label:
      Firestore Document field name for BigQuery SQL Time Partitioning field
      option
    description: >-
      This parameter will allow you to partition the BigQuery table  created by
      the extension based on selected. The Firestore Document field value must
      be a top-level TIMESTAMP, DATETIME, DATE field BigQuery string format or
      Firestore timestamp(will be converted to BigQuery TIMESTAMP). Cannot be
      changed if Table is already partitioned.
       example: `postDate`(Ensure that the Firestore-BigQuery export extension
      creates the dataset and table before initiating any backfill scripts.
       This step is crucial for the partitioning to function correctly. It is
      essential for the script to insert data into an already partitioned
      table.)
    type: string
    required: false

  - param: TIME_PARTITIONING_FIELD_TYPE
    label: BigQuery SQL Time Partitioning table schema field(column) type
    description: >-
      Parameter for BigQuery SQL schema field type for the selected Time
      Partitioning Firestore Document field option. Cannot be changed if Table
      is already partitioned.
    type: select
    options:
      - label: TIMESTAMP
        value: TIMESTAMP
      - label: DATETIME
        value: DATETIME
      - label: DATE
        value: DATE
      - label: omit
        value: omit
    default: omit
    required: false

  - param: CLUSTERING
    label: BigQuery SQL table clustering
    description: >-
      This parameter will allow you to set up Clustering for the BigQuery Table
      created by the extension. (for example: `data,document_id,timestamp`- no
      whitespaces). You can select up to 4 comma separated fields. The order of
      the specified columns determines the sort order of the data. Available
      schema extensions table fields for clustering: `document_id,
      document_name, timestamp, event_id, operation, data`.
    type: string
    validationRegex: ^[^,\s]+(?:,[^,\s]+){0,3}$
    validationErrorMessage:
      No whitespaces. Max 4 fields. e.g. `data,timestamp,event_id,operation`
    example: data,document_id,timestamp
    required: false

  - param: MAX_DISPATCHES_PER_SECOND
    label: Maximum number of synced documents per second
    description: >-
      This parameter will set the maximum number of syncronised documents per
      second with BQ. Please note, any other external updates to a Big Query
      table will be included within this quota. Ensure that you have a set a low
      enough number to compensate. Defaults to 10.
    type: string
    validationRegex: ^([1-9]|[1-9][0-9]|[1-4][0-9]{2}|500)$
    validationErrorMessage: Please select a number between 1 and 500
    default: 100
    required: false

  - param: BACKUP_COLLECTION
    label: Backup Collection Name
    description: >-
      This (optional) parameter will allow you to specify a collection for which
      failed BigQuery updates will be written to.
    type: string

  - param: TRANSFORM_FUNCTION
    label: Transform function URL
    description: >-
      Specify a function URL to call that will transform the payload that will
      be written to BigQuery. See the pre-install documentation for more
      details.
    example: https://us-west1-my-project-id.cloudfunctions.net/myTransformFunction
    type: string
    required: false

  - param: USE_NEW_SNAPSHOT_QUERY_SYNTAX
    label: Use new query syntax for snapshots
    description: >-
      If enabled, snapshots will be generated with the new query syntax, which
      should be more performant, and avoid potential resource limitations.
    type: select
    options:
      - label: Yes
        value: yes
      - label: No
        value: no
    default: no
    required: true

  - param: EXCLUDE_OLD_DATA
    label: Exclude old data payloads
    description: >-
      If enabled, table rows will never contain old data (document snapshot
      before the Firestore onDocumentUpdate event: `change.before.data()`). The
      reduction in data should be more performant, and avoid potential resource
      limitations.
    type: select
    required: false
    default: no
    options:
      - label: Yes
        value: yes
      - label: No
        value: no

  - param: KMS_KEY_NAME
    label: Cloud KMS key name
    description: >-
      Instead of Google managing the key encryption keys that protect your data,
      you control and manage key encryption keys in Cloud KMS. If this parameter
      is set, the extension will specify the KMS key name when creating the BQ
      table. See the PREINSTALL.md for more details.
    type: string
    validationRegex: "projects/([^/]+)/locations/([^/]+)/keyRings/([^/]+)/cryptoKeys/([^/]+)"
    validationErrorMessage:
      The key name must be of the format
      'projects/PROJECT_NAME/locations/KEY_RING_LOCATION/keyRings/KEY_RING_ID/cryptoKeys/KEY_ID'.
    required: false

  - param: MAX_ENQUEUE_ATTEMPTS
    label: Maximum number of enqueue attempts
    description: >-
      This parameter will set the maximum number of attempts to enqueue a
      document to cloud tasks for export to BigQuery.
    type: string
    validationRegex: ^(10|[1-9])$
    validationErrorMessage: Please select an integer between 1 and 10
    default: 3

events:
  - type: firebase.extensions.firestore-counter.v1.onStart
    description:
      Occurs when a trigger has been called within the Extension, and will
      include data such as the context of the trigger request.

  - type: firebase.extensions.firestore-counter.v1.onSuccess
    description:
      Occurs when image resizing completes successfully. The event will contain
      further details about specific formats and sizes.

  - type: firebase.extensions.firestore-counter.v1.onError
    description:
      Occurs when an issue has been experienced in the Extension. This will
      include any error data that has been included within the Error Exception.

  - type: firebase.extensions.firestore-counter.v1.onCompletion
    description:
      Occurs when the function is settled. Provides no customized data other
      than the context.
  - type: firebase.extensions.big-query-export.v1.sync.start
    description: Occurs on a firestore document write event.

lifecycleEvents:
  onInstall:
    function: initBigQuerySync
    processingMessage: Configuring BigQuery Sync.
  onUpdate:
    function: setupBigQuerySync
    processingMessage: Configuring BigQuery Sync
  onConfigure:
    function: setupBigQuerySync
    processingMessage: Configuring BigQuery Sync