Adds support for BigQuery

README.md

@@ -22,6 +22,7 @@ For a comparison between dbmate and other popular database schema migration tool
     - [MySQL](#mysql)
     - [SQLite](#sqlite)
     - [ClickHouse](#clickhouse)
+    - [BigQuery](#bigquery)
   - [Creating Migrations](#creating-migrations)
   - [Running Migrations](#running-migrations)
   - [Rolling Back Migrations](#rolling-back-migrations)
@@ -287,6 +288,28 @@ DATABASE_URL="clickhouse://username:password@127.0.0.1:9000/database_name?on_clu
 
 [See other supported connection options](https://github.com/ClickHouse/clickhouse-go#dsn).
 
+#### BigQuery
+Follow the following format for `DATABASE_URL` when connecting to actual BigQuery in GCP:
+
+```
+bigquery://projectid/location/dataset
+```
+`projectid` (mandatory) - Project ID
+
+`dataset` (mandatory) - Dataset name within the Project
+
+`location` (optional) - Where Dataset is created
+
+*NOTE: Follow [this doc](https://cloud.google.com/docs/authentication/provide-credentials-adc) on how to set `GOOGLE_APPLICATION_CREDENTIALS` environment variable for proper Authentication*
+
+Follow the following format if trying to connect to a custom endpoint e.g. [BigQuery Emulator](https://github.com/goccy/bigquery-emulator)
+
+```
+bigquery://host:port/projectid/location/dataset?disable_auth=true
+```
+
+`disable_auth` (optional) - Pass `true` to skip Authentication, use only for testing and connecting to emulator.
+
 ### Creating Migrations
 
 To create a new migration, run `dbmate new create_users_table`. You can name the migration anything you like. This will create a file `db/migrations/20151127184807_create_users_table.sql` in the current directory:

docker-compose.yml

@@ -12,13 +12,15 @@ services:
       - clickhouse
       - clickhouse-cluster-01
       - clickhouse-cluster-02
+      - bigquery
     environment:
       CLICKHOUSE_TEST_URL: clickhouse://clickhouse:9000/dbmate_test
       CLICKHOUSE_CLUSTER_01_TEST_URL: clickhouse://ch-cluster-01:9000/dbmate_test
       CLICKHOUSE_CLUSTER_02_TEST_URL: clickhouse://ch-cluster-02:9000/dbmate_test
       MYSQL_TEST_URL: mysql://root:root@mysql/dbmate_test
       POSTGRES_TEST_URL: postgres://postgres:postgres@postgres/dbmate_test?sslmode=disable
       SQLITE_TEST_URL: sqlite3:/tmp/dbmate_test.sqlite3
+      BIGQUERY_TEST_URL: bigquery://test/us-east5/dbmate_test?disable_auth=true&endpoint=http%3A%2F%2Fbigquery%3A9050
 
   dbmate:
     build:
@@ -63,3 +65,8 @@ services:
       - zookeeper
     volumes:
       - ./pkg/driver/clickhouse/testdata/cluster_config/ch-cluster-02:/etc/clickhouse-server
+
+  bigquery:
+    image: ghcr.io/goccy/bigquery-emulator:0.4.4
+    command: |
+      --project=test --dataset=dbmate_test

fixtures/bigquery/.gitignore

@@ -0,0 +1 @@
+credentials.json

fixtures/bigquery/README.md

@@ -0,0 +1,49 @@
+# Google BigQuery Test Fixtures
+
+## Creating a service account for testing
+
+From the `dbmate` top-level directory:
+
+```sh
+$ PROJECT_ID=your-google-cloud-project-id
+$ LOCATION=us-east5
+$ DATASET=test_dataset
+$ SERVICE_ACCOUNT=dbmate-test-sa
+
+$ gcloud auth login
+
+$ gcloud iam service-accounts create $SERVICE_ACCOUNT
+
+$ gcloud projects add-iam-policy-binding $PROJECT_ID \
+  --role="roles/bigquery.dataEditor" \
+  --member=serviceAccount:${SERVICE_ACCOUNT}@${PROJECT_ID}.iam.gserviceaccount.com
+
+$ gcloud projects add-iam-policy-binding $PROJECT_ID \
+  --role="roles/bigquery.jobUser" \
+  --member=serviceAccount:${SERVICE_ACCOUNT}@${PROJECT_ID}.iam.gserviceaccount.com
+
+$ gcloud iam service-accounts keys create \
+  fixtures/bigquery/credentials.json \
+  --iam-account=${SERVICE_ACCOUNT}@${PROJECT_ID}.iam.gserviceaccount.com
+
+## WARNING: Only do this on a private machine, as anyone else with
+## access to the system will also be able to read the credentials file's
+## contents once it's made world-readable and use it to access Google as
+## the service account.  This is necessary for dbmate running as root
+## inside the Docker container to be able to read the file, though.
+
+$ chmod a+r fixtures/bigquery/credentials.json
+
+$ docker compose run --rm dev
+
+## The rest of these commands should be executed from inside the Docker
+## container:
+
+$ make build
+
+$ make test \
+  GOOGLE_APPLICATION_CREDENTIALS=/src/fixtures/bigquery/credentials.json \
+  GOOGLE_BIGQUERY_TEST_URL=bigquery://$PROJECT_ID/$LOCATION/$DATASET \
+  FLAGS+="-count 1 -v ./pkg/driver/bigquery #"
+
+```

fixtures/bigquery/migrations/20151129054053_test_migration.sql

@@ -0,0 +1,9 @@
+-- migrate:up
+create table users (
+  id int64,
+  name string
+);
+insert into users (id, name) values (1, 'alice');
+
+-- migrate:down
+drop table users;

fixtures/bigquery/migrations/20200227231541_test_posts.sql

@@ -0,0 +1,8 @@
+-- migrate:up
+create table posts (
+  id int64,
+  name string
+);
+
+-- migrate:down
+drop table posts;

go.mod

@@ -16,25 +16,67 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.110.4 // indirect
+	cloud.google.com/go/bigquery v1.52.0 // indirect
+	cloud.google.com/go/compute v1.21.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/iam v1.1.1 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/ClickHouse/ch-go v0.61.5 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/apache/arrow/go/v12 v12.0.0 // indirect
+	github.com/apache/thrift v0.16.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
 	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/goccy/go-json v0.9.11 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/flatbuffers v2.0.8+incompatible // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/s2a-go v0.1.4 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
+	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/klauspost/asmfmt v1.3.2 // indirect
 	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
+	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/paulmach/orb v0.11.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel v1.24.0 // indirect
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/oauth2 v0.10.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/api v0.126.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/grpc v1.58.3 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gorm.io/driver/bigquery v1.2.0 // indirect
+	gorm.io/gorm v1.24.0 // indirect
 )