Skip to content

Commit

Permalink
Add support for google_datastream_stream to TGC (#10703)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dfdossantos
dfdossantos committed May 16, 2024
1 parent caee19a commit ed21548
Show file tree
Hide file tree
Showing 3 changed files with 276 additions and 1 deletion.
3 changes: 2 additions & 1 deletion mmv1/templates/tgc/resource_converters.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ func ResourceConverters() map[string][]cai.ResourceConverter {
"google_bigquery_dataset_iam_binding": {bigquery.ResourceConverterBigqueryDatasetIamBinding()},
"google_bigquery_dataset_iam_member": {bigquery.ResourceConverterBigqueryDatasetIamMember()},
"google_bigquery_table": {resourceConverterBigQueryTable()},
"google_datastream_connection_profile": {datastream.ResourceConverterDatastreamConnectionProfile()},
"google_datastream_connection_profile": {datastream.ResourceConverterDatastreamConnectionProfile()},
"google_datastream_stream": {datastream.ResourceConverterDatastreamStream()},
"google_org_policy_policy": {resourceConverterOrgPolicyPolicy()},
"google_redis_instance": {redis.ResourceConverterRedisInstance()},
"google_spanner_database": {spanner.ResourceConverterSpannerDatabase()},
Expand Down
156 changes: 156 additions & 0 deletions mmv1/third_party/tgc/tests/data/example_google_datastream_stream.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
[
{
"name": "//bigquery.googleapis.com/projects/{{.Provider.project}}/datasets/stpostgres",
"asset_type": "bigquery.googleapis.com/Dataset",
"resource": {
"version": "v2",
"discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/bigquery/v2/rest",
"discovery_name": "Dataset",
"parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}",
"data": {
"datasetReference": {
"datasetId": "stpostgres"
},
"description": "Database of postgres",
"friendlyName": "stpostgres",
"location": "us-central1"
}
},
"ancestors": ["organizations/{{.OrgID}}"],
"ancestry_path": "{{.Ancestry}}/project/{{.Provider.project}}"
},
{
"name": "//cloudsql.googleapis.com/projects/{{.Provider.project}}/instances/instance-stream-name",
"asset_type": "sqladmin.googleapis.com/Instance",
"resource": {
"version": "v1beta4",
"discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/sqladmin/v1beta4/rest",
"discovery_name": "DatabaseInstance",
"parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}",
"data": {
"databaseVersion": "MYSQL_8_0",
"name": "instance-stream-name",
"project": "{{.Provider.project}}",
"region": "us-central1",
"settings": {
"activationPolicy": "ALWAYS",
"availabilityType": "ZONAL",
"backupConfiguration": {
"binaryLogEnabled": true,
"enabled": true
},
"dataDiskType": "PD_SSD",
"ipConfiguration": {
"authorizedNetworks": [
{
"value": "34.72.28.29"
},
{
"value": "34.71.242.81"
},
{
"value": "34.67.234.134"
},
{
"value": "34.67.6.157"
},
{
"value": "34.72.239.218"
}
],
"ipv4Enabled": true,
"requireSsl": false
},
"pricingPlan": "PER_USE",
"storageAutoResize": true,
"tier": "db-f1-micro"
}
}
},
"ancestors": ["organizations/{{.OrgID}}"],
"ancestry_path": "{{.Ancestry}}/project/{{.Provider.project}}"
},
{
"name": "//datastream.googleapis.com/projects/{{.Provider.project}}/locations/us-central1/connectionProfiles/dest-st-profile",
"asset_type": "datastream.googleapis.com/ConnectionProfile",
"resource": {
"version": "v1",
"discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/datastream/v1/rest",
"discovery_name": "ConnectionProfile",
"parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}",
"data": {
"bigqueryProfile": {},
"displayName": "Connection profile"
}
},
"ancestors": ["organizations/{{.OrgID}}"],
"ancestry_path": "{{.Ancestry}}/project/{{.Provider.project}}"
},
{
"name": "//datastream.googleapis.com/projects/{{.Provider.project}}/locations/us-central1/connectionProfiles/source-profile-st",
"asset_type": "datastream.googleapis.com/ConnectionProfile",
"resource": {
"version": "v1",
"discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/datastream/v1/rest",
"discovery_name": "ConnectionProfile",
"parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}",
"data": {
"bigqueryProfile": null,
"displayName": "Source connection profile",
"mysqlProfile": {
"port": 3306,
"username": "my-user"
}
}
},
"ancestors": ["organizations/{{.OrgID}}"],
"ancestry_path": "{{.Ancestry}}/project/{{.Provider.project}}"
},
{
"name": "//datastream.googleapis.com/projects/{{.Provider.project}}/locations/us-central1/streams/postgres-bigquery",
"asset_type": "datastream.googleapis.com/Stream",
"resource": {
"version": "v1",
"discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/datastream/v1/rest",
"discovery_name": "Stream",
"parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}",
"data": {
"backfillAll": {},
"backfillNone": null,
"destinationConfig": {
"bigqueryDestinationConfig": {
"dataFreshness": "900s"
}
},
"displayName": "postgres to bigQuery",
"sourceConfig": {
"mysqlSourceConfig": {
"maxConcurrentBackfillTasks": 0,
"maxConcurrentCdcTasks": 0
},
"oracleSourceConfig": null,
"postgresqlSourceConfig": null,
"sqlServerSourceConfig": null
}
}
},
"ancestors": ["organizations/{{.OrgID}}"],
"ancestry_path": "{{.Ancestry}}/project/{{.Provider.project}}"
},
{
"name": "//sqladmin.googleapis.com/projects/{{.Provider.project}}/instances/instance-stream-name/databases/db",
"asset_type": "sqladmin.googleapis.com/Database",
"resource": {
"version": "v1beta4",
"discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/sqladmin/v1beta4/rest",
"discovery_name": "Database",
"parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}",
"data": {
"instance": "instance-stream-name",
"name": "db"
}
},
"ancestors": ["organizations/{{.OrgID}}"],
"ancestry_path": "{{.Ancestry}}/project/{{.Provider.project}}"
}
]
118 changes: 118 additions & 0 deletions mmv1/third_party/tgc/tests/data/example_google_datastream_stream.tf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
terraform {
required_providers {
google = {
source = "hashicorp/google-beta"
version = "~> {{.Provider.version}}"
}
}
}

provider "google" {
{{if .Provider.credentials }}credentials = "{{.Provider.credentials}}"{{end}}
}

resource "google_bigquery_dataset" "postgres" {
dataset_id = "stpostgres"
friendly_name = "stpostgres"
description = "Database of postgres"
location = "us-central1"
}

resource "google_datastream_stream" "default" {
display_name = "postgres to bigQuery"
location = "us-central1"
stream_id = "postgres-bigquery"

source_config {
source_connection_profile = google_datastream_connection_profile.source_connection_profile.id
mysql_source_config {}
}

destination_config {
destination_connection_profile = google_datastream_connection_profile.destination_connection_profile2.id
bigquery_destination_config {
data_freshness = "900s"
single_target_dataset {
dataset_id = google_bigquery_dataset.postgres.id
}
}
}

backfill_all {
}

}

resource "google_datastream_connection_profile" "destination_connection_profile2" {
display_name = "Connection profile"
location = "us-central1"
connection_profile_id = "dest-st-profile"
bigquery_profile {}
}

resource "google_sql_database_instance" "instance" {
name = "instance-stream-name"
database_version = "MYSQL_8_0"
region = "us-central1"
settings {
tier = "db-f1-micro"
backup_configuration {
enabled = true
binary_log_enabled = true
}

ip_configuration {
// Datastream IPs will vary by region.
authorized_networks {
value = "34.71.242.81"
}

authorized_networks {
value = "34.72.28.29"
}

authorized_networks {
value = "34.67.6.157"
}

authorized_networks {
value = "34.67.234.134"
}

authorized_networks {
value = "34.72.239.218"
}
}
}

deletion_protection = false
}

resource "google_sql_database" "db" {
instance = google_sql_database_instance.instance.name
name = "db"
}

resource "random_password" "pwd" {
length = 16
special = false
}

resource "google_sql_user" "user" {
name = "my-user"
instance = google_sql_database_instance.instance.name
host = "%"
password = random_password.pwd.result
}

resource "google_datastream_connection_profile" "source_connection_profile" {
display_name = "Source connection profile"
location = "us-central1"
connection_profile_id = "source-profile-st"

mysql_profile {
hostname = google_sql_database_instance.instance.public_ip_address
username = google_sql_user.user.name
password = google_sql_user.user.password
}
}

0 comments on commit ed21548

Please sign in to comment.