Marcwo/dataform module

modules/README.md

@@ -85,6 +85,7 @@ These modules are used in the examples included in this repository. If you are u
 - [Dataproc](./dataproc)
 - [GCS](./gcs)
 - [Pub/Sub](./pubsub)
+- [Dataform Repository](./dataform-repository/)
 
 ## Development
 

modules/dataform-repository/README.md

@@ -0,0 +1,69 @@
+# Google Cloud Dataform Repository module
+
+This module allows managing a dataform repository, allows adding IAM permissions. Also enables attaching a remote repository.
+
+## TODO
+[] Add validation rules to variable.
+
+## Examples
+
+### Simple dataform repository with access configration
+
+Simple dataform repository and specifying repository access via the IAM variable.
+
+```hcl
+module "dataform" {
+  source     = "./fabric/modules/dataform-repository"
+  project_id = "my-project"
+  repository = {
+    my-repository = {
+      name = "myrepository"
+      iam = {
+        "roles/dataform.editor" = ["user:user1@example.org"]
+      }
+    }
+  }
+}
+# tftest modules=1 resources=2
+```
+
+### Repository with an attached remote repository
+
+This creates a dataform repository with a remote repository attached to it. In order to enable dataform to communicate with a 3P GIT provider, an access token must be generated and stored as a secret on GCP. For that, we utilize the existing [secret-manager module](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/modules/secret-manager).
+
+```hcl
+module "secret" {
+  source     = "./fabric/modules/secret-manager"
+  project_id = "my-project"
+  secrets = {
+    my-secret = {
+    }
+  }
+  versions = {
+    my-secret = {
+      v1 = { enabled = true, data = "MYTOKEN" }
+    }
+  }
+}
+
+module "dataform" {
+  source     = "./fabric/modules/dataform-repository"
+  project_id = "my-project"
+  repository = {
+    my-repository = {
+      name           = "myrepository"
+      remote_url     = "https://myremoteurl"
+      secret_version = module.secret.version_ids["my-secret:v1"]
+    }
+  }
+}
+# tftest modules=1 resources=3
+```
+<!-- BEGIN TFDOC -->
+## Variables
+
+| name | description | type | required | default |
+|---|---|:---:|:---:|:---:|
+| [project_id](variables.tf#L17) | Id of the project where resources will be created. | <code>string</code> | ✓ |  |
+| [repository](variables.tf#L22) | Map of repositories to manage, including setting IAM permissions. | <code title="map&#40;object&#40;&#123;&#10;  name            &#61; string&#10;  branch          &#61; optional&#40;string, &#34;main&#34;&#41;&#10;  remote_url      &#61; optional&#40;string, null&#41;&#10;  secret_name     &#61; optional&#40;string, null&#41;&#10;  secret_version  &#61; optional&#40;string, &#34;v1&#34;&#41;&#10;  token           &#61; optional&#40;string, null&#41;&#10;  service_account &#61; optional&#40;string, null&#41;&#10;  region          &#61; optional&#40;string, null&#41;&#10;  iam             &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  iam_bindings &#61; optional&#40;map&#40;object&#40;&#123;&#10;    members &#61; list&#40;string&#41;&#10;    condition &#61; optional&#40;object&#40;&#123;&#10;      expression  &#61; string&#10;      title       &#61; string&#10;      description &#61; optional&#40;string&#41;&#10;    &#125;&#41;&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;  iam_bindings_additive &#61; optional&#40;map&#40;object&#40;&#123;&#10;    member &#61; string&#10;    role   &#61; string&#10;    condition &#61; optional&#40;object&#40;&#123;&#10;      expression  &#61; string&#10;      title       &#61; string&#10;      description &#61; optional&#40;string&#41;&#10;    &#125;&#41;&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | ✓ |  |
+<!-- END TFDOC -->

modules/dataform-repository/iam.tf

@@ -0,0 +1,95 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  repository_iam = flatten([
+    for k, v in var.repository : [
+      for role, members in v.iam : {
+        key     = k
+        role    = role
+        members = members
+      }
+    ]
+  ])
+  repository_iam_bindings = merge([
+    for k, v in var.repository : {
+      for binding_key, data in v.iam_bindings :
+      binding_key => {
+        repository = k
+        role       = data.role
+        members    = data.members
+        condition  = data.condition
+      }
+    }
+  ]...)
+  repository_iam_bindings_additive = merge([
+    for k, v in var.repository : {
+      for binding_key, data in v.iam_bindings_additive :
+      binding_key => {
+        repository = k
+        role       = data.role
+        member     = data.member
+        condition  = data.condition
+      }
+    }
+  ]...)
+}
+
+resource "google_dataform_repository_iam_binding" "authoritative" {
+  provider = google-beta
+  for_each = {
+    for binding in local.repository_iam :
+    "${binding.key}.${binding.role}" => binding
+  }
+  role       = each.value.role
+  members    = each.value.members
+  repository = google_dataform_repository.default[each.value.key].name
+
+}
+
+resource "google_dataform_repository_iam_binding" "bindings" {
+  provider   = google-beta
+  for_each   = local.repository_iam_bindings
+  role       = each.value.role
+  members    = each.value.members
+  repository = google_dataform_repository.default[each.value.key].name
+
+  dynamic "condition" {
+    for_each = each.value.condition == null ? [] : [""]
+    content {
+      expression  = each.value.condition.expression
+      title       = each.value.condition.title
+      description = each.value.condition.description
+    }
+  }
+}
+
+resource "google_dataform_repository_iam_member" "members" {
+  provider   = google-beta
+  for_each   = local.repository_iam_bindings_additive
+  role       = each.value.role
+  member     = each.value.member
+  repository = google_dataform_repository.default[each.value.key].name
+
+  dynamic "condition" {
+    for_each = each.value.condition == null ? [] : [""]
+    content {
+      expression  = each.value.condition.expression
+      title       = each.value.condition.title
+      description = each.value.condition.description
+    }
+  }
+}

modules/dataform-repository/main.tf

@@ -0,0 +1,33 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "google_dataform_repository" "default" {
+  provider        = google-beta
+  for_each        = var.repository
+  project         = var.project_id
+  name            = each.value.name
+  region          = each.value.region
+  service_account = each.value.service_account
+
+  dynamic "git_remote_settings" {
+    for_each = each.value.remote_url != null ? [1] : []
+    content {
+      url                                 = each.value.remote_url
+      default_branch                      = each.value.branch
+      authentication_token_secret_version = each.value.secret_version
+    }
+  }
+}

modules/dataform-repository/variables.tf

@@ -0,0 +1,54 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "Id of the project where resources will be created."
+  type        = string
+}
+
+variable "repository" {
+  description = "Map of repositories to manage, including setting IAM permissions."
+  type = map(object({
+    name            = string
+    branch          = optional(string, "main")
+    remote_url      = optional(string, null)
+    secret_name     = optional(string, null)
+    secret_version  = optional(string, "v1")
+    token           = optional(string, null)
+    service_account = optional(string, null)
+    region          = optional(string, null)
+    iam             = optional(map(list(string)), {})
+    iam_bindings = optional(map(object({
+      members = list(string)
+      condition = optional(object({
+        expression  = string
+        title       = string
+        description = optional(string)
+      }))
+    })), {})
+    iam_bindings_additive = optional(map(object({
+      member = string
+      role   = string
+      condition = optional(object({
+        expression  = string
+        title       = string
+        description = optional(string)
+      }))
+    })), {})
+  }))
+  nullable = false
+}
+

modules/dataform-repository/versions.tf

@@ -0,0 +1,27 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+terraform {
+  required_version = ">= 1.5.1"
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 5.11.0, < 6.0.0" # tftest
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = ">= 5.11.0, < 6.0.0" # tftest
+    }
+  }
+}