Update and refactor artifact registry module

blueprints/data-solutions/data-platform-foundations/03-orchestration.tf

@@ -166,9 +166,8 @@ module "orch-nat" {
 module "orch-artifact-reg" {
   source      = "../../../modules/artifact-registry"
   project_id  = module.orch-project.project_id
-  id          = "${var.prefix}-app-images"
+  name        = "${var.prefix}-app-images"
   location    = var.region
-  format      = "DOCKER"
   description = "Docker repository storing application images e.g. Dataflow, Cloud Run etc..."
 }
 

blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py

@@ -283,7 +283,7 @@
           'datasetId': DWH_CURATED_BQ_DATASET,
           'tableId': 'customer_purchase'
         },
-        'writeDisposition':'WRITE_APPEND',
+        'writeDisposition':'WRITE_TRUNCATE',
         "useLegacySql": False
       }
     },
@@ -313,7 +313,7 @@
           'datasetId': DWH_CONFIDENTIAL_BQ_DATASET,
           'tableId': 'customer_purchase'
         },
-        'writeDisposition':'WRITE_APPEND',
+        'writeDisposition':'WRITE_TRUNCATE',
         "useLegacySql": False
       }
     },

blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags_flex.py

@@ -414,7 +414,7 @@
                     'tableId': 'customer_purchase'
                 },
                 'writeDisposition':
-                'WRITE_APPEND',
+                'WRITE_TRUNCATE',
                 "useLegacySql":
                 False
             }
@@ -449,7 +449,7 @@
                     'tableId': 'customer_purchase'
                 },
                 'writeDisposition':
-                'WRITE_APPEND',
+                'WRITE_TRUNCATE',
                 "useLegacySql":
                 False
             }

blueprints/data-solutions/vertex-mlops/ci-cd.tf

@@ -40,10 +40,9 @@ resource "google_iam_workload_identity_pool_provider" "github_provider" {
 
 module "artifact_registry" {
   source     = "../../../modules/artifact-registry"
-  id         = "docker-repo"
+  name       = "docker-repo"
   project_id = module.project.project_id
   location   = var.region
-  format     = "DOCKER"
 }
 
 module "service-account-github" {

blueprints/gke/autopilot/main.tf

@@ -58,8 +58,7 @@ module "docker_artifact_registry" {
   source     = "../../../modules/artifact-registry"
   project_id = module.project.project_id
   location   = var.region
-  format     = "DOCKER"
-  id         = "registry"
+  name       = "registry"
   iam = {
     "roles/artifactregistry.reader" = [module.node_sa.iam_email]
   }

blueprints/gke/binauthz/main.tf

@@ -172,8 +172,7 @@ module "docker_artifact_registry" {
   source     = "../../../modules/artifact-registry"
   project_id = module.project.project_id
   location   = var.region
-  format     = "DOCKER"
-  id         = "${var.prefix}-registry"
+  name       = "${var.prefix}-registry"
   iam = {
     "roles/artifactregistry.writer" = [module.image_cb_sa.iam_email]
     "roles/artifactregistry.reader" = [module.cluster_nodepool.service_account_iam_email]

modules/artifact-registry/README.md

@@ -2,37 +2,111 @@
 
 This module simplifies the creation of repositories using Google Cloud Artifact Registry.
 
-Note: Artifact Registry is still in beta, hence this module currently uses the beta provider.
-
-## Example
+## Standard Repository
 
 ```hcl
 module "docker_artifact_registry" {
   source     = "./fabric/modules/artifact-registry"
   project_id = "myproject"
   location   = "europe-west1"
-  format     = "DOCKER"
-  id         = "myregistry"
+  name       = "myregistry"
   iam = {
     "roles/artifactregistry.admin" = ["group:cicd@example.com"]
   }
 }
 # tftest modules=1 resources=2
 ```
+
+## Remote and Virtual Repositories
+
+```hcl
+
+module "registry-local" {
+  source     = "./fabric/modules/artifact-registry"
+  project_id = var.project_id
+  location   = "europe-west1"
+  name       = "local"
+  format     = { python = {} }
+}
+
+module "registry-remote" {
+  source     = "./fabric/modules/artifact-registry"
+  project_id = var.project_id
+  location   = "europe-west1"
+  name       = "remote"
+  format     = { python = {} }
+  mode       = { remote = true }
+}
+
+module "registry-virtual" {
+  source     = "./fabric/modules/artifact-registry"
+  project_id = var.project_id
+  location   = "europe-west1"
+  name       = "virtual"
+  format     = { python = {} }
+  mode = {
+    virtual = {
+      remote = {
+        repository = module.registry-remote.id
+        priority   = 1
+      }
+      local = {
+        repository = module.registry-local.id
+        priority   = 10
+      }
+    }
+  }
+}
+
+# tftest modules=3 resources=3
+```
+
+## Additional Docker and Maven Options
+
+```hcl
+
+module "registry-docker" {
+  source     = "./fabric/modules/artifact-registry"
+  project_id = var.project_id
+  location   = "europe-west1"
+  name       = "docker"
+  format = {
+    docker = {
+      immutable_tags = true
+    }
+  }
+}
+
+module "registry-maven" {
+  source     = "./fabric/modules/artifact-registry"
+  project_id = var.project_id
+  location   = "europe-west1"
+  name       = "maven"
+  format = {
+    maven = {
+      allow_snapshot_overwrites = true
+      version_policy            = "RELEASE"
+    }
+  }
+}
+
+# tftest modules=2 resources=2
+```
 <!-- BEGIN TFDOC -->
 
 ## Variables
 
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [id](variables.tf#L41) | Repository id. | <code>string</code> | ✓ |  |
-| [project_id](variables.tf#L58) | Registry project id. | <code>string</code> | ✓ |  |
+| [location](variables.tf#L68) | Registry location. Use `gcloud beta artifacts locations list' to get valid values. | <code>string</code> | ✓ |  |
+| [name](variables.tf#L93) | Registry name. | <code>string</code> | ✓ |  |
+| [project_id](variables.tf#L98) | Registry project id. | <code>string</code> | ✓ |  |
 | [description](variables.tf#L17) | An optional description for the repository. | <code>string</code> |  | <code>&#34;Terraform-managed registry&#34;</code> |
 | [encryption_key](variables.tf#L23) | The KMS key name to use for encryption at rest. | <code>string</code> |  | <code>null</code> |
-| [format](variables.tf#L29) | Repository format. One of DOCKER or UNSPECIFIED. | <code>string</code> |  | <code>&#34;DOCKER&#34;</code> |
-| [iam](variables.tf#L35) | IAM bindings in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [labels](variables.tf#L46) | Labels to be attached to the registry. | <code>map&#40;string&#41;</code> |  | <code>&#123;&#125;</code> |
-| [location](variables.tf#L52) | Registry location. Use `gcloud beta artifacts locations list' to get valid values. | <code>string</code> |  | <code>null</code> |
+| [format](variables.tf#L29) | Repository format. | <code title="object&#40;&#123;&#10;  apt &#61; optional&#40;object&#40;&#123;&#125;&#41;&#41;&#10;  docker &#61; optional&#40;object&#40;&#123;&#10;    immutable_tags &#61; optional&#40;bool&#41;&#10;  &#125;&#41;&#41;&#10;  kfp &#61; optional&#40;object&#40;&#123;&#125;&#41;&#41;&#10;  go  &#61; optional&#40;object&#40;&#123;&#125;&#41;&#41;&#10;  maven &#61; optional&#40;object&#40;&#123;&#10;    allow_snapshot_overwrites &#61; optional&#40;bool&#41;&#10;    version_policy            &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;  npm    &#61; optional&#40;object&#40;&#123;&#125;&#41;&#41;&#10;  python &#61; optional&#40;object&#40;&#123;&#125;&#41;&#41;&#10;  yum    &#61; optional&#40;object&#40;&#123;&#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123; docker &#61; &#123;&#125; &#125;</code> |
+| [iam](variables.tf#L56) | IAM bindings in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [labels](variables.tf#L62) | Labels to be attached to the registry. | <code>map&#40;string&#41;</code> |  | <code>&#123;&#125;</code> |
+| [mode](variables.tf#L73) | Repository mode. | <code title="object&#40;&#123;&#10;  standard &#61; optional&#40;bool&#41;&#10;  remote   &#61; optional&#40;bool&#41;&#10;  virtual &#61; optional&#40;map&#40;object&#40;&#123;&#10;    repository &#61; string&#10;    priority   &#61; number&#10;  &#125;&#41;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123; standard &#61; true &#125;</code> |
 
 ## Outputs
 

modules/artifact-registry/main.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,15 +14,89 @@
  * limitations under the License.
  */
 
+locals {
+  format_string = one([for k, v in var.format : k if v != null])
+  mode_string   = one([for k, v in var.mode : k if v != null && v != false])
+}
+
 resource "google_artifact_registry_repository" "registry" {
-  provider      = google-beta
   project       = var.project_id
   location      = var.location
   description   = var.description
-  format        = var.format
+  format        = upper(local.format_string)
   labels        = var.labels
-  repository_id = var.id
+  repository_id = var.name
+  mode          = "${upper(local.mode_string)}_REPOSITORY"
   kms_key_name  = var.encryption_key
+
+  dynamic "docker_config" {
+    for_each = local.format_string == "docker" ? [""] : []
+    content {
+      immutable_tags = var.format.docker.immutable_tags
+    }
+  }
+
+  dynamic "maven_config" {
+    for_each = local.format_string == "maven" ? [""] : []
+    content {
+      allow_snapshot_overwrites = var.format.maven.allow_snapshot_overwrites
+      version_policy            = var.format.maven.version_policy
+    }
+  }
+
+  dynamic "remote_repository_config" {
+    for_each = local.mode_string == "remote" ? [""] : []
+    content {
+      dynamic "docker_repository" {
+        for_each = local.format_string == "docker" ? [""] : []
+        content {
+          public_repository = "DOCKER_HUB"
+        }
+      }
+      dynamic "maven_repository" {
+        for_each = local.format_string == "maven" ? [""] : []
+        content {
+          public_repository = "MAVEN_CENTRAL"
+        }
+      }
+      dynamic "npm_repository" {
+        for_each = local.format_string == "npm" ? [""] : []
+        content {
+          public_repository = "NPMJS"
+        }
+      }
+      dynamic "python_repository" {
+        for_each = local.format_string == "python" ? [""] : []
+        content {
+          public_repository = "PYPI"
+        }
+      }
+    }
+  }
+
+  dynamic "virtual_repository_config" {
+    for_each = local.mode_string == "virtual" ? [""] : []
+    content {
+      dynamic "upstream_policies" {
+        for_each = var.mode.virtual
+        content {
+          id         = upstream_policies.key
+          repository = upstream_policies.value.repository
+          priority   = upstream_policies.value.priority
+        }
+      }
+    }
+  }
+
+  lifecycle {
+    precondition {
+      condition = local.mode_string != "remote" || contains(
+        ["docker", "maven", "npm", "python"], local.format_string
+      )
+      error_message = "Invalid format for remote repository."
+    }
+  }
+
 }
 
 resource "google_artifact_registry_repository_iam_binding" "bindings" {

modules/artifact-registry/variables.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,9 +27,30 @@ variable "encryption_key" {
 }
 
 variable "format" {
-  description = "Repository format. One of DOCKER or UNSPECIFIED."
-  type        = string
-  default     = "DOCKER"
+  description = "Repository format."
+  type = object({
+    apt = optional(object({}))
+    docker = optional(object({
+      immutable_tags = optional(bool)
+    }))
+    kfp = optional(object({}))
+    go  = optional(object({}))
+    maven = optional(object({
+      allow_snapshot_overwrites = optional(bool)
+      version_policy            = optional(string)
+    }))
+    npm    = optional(object({}))
+    python = optional(object({}))
+    yum    = optional(object({}))
+  })
+  nullable = false
+  default  = { docker = {} }
+  validation {
+    condition = (
+      length([for k, v in var.format : k if v != null]) == 1
+    )
+    error_message = "Multiple or zero formats are not supported."
+  }
 }
 
 variable "iam" {
@@ -38,11 +59,6 @@ variable "iam" {
   default     = {}
 }
 
-variable "id" {
-  description = "Repository id."
-  type        = string
-}
-
 variable "labels" {
   description = "Labels to be attached to the registry."
   type        = map(string)
@@ -52,7 +68,31 @@ variable "labels" {
 variable "location" {
   description = "Registry location. Use `gcloud beta artifacts locations list' to get valid values."
   type        = string
-  default     = null
+}
+
+variable "mode" {
+  description = "Repository mode."
+  type = object({
+    standard = optional(bool)
+    remote   = optional(bool)
+    virtual = optional(map(object({
+      repository = string
+      priority   = number
+    })))
+  })
+  nullable = false
+  default  = { standard = true }
+  validation {
+    condition = (
+      length([for k, v in var.mode : k if v != null && v != false]) == 1
+    )
+    error_message = "Multiple or zero modes are not supported."
+  }
+}
+
+variable "name" {
+  description = "Registry name."
+  type        = string
 }
 
 variable "project_id" {