v3

bin/create-project.sh

@@ -86,7 +86,7 @@ do
   gcloud services --project $project enable iam.googleapis.com
   gcloud services --project $project enable container.googleapis.com
   gcloud services --project $project enable cloudresourcemanager.googleapis.com
-  gcloud services --project $project enable dns.googleapis.com
+  gcloud services --project $project enable dns.googleapis.com # probably to delete - Airflow DNS
 done
 
 

main.tf

@@ -24,6 +24,7 @@ provider "helm" {
   }
 }
 
+# needed e.g. for creating k8s secrets
 provider "kubernetes" {
   host = "https://${module.gke.endpoint}"
   token = "${data.google_client_config.default.access_token}"
@@ -66,18 +67,6 @@ resource "kubernetes_secret" "airflow-cluster1-fernet-key" {
   type = "Secret"
 }
 
-// password is created by postgresql operator postgres.acid-minimal-cluster.credentials.postgresql.acid.zalan.do
-//resource "kubernetes_secret" "airflow-cluster1-mysql-password" {
-//  metadata {
-//    name = "airflow-cluster1-mysql-password"
-//    namespace = "airflow-cluster1"
-//  }
-//  data = {
-//    "mysql-password" = file("./modules/airflow/gke_resources/secret-mysql-password.yaml")
-//  }
-//  type = "Secret"
-//}
-
 resource "kubernetes_secret" "airflow-cluster1-redis-password" {
   metadata {
     name = "airflow-cluster1-redis-password"
@@ -119,26 +108,21 @@ resource "helm_release" "kube-cert-manager" {
   ]
 }
 
-resource "helm_release" "postgres" {
-  chart = "../postgres-operator/charts/postgres-operator"
-  name = "postgres-operator"
-  namespace = "airflow-cluster1"
-
-  values = [
-    file("../postgres-operator/charts/postgres-operator/my-config.yaml")
-  ]
+module "postgres" {
+  source = "./modules/postgres"
+  depends_on = [module.gke]
 }
 
-// run in second iteration after creating postgres cluster
-
 module "airflow" {
   source = "./modules/airflow"
   depends_on = [module.gke]
   project_name = var.project_name
   location = var.location
 }
 
-// ok
+
+
+// by now focus on Airflow
 
 //module "spark" {
 //  source   = "./modules/spark"

modules/airflow/module.tf

@@ -4,7 +4,7 @@ resource "google_service_account" "airflow-service-account" {
 }
 
 resource "google_storage_bucket" "airflow-logs-storage" {
-  name = "airflow-cluster1"  # todo find way of using variables in config.yaml to use var.project_name in a bucket name
+  name = "airflow-cluster1"
   location = var.location
   force_destroy = true
 }
@@ -18,15 +18,6 @@ resource "google_storage_bucket_iam_binding" "binding" {
   ]
 }
 
-//sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) FATAL:  password authentication failed for user "airflow_user"
-//FATAL:  pg_hba.conf rejects connection for host "10.24.3.10", user "airflow_user", database "airflow_db", SSL off
-
-//my-min-manifest.yaml
-//kubectl create -f ../postgres-operator/charts/postgres-operator/my-min-manifest.yaml
-//
-//kubectl describe pod airflow-stable-worker-1 --namespace airflow-cluster1
-//kubectl logs airflow-stable-worker-1 -c check-db --namespace airflow-cluster1
-
 resource "helm_release" "kube-airflow" {
   name = "airflow-stable"
   repository = "https://airflow-helm.github.io/charts"

modules/postgres/module.tf

@@ -0,0 +1,9 @@
+resource "helm_release" "postgres" {
+  chart = "../postgres-operator/charts/postgres-operator"
+  name = "postgres-operator"
+  namespace = "airflow-cluster1"
+
+  values = [
+    file("./modules/postgres/resources/config.yaml")
+  ]
+}

modules/postgres/resources/config.yaml

@@ -0,0 +1,170 @@
+apiVersion: "acid.zalan.do/v1"
+kind: OperatorConfiguration
+metadata:
+  name: postgresql-operator-default-configuration
+configuration:
+  docker_image: registry.opensource.zalan.do/acid/spilo-13:2.0-p6
+  # enable_crd_validation: true
+  # enable_lazy_spilo_upgrade: false
+  enable_pgversion_env_var: true
+  # enable_shm_volume: true
+  enable_spilo_wal_path_compat: false
+  etcd_host: ""
+  # kubernetes_use_configmaps: false
+  max_instances: -1
+  min_instances: -1
+  resync_period: 30m
+  repair_period: 5m
+  # set_memory_request_to_limit: false
+  # sidecars:
+  # - image: image:123
+  #   name: global-sidecar-1
+  #   ports:
+  #   - containerPort: 80
+  #     protocol: TCP
+  workers: 8
+  users:
+    replication_username: standby
+    super_username: postgres
+  major_version_upgrade:
+    major_version_upgrade_mode: "off"
+    minimal_major_version: "9.5"
+    target_major_version: "13"
+  kubernetes:
+    # additional_pod_capabilities:
+    # - "SYS_NICE"
+    cluster_domain: cluster.local
+    cluster_labels:
+      application: spilo
+    cluster_name_label: cluster-name
+    # custom_pod_annotations:
+    #   keya: valuea
+    #   keyb: valueb
+    # delete_annotation_date_key: delete-date
+    # delete_annotation_name_key: delete-clustername
+    # downscaler_annotations:
+    # - deployment-time
+    # - downscaler/*
+    enable_init_containers: true
+    enable_pod_antiaffinity: false
+    enable_pod_disruption_budget: true
+    enable_sidecars: true
+    # infrastructure_roles_secret_name: "postgresql-infrastructure-roles"
+    # infrastructure_roles_secrets:
+    # - secretname: "monitoring-roles"
+    #   userkey: "user"
+    #   passwordkey: "password"
+    #   rolekey: "inrole"
+    # - secretname: "other-infrastructure-role"
+    #   userkey: "other-user-key"
+    #   passwordkey: "other-password-key"
+    # inherited_annotations:
+    # - owned-by
+    # inherited_labels:
+    # - application
+    # - environment
+    master_pod_move_timeout: 20m
+    # node_readiness_label:
+    #   status: ready
+    oauth_token_secret_name: postgresql-operator
+    pdb_name_format: "postgres-{cluster}-pdb"
+    pod_antiaffinity_topology_key: "kubernetes.io/hostname"
+    # pod_environment_configmap: "default/my-custom-config"
+    # pod_environment_secret: "my-custom-secret"
+    pod_management_policy: "ordered_ready"
+    # pod_priority_class_name: "postgres-pod-priority"
+    pod_role_label: spilo-role
+    # pod_service_account_definition: ""
+    pod_service_account_name: postgres-pod
+    # pod_service_account_role_binding_definition: ""
+    pod_terminate_grace_period: 5m
+    secret_name_template: "{username}.{cluster}.credentials.{tprkind}.{tprgroup}"
+    spilo_allow_privilege_escalation: true
+    # spilo_runasuser: 101
+    # spilo_runasgroup: 103
+    # spilo_fsgroup: 103
+    spilo_privileged: false
+    storage_resize_mode: pvc
+    # toleration: {}
+    # watched_namespace: ""
+  postgres_pod_resources:
+    default_cpu_limit: "1"
+    default_cpu_request: 100m
+    default_memory_limit: 500Mi
+    default_memory_request: 100Mi
+    # min_cpu_limit: 250m
+    # min_memory_limit: 250Mi
+  timeouts:
+    pod_label_wait_timeout: 10m
+    pod_deletion_wait_timeout: 10m
+    ready_wait_interval: 4s
+    ready_wait_timeout: 30s
+    resource_check_interval: 3s
+    resource_check_timeout: 10m
+  load_balancer:
+    # custom_service_annotations:
+    #   keyx: valuex
+    #   keyy: valuey
+    # db_hosted_zone: ""
+    enable_master_load_balancer: false
+    enable_replica_load_balancer: false
+    external_traffic_policy: "Cluster"
+    master_dns_name_format: "{cluster}.{team}.{hostedzone}"
+    replica_dns_name_format: "{cluster}-repl.{team}.{hostedzone}"
+  aws_or_gcp:
+    # additional_secret_mount: "some-secret-name"
+    # additional_secret_mount_path: "/some/dir"
+    aws_region: eu-central-1
+    enable_ebs_gp3_migration: false
+    # enable_ebs_gp3_migration_max_size: 1000
+    # gcp_credentials: ""
+    # kube_iam_role: ""
+    # log_s3_bucket: ""
+    # wal_gs_bucket: ""
+    # wal_s3_bucket: ""
+  logical_backup:
+    logical_backup_docker_image: "registry.opensource.zalan.do/acid/logical-backup:v1.6.2"
+    # logical_backup_google_application_credentials: ""
+    logical_backup_job_prefix: "logical-backup-"
+    logical_backup_provider: "s3"
+    # logical_backup_s3_access_key_id: ""
+    logical_backup_s3_bucket: "my-bucket-url"
+    # logical_backup_s3_endpoint: ""
+    # logical_backup_s3_region: ""
+    # logical_backup_s3_secret_access_key: ""
+    logical_backup_s3_sse: "AES256"
+    logical_backup_schedule: "30 00 * * *"
+  debug:
+    debug_logging: true
+    enable_database_access: true
+  teams_api:
+    # enable_admin_role_for_users: true
+    # enable_postgres_team_crd: false
+    # enable_postgres_team_crd_superusers: false
+    enable_team_superuser: false
+    enable_teams_api: false
+    # pam_configuration: ""
+    pam_role_name: zalandos
+    # postgres_superuser_teams:
+    # - postgres_superusers
+    protected_role_names:
+      - admin
+    team_admin_role: admin
+    team_api_role_configuration:
+      log_statement: all
+    # teams_api_url: ""
+  logging_rest_api:
+    api_port: 8080
+    cluster_history_entries: 1000
+    ring_log_lines: 100
+  connection_pooler:
+    connection_pooler_default_cpu_limit: "1"
+    connection_pooler_default_cpu_request: "500m"
+    connection_pooler_default_memory_limit: 100Mi
+    connection_pooler_default_memory_request: 100Mi
+    connection_pooler_image: "registry.opensource.zalan.do/acid/pgbouncer:master-16"
+    # connection_pooler_max_db_connections: 60
+    connection_pooler_mode: "transaction"
+    connection_pooler_number_of_instances: 2
+    # connection_pooler_schema: "pooler"
+    # connection_pooler_user: "pooler"

modules/postgres/resources/manifest.yaml

@@ -0,0 +1,24 @@
+apiVersion: "acid.zalan.do/v1"
+kind: postgresql
+metadata:
+  name: acid-minimal-cluster
+  namespace: airflow-cluster1
+spec:
+  teamId: "acid"
+  volume:
+    size: 1Gi
+  numberOfInstances: 2
+  users:
+    zalando:  # database owner
+      - superuser
+      - createdb
+    airflow_user:
+      - superuser
+      - createdb  # role for application foo
+  databases:
+    foo: zalando  # dbname: owner
+    airflow_db: airflow_user
+  preparedDatabases:
+    bar: {}
+  postgresql:
+    version: "13"