Feat/staff api (#53)

* feat(helm/charts): add staff-api UNTESTED, needs env vars, testing * chore(staff): set Always pull policy * feat(staff): use `edge` image tag * fix(staff/secrets): use place_auth_client_id & secret not secret_key_base * feat(ingress/frontends): use gzip * chore(staff): remove STAFF_DOMAINS env var * chore(staff/ingress): add x-frame-options response header * fix(staff-api): default SG_ENV to production * added functionality to enabled GCP only layer 7 loadbalancer * docs: helm chart docs updated * feat(https-lb): add staff config for gcp https-lb, ansible. nginx readiness probe * fix(staff): gke inventory config * fix(ingress): healthchecks * fix(staff): healthceck * fix(lb): staff path * fix(network-policies): allow external (lb) to placeos namespace * Squashed commit of the following: openshift v4 single namespace internal registry option placeos image version update staff & postgres configuration * chore(readme): set kubectl context for openshift * chore(readme): define placeDomain when running playbook on gke * fix(gcp): fix postgres service name * chore(git): ignore swap files * fix(ansible): seperate release names for third party charts * chore(readme): clarify gcp external IP * chore(readme): gcp external ip * chore(fix): fix stable chart repo * fix(postgres): ansible templating * fix(postgres): ansible generate postgres password * chore(readme): gke- add note about cloud armor security policy requirement * chore(ansible): add staff and postgres templating for aws, aks, k3d * feat(placeos): update to 2109.0 * fix(frontends): use new frontend-loader image name * feat(staff): use place version image tag * fix(frontends): update healthcheck path * fix(rubbersoul): set ES_DISABLE_BULK: false * fix(postgres): correct name for postgres chart template * fix(staff): postgres service name * feat(health): enable readiness & liveness probes for rubber-soul & triggers * feat(place): upgrade to placeos-1.2202.2 * fix(helm): fix frontend-loader/search-ingest service names for helm * fix(gke): remove merge artifacts * fix(staff): postgresql service name * fix(staff): add PLACE_PASSWORD & PLACE_SERVER_SECRET overrides * fix(ansible): frontend-loader & search-ingest chart names * fix(auth): generate & add SECRET_KEY_BASE * fix(charts): update from deprecated bitnami chart versions * fix(redis): update chart definitions * fix(postgres): update chart definitions * fix(network): seperate network policy for GCP LB deny gcp vpc range, allowing namespace and lb * fix(ansible): update service name template * feat(placeos): update to 1.2207.0 * fix(staff): postgres auth template * chore(frontends): delete old template * fix(frontendloader): read-only file system: false Co-authored-by: William Le <will@place.technology> Co-authored-by: William Le <w.le@acaprojects.com> Co-authored-by: jahoward <jahoward@deloitte.com.au> Co-authored-by: William Le <will@place.tech>
place-labs · Jul 12, 2022 · c7d60ef · c7d60ef
1 parent 5068332
commit c7d60ef
Show file tree

Hide file tree

Showing 98 changed files with 1,503 additions and 352 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+*.swp
+*.swo
diff --git a/ansible/README.md b/ansible/README.md
@@ -19,6 +19,7 @@ Contains 4 roles:
 
 - Review the requirements for the [Ansible helm wrapper](https://docs.ansible.com/ansible/2.10/collections/community/kubernetes/helm_module.html)
 
+- GKE: a Cloud Armor Security must exist for the Load Balancer to associate with
 
 Note: Tested with:
 
@@ -42,10 +43,13 @@ ansible-playbook placeos.yaml -i inventories/k3d/
 
 
 # GKE deployment
+# Set the Cloud Armor security policy name in inventories/gke/host_vars/k8s.yaml as placeos.global.gcpbackendConfig.config.securityPolicy
 # Check first be for deploying
 ansible-playbook placeos.yaml -i inventories/gke/  --check
-ansible-playbook placeos.yaml -i inventories/gke/
-ansible-playbook placeos-network-policies.yaml
+# Define the placeDomain value when running:
+# Terraform will output the created External IP or find `l7-ip` at `VPC Network -> External IP Addresses`
+ansible-playbook placeos.yaml -i inventories/gke/ -e "placeDomain={domain/{external IP.sslip.io}}"
+ansible-playbook placeos-network-policies.yaml -e "gke=true"
 
 # AKS deployment
 # Check first be for deploying

diff --git a/ansible/inventories/aks/host_vars/k8s.yaml b/ansible/inventories/aks/host_vars/k8s.yaml
@@ -2,6 +2,9 @@
 #  Used to set the admin password in RethinkDB if enabled then passed to multiple containers in a k8s secret
 xrethinkdbPassword: &rethinkdbPassword "{{ rethinkdbPassword }}"
 
+#  Used to set the postgres user password in PostgresQL if enabled then passed to multiple containers in a k8s secret
+xpostgresqlPassword: &postgresqlPassword "{{ postgresqlPassword }}"
+
 # xsmtpPassword. the smtp password used by triggers
 xsmtpPassword: &smtpPassword "{{ smtpPassword | default( '', true ) }}"
 
@@ -11,12 +14,15 @@ xplacePassword: &placePassword "{{ placePassword | default( '', true ) }}"
 # xplaceServerSecret for the placeos service
 xplaceServerSecret: &placeServerSecret "{{ placeServerSecret | default( '', true ) }}"
 
+# xplaceServerSecret for the placeos service
+xauthSecretKeyBase: &authSecretKeyBase "{{ authSecretKeyBase | default( '', true ) }}"
+
 # xserverSecret for the dispatch service
 xserverSecret: &serverSecret "{{ serverSecret | default( '', true ) }}"
 
 # xelasticClientEnv configmap values for ElasticSearch Service exposed as environment variables to PlaceOS containers
 xelasticClientEnv: &elasticClientEnv
-  ES_HOST: "{{ chart_release_name }}-elasticsearch-master.{{ elasticsearch_namespace }}"
+  ES_HOST: "{{ elasticsearch_release_name }}-elasticsearch-master.{{ elasticsearch_namespace }}"
   ES_PORT: 9200
 
 # xetcdClientEnv configmap values for Etcd end point Service as environment variables to PlaceOS containers
@@ -30,14 +36,23 @@ xredisClientEnv: &redisClientEnv
 
 # xrethinkdbClientEnv configmap values for RethinkDB Service exposed as environment variables to PlaceOS containers
 xrethinkdbClientEnv: &rethinkdbClientEnv
-  RETHINKDB_HOST: "{{ chart_release_name }}-rethinkdb-proxy.{{ rethinkdb_namespace }}"
+  RETHINKDB_HOST: "{{ rethinkdb_release_name }}-rethinkdb-proxy.{{ rethinkdb_namespace }}"
   RETHINKDB_PORT: 28015
   RETHINKDB_DB: place_development
   RETHINKDB_USER: admin
 
 xrethinkdbClientSecrets: &rethinkdbClientSecrets
   RETHINKDB_PASSWORD: *rethinkdbPassword
 
+xpostgresqlValues: &postgresqlValues
+  global:
+    postgresql:
+      auth:
+        username: placeos
+        database: placeos
+        password: *postgresqlPassword
+        postgresPassword: *postgresqlPassword
+
 # xsmtpClientSecrets secrets values exposed as environment variables for the trigger service
 xsmtpClientSecrets: &smtpClientSecrets
   SMTP_PASS: *smtpPassword
@@ -72,6 +87,7 @@ placeos:
     configmap:
       << : *rethinkdbClientEnv
     secrets:
+      SECRET_KEY_BASE: *authSecretKeyBase
       << : *rethinkdbClientSecrets
   # core is the overide configuration for the embedded core subchart
   core:
@@ -93,8 +109,8 @@ placeos:
         service.beta.kubernetes.io/azure-load-balancer-internal: "true"
     secrets:
       << : *dispatchSecrets
-  # frontends is the overide configuration for the embedded frontends subchart
-  frontends:
+  # frontend-loader is the overide configuration for the embedded frontend-loader subchart
+  frontendloader:
     ingress:
       annotations:
         kubernetes.io/ingress.class: nginx
@@ -105,8 +121,8 @@ placeos:
     secrets:
       << : *rethinkdbClientSecrets
       PLACE_SERVER_SECRET: *placeServerSecret
-  # rubbersoul is the overide configuration for the embedded rubber-soul subchart
-  rubbersoul:
+  # search-ingest is the overide configuration for the embedded search-ingest subchart
+  searchingest:
     configmap:
       # Service Hosts
       << : *rethinkdbClientEnv
@@ -134,6 +150,17 @@ placeos:
     secrets:
       PLACE_PASSWORD: *placePassword
       PLACE_SERVER_SECRET: *placeServerSecret
+  # staff is the overide configuration for the embedded staff subchart
+  staff:
+    ingress:
+      enabled: false
+      annotations:
+        kubernetes.io/ingress.class: nginx
+        ingress.kubernetes.io/ssl-redirect: "true"
+    secrets:
+      PLACE_PASSWORD: *placePassword
+      PLACE_SERVER_SECRET: *placeServerSecret
+    << : *postgresqlValues
   global:
     placeDomain: "{{ placeDomain | default('localhost', true ) }}"
     customRedirectPort: "{{ customRedirectPort | default( omit ) }}"
@@ -165,14 +192,25 @@ rethinkdb:
     replicas: 1
   rethinkdbPassword: "{{ rethinkdbPassword }}"
 redis:
-  fullnameOverride: redis
   enabled: true
-  usePassword: false
-  cluster:
+  fullnameOverride: redis
+  auth:
     enabled: false
+  architecture: standalone
+  commonConfiguration: |-
+    notify-keyspace-events Kg$
+  master:
+    persistence:
+      enabled: true
+      size: 1Gi
+  fullnameOverride: redis
 influxdb:
   fullnameOverride: influxdb
   enabled: false
 mosquitto:
   enabled: false
   fullnameOverride: mosquitto
+postgresql:
+  enabled: true
+  fullnameOverride: postgresql
+  << : *postgresqlValues
diff --git a/ansible/inventories/aws/host_vars/k8s.yaml b/ansible/inventories/aws/host_vars/k8s.yaml
@@ -2,6 +2,9 @@
 #  Used to set the admin password in RethinkDB if enabled then passed to multiple containers in a k8s secret
 xrethinkdbPassword: &rethinkdbPassword "{{ rethinkdbPassword }}"
 
+#  Used to set the postgres user password in PostgresQL if enabled then passed to multiple containers in a k8s secret
+xpostgresqlPassword: &postgresqlPassword "{{ postgresqlPassword }}"
+
 # xsmtpPassword. the smtp password used by triggers
 xsmtpPassword: &smtpPassword "{{ smtpPassword | default( '', true ) }}"
 
@@ -11,12 +14,15 @@ xplacePassword: &placePassword "{{ placePassword | default( '', true ) }}"
 # xplaceServerSecret for the placeos service
 xplaceServerSecret: &placeServerSecret "{{ placeServerSecret | default( '', true ) }}"
 
+# xplaceServerSecret for the placeos service
+xauthSecretKeyBase: &authSecretKeyBase "{{ authSecretKeyBase | default( '', true ) }}"
+
 # xserverSecret for the dispatch service
 xserverSecret: &serverSecret "{{ serverSecret | default( '', true ) }}"
 
 # xelasticClientEnv configmap values for ElasticSearch Service exposed as environment variables to PlaceOS containers
 xelasticClientEnv: &elasticClientEnv
-  ES_HOST: "{{ chart_release_name }}-elasticsearch-master.{{ elasticsearch_namespace }}"
+  ES_HOST: "{{ elasticsearch_release_name }}-elasticsearch-master.{{ elasticsearch_namespace }}"
   ES_PORT: 9200
 
 # xetcdClientEnv configmap values for Etcd end point Service as environment variables to PlaceOS containers
@@ -30,14 +36,23 @@ xredisClientEnv: &redisClientEnv
 
 # xrethinkdbClientEnv configmap values for RethinkDB Service exposed as environment variables to PlaceOS containers
 xrethinkdbClientEnv: &rethinkdbClientEnv
-  RETHINKDB_HOST: "{{ chart_release_name }}-rethinkdb-proxy.{{ rethinkdb_namespace }}"
+  RETHINKDB_HOST: "{{ rethinkdb_release_name }}-rethinkdb-proxy.{{ rethinkdb_namespace }}"
   RETHINKDB_PORT: 28015
   RETHINKDB_DB: place_development
   RETHINKDB_USER: admin
 
 xrethinkdbClientSecrets: &rethinkdbClientSecrets
   RETHINKDB_PASSWORD: *rethinkdbPassword
 
+xpostgresqlValues: &postgresqlValues
+  global:
+    postgresql:
+      auth:
+        username: placeos
+        database: placeos
+        password: *postgresqlPassword
+        postgresPassword: *postgresqlPassword
+
 # xsmtpClientSecrets secrets values exposed as environment variables for the trigger service
 xsmtpClientSecrets: &smtpClientSecrets
   SMTP_PASS: *smtpPassword
@@ -72,6 +87,7 @@ placeos:
     configmap:
       << : *rethinkdbClientEnv
     secrets:
+      SECRET_KEY_BASE: *authSecretKeyBase
       << : *rethinkdbClientSecrets
   # core is the overide configuration for the embedded core subchart
   core:
@@ -94,8 +110,8 @@ placeos:
       service.beta.kubernetes.io/aws-load-balancer-internal: "true"
     secrets:
       << : *dispatchSecrets
-  # frontends is the overide configuration for the embedded frontends subchart
-  frontends:
+  # frontend-loader is the overide configuration for the embedded frontend-loader subchart
+  frontendloader:
     ingress:
       annotations:
         kubernetes.io/ingress.class: nginx
@@ -106,8 +122,8 @@ placeos:
     secrets:
       << : *rethinkdbClientSecrets
       PLACE_SERVER_SECRET: *placeServerSecret
-  # rubbersoul is the overide configuration for the embedded rubber-soul subchart
-  rubbersoul:
+  # search-ingest is the overide configuration for the embedded search-ingest subchart
+  searchingest:
     configmap:
       # Service Hosts
       << : *rethinkdbClientEnv
@@ -135,6 +151,17 @@ placeos:
     secrets:
       PLACE_PASSWORD: *placePassword
       PLACE_SERVER_SECRET: *placeServerSecret
+  # staff is the overide configuration for the embedded staff subchart
+  staff:
+    ingress:
+      enabled: false
+      annotations:
+        kubernetes.io/ingress.class: nginx
+        ingress.kubernetes.io/ssl-redirect: "true"
+    secrets:
+      PLACE_PASSWORD: *placePassword
+      PLACE_SERVER_SECRET: *placeServerSecret
+    << : *postgresqlValues
   global:
     placeDomain: "{{ placeDomain | default('localhost', true ) }}"
     customRedirectPort: "{{ customRedirectPort | default( omit ) }}"
@@ -166,14 +193,24 @@ rethinkdb:
     replicas: 1
   rethinkdbPassword: "{{ rethinkdbPassword }}"
 redis:
-  fullnameOverride: redis
   enabled: true
-  usePassword: false
-  cluster:
+  fullnameOverride: redis
+  auth:
     enabled: false
+  architecture: standalone
+  commonConfiguration: |-
+    notify-keyspace-events Kg$
+  master:
+    persistence:
+      enabled: true
+      size: 1Gi
 influxdb:
   fullnameOverride: influxdb
   enabled: false
 mosquitto:
   enabled: false
   fullnameOverride: mosquitto
+postgresql:
+  enabled: true
+  fullnameOverride: postgresql
+  << : *postgresqlValues