Merge pull request #21 from europeana/EA-3252_k8s

EA-3252: Setup kustomize for k8s deployment

.gitignore

@@ -8,4 +8,5 @@
 *.iml
 
 # Kubernetes config properties
-*.properties.yaml
+*.properties.yaml
+*.user.properties

Dockerfile

@@ -1,18 +1,13 @@
-# NB: ensure the version matches that in buildpack.yml
-FROM nginx:mainline
+FROM nginx:1.23
 
 LABEL Author="Europeana Foundation <development@europeana.eu>"
 
-# TODO: add a self-signed SSL certificate
-
-COPY docker/env-nginx docker/rewrite-nginx-conf /usr/sbin/
+COPY env-nginx  /usr/sbin/
 
 COPY public /usr/share/nginx/html
 COPY mime.types /etc/nginx/
 COPY nginx.conf.d/ /etc/nginx/nginx.conf.d/
 
-COPY nginx.conf /etc/nginx/nginx.template.conf
-
-RUN rewrite-nginx-conf
+COPY nginx.conf.template /etc/nginx/nginx.conf.template
 
 ENTRYPOINT ["env-nginx"]

docker/env-nginx → env-nginx

@@ -2,6 +2,10 @@
 
 # Substitute env var strings in NGINX config with values from env
 
+# Also use default nameserver for resolving hostnames (necessary for k8s deployments so we can use service names)
+NAMESERVER=$(cat /etc/resolv.conf | grep "nameserver" | awk '{print $2}' | tr '\n' ' ')
+export RESOLVER_ADDRESS="${NAMESERVER} ${RESOLVER_ADDRESS}"
+
 vars='$ANNOTATION_API_HOST
       $ENTITY_API_HOST
       $ENTITY_MANAGEMENT_HOST
@@ -15,7 +19,7 @@ vars='$ANNOTATION_API_HOST
       $THUMBNAIL_API_HOST
       $RECOMMENDATION_API_HOST'
 
-envsubst "${vars}" < /etc/nginx/nginx.template.conf > /etc/nginx/nginx.conf
+envsubst "${vars}" < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf
 
 if [ -z "$1" ]; then
       echo "No container params. Running default nginx command"

k8s/deployment.yaml → k8s/base/deployment.yaml

@@ -3,14 +3,8 @@ kind: Deployment
 metadata:
   name: api-gateway-deployment
 spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: api-gateway
+# selector.matchLabels provided via Kustomize
   template:
-    metadata:
-      labels:
-        app: api-gateway
     spec:
       containers:
         - name: api-gateway
@@ -20,3 +14,4 @@ spec:
           envFrom:
             - configMapRef:
                 name: api-gateway-config
+

k8s/base/kustomization.yaml

@@ -0,0 +1,8 @@
+resources:
+  - deployment.yaml
+
+# expects file to be in the same directory
+configMapGenerator:
+  - name: api-gateway-config
+    envs:
+      - .env

k8s/overlays/cloud/deployment_patch.properties.yaml.template

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: api-gateway-deployment
+spec:
+    replicas: ${MIN_REPLICAS}
+    template:
+      spec:
+        containers:
+          - name: api-gateway
+            imagePullPolicy: Always
+            resources:
+              requests:
+                memory: "${MEMORY_REQUEST}M"
+                cpu: "${CPU_REQUEST}m"
+              limits:
+                memory: "${MEMORY_LIMIT}M"
+                cpu: "${CPU_LIMIT}m"

k8s/overlays/cloud/hpa.properties.yaml.template

@@ -0,0 +1,21 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: api-gateway-hpa
+spec:
+  maxReplicas: ${MAX_REPLICAS}
+  minReplicas: ${MIN_REPLICAS}
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: api-gateway-deployment
+  behavior:
+    scaleDown:
+      stabilizationWindowSeconds: 900
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 70

k8s/ingress.yaml.template → k8s/overlays/cloud/ingress.properties.yaml.template

@@ -8,20 +8,21 @@ metadata:
   name: api-gateway-ingress
   annotations:
     cert-manager.io/issuer: letsencrypt-production
+    nginx.ingress.kubernetes.io/server-alias: ${K8S_SERVER_ALIASES}
 spec:
   ingressClassName: public-iks-k8s-nginx
   tls:
     - hosts:
-        - HOSTNAME
-      secretName: SECRET_NAME
+        - ${K8S_HOSTNAME}
+      secretName: ${K8S_HOSTNAME}
   rules:
-    - host: HOSTNAME
+    - host: ${K8S_HOSTNAME}
       http:
         paths:
           - path: /
             pathType: Prefix
             backend:
               service:
-                name: api-gateway-cluster-ip-service
+                name: api-gateway-service
                 port:
                   number: 80

k8s/overlays/cloud/kustomization.yaml

@@ -0,0 +1,13 @@
+# name suffix and commonLabels to be provided imperatively during deployment
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../base
+- service.yaml
+- ingress.properties.yaml
+- hpa.properties.yaml
+
+patchesStrategicMerge:
+- deployment_patch.properties.yaml
+

k8s/overlays/cloud/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: api-gateway-service
+spec:
+  # selector provided via Kustomize
+  ports:
+    - name: http
+      port: 80

k8s/overlays/dev/deployment_patch.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: api-gateway-deployment
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+        - name: api-gateway
+          # required to use local image
+          imagePullPolicy: Never

k8s/overlays/dev/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../base
+  - nodeport.yaml
+
+patchesStrategicMerge:
+  - deployment_patch.yaml
+
+commonLabels:
+  app: api-gateway-dev

k8s/node-port.yaml → k8s/overlays/dev/nodeport.yaml

@@ -5,7 +5,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: api-gateway-node-port
+  name: api-gateway-service
 spec:
   type: NodePort
   ports:

nginx.conf → nginx.conf.template

@@ -1,8 +1,4 @@
-# Based on: https://raw.githubusercontent.com/cloudfoundry/nginx-buildpack/v1.0.1/fixtures/mainline/nginx.conf
-
 worker_processes auto;
-
-# see https://github.com/cloudfoundry/nginx-buildpack/blob/master/fixtures/mainline/nginx.conf#L2
 daemon off;
 
 error_log stderr warn;
@@ -24,42 +20,38 @@ http {
     tcp_nodelay on;
     tcp_nopush on;
     keepalive_timeout 30;
-    port_in_redirect off; # Ensure that redirects don't include the internal container PORT - 8080
 
     server {
-        listen {{port}};
+        listen 80;
         root public;
         index index.html;
 
-        # Enforce SSL
-        if ($http_x_forwarded_proto != "https") {
-            return 301 https://$host$request_uri;
-        }
+        # HTTPS enforcement should be done within Kubernetes ingress
 
         # "upstream" servers do not use resolver settings by default (paid option),
         # so we proxy directly to the urls
-        set $search_api {{env "SEARCH_API_HOST"}};
-        set $thumbnail_api {{env "THUMBNAIL_API_HOST"}};
-        set $oai_record {{env "OAI_RECORD_HOST"}};
-        set $annotation_api {{env "ANNOTATION_API_HOST"}};
-        set $entity_api {{env "ENTITY_API_HOST"}};
-        set $entity_management {{env "ENTITY_MANAGEMENT_HOST"}};
-        set $manifest_api {{env "MANIFEST_API_HOST"}};
-        set $set_api {{env "SET_API_HOST"}};
-        set $fulltext_api {{env "FULLTEXT_API_HOST"}};
-        set $recommendation_api {{env "RECOMMENDATION_API_HOST"}};
+        set $search_api ${SEARCH_API_HOST};
+        set $thumbnail_api ${THUMBNAIL_API_HOST};
+        set $oai_record ${OAI_RECORD_HOST};
+        set $annotation_api ${ANNOTATION_API_HOST};
+        set $entity_api ${ENTITY_API_HOST};
+        set $entity_management ${ENTITY_MANAGEMENT_HOST};
+        set $manifest_api ${MANIFEST_API_HOST};
+        set $set_api ${SET_API_HOST};
+        set $fulltext_api ${FULLTEXT_API_HOST};
+        set $recommendation_api ${RECOMMENDATION_API_HOST};
 
 
         # Override TTL from responses; cache IP addresses for 5 minutes 
-        resolver {{env "RESOLVER_ADDRESS"}} valid=5m;
+        resolver ${RESOLVER_ADDRESS} valid=5m;
 
         add_header Link "<http://creativecommons.org/publicdomain/zero/1.0/>;rel=license";
 
         include nginx.conf.d/*.conf;
 
         # Redirect root
         location = / {
-            return 302 {{env "ROOT_REDIRECT_URL"}};
+            return 302 ${ROOT_REDIRECT_URL};
         }