From 8224e26ae1be43de42dc2474801499ac898441e5 Mon Sep 17 00:00:00 2001 From: Julien Eluard Date: Wed, 22 Apr 2020 17:33:17 +0200 Subject: [PATCH] [Fixed #17] Moved to kustomize. Use dedicated cluster per environment --- Makefile | 4 +- backend/Rocket.toml | 8 -- backend/src/kubernetes.rs | 46 ++++---- backend/src/main.rs | 6 +- conf/Dockerfile | 2 +- conf/k8s/base/deployment.yaml | 25 +++-- conf/k8s/base/ingress.yaml | 1 + conf/k8s/base/kustomization.yaml | 7 +- conf/k8s/{ => base}/nginx.yaml | 101 +++++++++++------- conf/k8s/base/nginx/config-tcp-services.yaml | 7 -- conf/k8s/base/nginx/config-udp-services.yaml | 7 -- conf/k8s/base/nginx/config.yaml | 7 -- conf/k8s/base/nginx/role.yaml | 42 -------- conf/k8s/base/nginx/service-account.yaml | 7 -- conf/k8s/base/nginx/service.yaml | 23 ---- conf/k8s/base/service.yaml | 2 +- conf/k8s/deployment.yaml.tmpl | 98 ----------------- conf/k8s/overlays/dev/theia-images/template | 1 - .../production/custom-load-balancer-ip.yaml | 6 ++ .../overlays/production/ingress-patch.yaml | 6 ++ .../overlays/production/kustomization.yaml | 29 ++++- conf/k8s/overlays/production/namespace.yaml | 4 + .../overlays/production/theia-images/template | 2 +- .../staging/custom-load-balancer-ip.yaml | 2 +- conf/k8s/overlays/staging/kustomization.yaml | 10 +- conf/k8s/overlays/staging/namespace.yaml | 4 + .../overlays/staging/theia-images/template | 2 +- 27 files changed, 174 insertions(+), 285 deletions(-) delete mode 100644 backend/Rocket.toml rename conf/k8s/{ => base}/nginx.yaml (77%) delete mode 100644 conf/k8s/base/nginx/config-tcp-services.yaml delete mode 100644 conf/k8s/base/nginx/config-udp-services.yaml delete mode 100644 conf/k8s/base/nginx/config.yaml delete mode 100644 conf/k8s/base/nginx/role.yaml delete mode 100644 conf/k8s/base/nginx/service-account.yaml delete mode 100644 conf/k8s/base/nginx/service.yaml delete mode 100644 conf/k8s/deployment.yaml.tmpl delete mode 100644 conf/k8s/overlays/dev/theia-images/template create mode 100644 conf/k8s/overlays/production/custom-load-balancer-ip.yaml create mode 100644 conf/k8s/overlays/production/ingress-patch.yaml create mode 100644 conf/k8s/overlays/production/namespace.yaml create mode 100644 conf/k8s/overlays/staging/namespace.yaml diff --git a/Makefile b/Makefile index 9ab0d35db..809c0790b 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ k8s-gke-static-ip: k8s-assert # Deploy playground on kubernetes k8s-deploy-playground: k8s-assert - kubectl apply --validate=true --record -k conf/k8s/overlays/${ENVIRONMENT} + kubectl apply --record -k conf/k8s/overlays/${ENVIRONMENT} # Undeploy playground from kubernetes k8s-undeploy-playground: k8s-assert @@ -106,4 +106,4 @@ k8s-undeploy-theia: k8s-assert # Creates or replaces the `images` config map from `conf/k8s/images/*.properties` k8s-update-images-config: k8s-assert - kubectl create configmap theia-images --namespace=${IDENTIFIER} --from-env-file=conf/k8s/images/${ENVIRONMENT}.properties --dry-run -o yaml | kubectl apply -f - \ No newline at end of file + kubectl create configmap theia-images --namespace=${IDENTIFIER} --from-file=conf/k8s/overlays/${ENVIRONMENT}/theia-images/ --dry-run -o yaml | kubectl apply -f - \ No newline at end of file diff --git a/backend/Rocket.toml b/backend/Rocket.toml deleted file mode 100644 index 096083d02..000000000 --- a/backend/Rocket.toml +++ /dev/null @@ -1,8 +0,0 @@ -[development] -address = "localhost" -port = 8000 -log = "normal" - -[production] -address = "0.0.0.0" -log = "normal" \ No newline at end of file diff --git a/backend/src/kubernetes.rs b/backend/src/kubernetes.rs index 577079b5b..cd40ff2f9 100644 --- a/backend/src/kubernetes.rs +++ b/backend/src/kubernetes.rs @@ -18,10 +18,12 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use std::{collections::BTreeMap, error::Error, time::SystemTime}; use uuid::Uuid; -const APP_LABEL: &str = "app"; -const APP_VALUE: &str = "theia-substrate"; -const USER_UUID_LABEL: &str = "user-uuid"; -const INSTANCE_UUID_LABEL: &str = "instance-uuid"; +const APP_LABEL: &str = "app.kubernetes.io/name"; +const APP_VALUE: &str = "playground"; +const COMPONENT_LABEL: &str = "app.kubernetes.io/component"; +const COMPONENT_VALUE: &str = "theia"; +const OWNER_LABEL: &str = "app.kubernetes.io/owner"; +const INSTANCE_LABEL: &str = "app.kubernetes.io/instance"; const INGRESS_NAME: &str = "ingress"; fn error_to_string(err: T) -> String { @@ -45,17 +47,18 @@ async fn list_by_selector( fn create_pod(user_uuid: &str, instance_uuid: &str, image: &str) -> Pod { let mut labels = BTreeMap::new(); labels.insert(APP_LABEL.to_string(), APP_VALUE.to_string()); - labels.insert(USER_UUID_LABEL.to_string(), user_uuid.to_string()); - labels.insert(INSTANCE_UUID_LABEL.to_string(), instance_uuid.to_string()); + labels.insert(COMPONENT_LABEL.to_string(), COMPONENT_VALUE.to_string()); + labels.insert(OWNER_LABEL.to_string(), user_uuid.to_string()); + labels.insert(INSTANCE_LABEL.to_string(), instance_uuid.to_string()); Pod { metadata: Some(ObjectMeta { - generate_name: Some(format!("{}-", APP_VALUE).to_string()), + generate_name: Some(format!("{}-", COMPONENT_VALUE).to_string()), labels: Some(labels), ..Default::default() }), spec: Some(PodSpec { containers: vec![Container { - name: format!("{}-container", APP_VALUE).to_string(), + name: format!("{}-container", COMPONENT_VALUE).to_string(), image: Some(image.to_string()), ..Default::default() }], @@ -68,12 +71,13 @@ fn create_pod(user_uuid: &str, instance_uuid: &str, image: &str) -> Pod { fn create_service(instance_uuid: &str) -> Service { let mut labels = BTreeMap::new(); labels.insert(APP_LABEL.to_string(), APP_VALUE.to_string()); - labels.insert(INSTANCE_UUID_LABEL.to_string(), instance_uuid.to_string()); + labels.insert(COMPONENT_LABEL.to_string(), COMPONENT_VALUE.to_string()); + labels.insert(INSTANCE_LABEL.to_string(), instance_uuid.to_string()); let mut selectors = BTreeMap::new(); - selectors.insert(INSTANCE_UUID_LABEL.to_string(), instance_uuid.to_string()); + selectors.insert(INSTANCE_LABEL.to_string(), instance_uuid.to_string()); Service { metadata: Some(ObjectMeta { - generate_name: Some(format!("{}-http-", APP_VALUE).to_string()), + generate_name: Some(format!("{}-http-", COMPONENT_VALUE).to_string()), labels: Some(labels), ..Default::default() }), @@ -149,7 +153,7 @@ pub struct Engine { namespace: String, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct InstanceDetails { pub user_uuid: String, pub instance_uuid: String, @@ -207,12 +211,12 @@ impl Engine { Ok(Engine { host, namespace }) } - fn user_selector(user_uuid: &str) -> String { - format!("{}={}", USER_UUID_LABEL, user_uuid) + fn owner_selector(user_uuid: &str) -> String { + format!("{}={}", OWNER_LABEL, user_uuid) } fn instance_selector(instance_uuid: &str) -> String { - format!("{}={}", INSTANCE_UUID_LABEL, instance_uuid) + format!("{}={}", INSTANCE_LABEL, instance_uuid) } fn pod_to_instance(self, pod: &Pod) -> Result { @@ -232,8 +236,8 @@ impl Engine { .as_ref() .and_then(|md| { Some(( - md.labels.clone()?.get(USER_UUID_LABEL)?.to_string(), - md.labels.clone()?.get(INSTANCE_UUID_LABEL)?.to_string(), + md.labels.clone()?.get(OWNER_LABEL)?.to_string(), + md.labels.clone()?.get(INSTANCE_LABEL)?.to_string(), )) }) .ok_or("Metadata unavailable")?; @@ -264,13 +268,13 @@ impl Engine { let config = config().await?; let client = APIClient::new(config); let pod_api: Api = Api::namespaced(client, &self.namespace); - let pods = list_by_selector(&pod_api, Engine::user_selector(user_uuid)).await?; + let pods = list_by_selector(&pod_api, Engine::owner_selector(user_uuid)).await?; let names: Vec = pods .iter() .flat_map(|pod| { pod.metadata .as_ref() - .and_then(|md| Some(md.labels.clone()?.get(INSTANCE_UUID_LABEL)?.to_string())) + .and_then(|md| Some(md.labels.clone()?.get(INSTANCE_LABEL)?.to_string())) }) .collect::>(); @@ -282,14 +286,14 @@ impl Engine { let client = APIClient::new(config); let pod_api: Api = Api::namespaced(client, &self.namespace); let pods = - list_by_selector(&pod_api, format!("{}={}", APP_LABEL, APP_VALUE).to_string()).await?; + list_by_selector(&pod_api, format!("{}={}", COMPONENT_LABEL, COMPONENT_VALUE).to_string()).await?; let names = pods .iter() .flat_map(|pod| { self.clone() .pod_to_instance(pod) .ok() - .map(|i| (/*i.user_uuid*/ "".to_string(), i)) + .map(|i| (i.clone().user_uuid, i)) }) .collect(); diff --git a/backend/src/main.rs b/backend/src/main.rs index 1e686b85a..f68e111a8 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,7 +7,7 @@ mod manager; mod metrics; use crate::manager::Manager; -use rocket::{http::Method, routes}; +use rocket::{config::Environment, http::Method, routes}; use rocket_contrib::serve::StaticFiles; use rocket_cors::{AllowedOrigins, CorsOptions}; use rocket_prometheus::PrometheusMetrics; @@ -24,7 +24,7 @@ pub struct Context { async fn main() -> Result<(), Box> { // Initialize log configuration. Reads `RUST_LOG` if any, otherwise fallsback to `default` if env::var("RUST_LOG").is_err() { - env::set_var("RUST_LOG", "info,kube=info"); + env::set_var("RUST_LOG", "warn"); } env_logger::init(); @@ -40,6 +40,8 @@ async fn main() -> Result<(), Box> { } .to_cors()?; + log::info!("Running in {:?} mode", Environment::active()?); + let manager = Manager::new().await?; manager.clone().spawn_background_thread(); let prometheus = PrometheusMetrics::with_registry(manager.clone().metrics.create_registry()?); diff --git a/conf/Dockerfile b/conf/Dockerfile index f9e78a5ca..32f6dc48c 100644 --- a/conf/Dockerfile +++ b/conf/Dockerfile @@ -62,7 +62,7 @@ LABEL stage=builder FROM scratch ENV RUST_BACKTRACE=full\ - RUST_LOG="error,$BINARY_NAME=info" + RUST_LOG="warn" COPY --from=builder-backend /opt/bin/$BINARY_NAME / COPY --from=builder-frontend /opt/dist/ /static diff --git a/conf/k8s/base/deployment.yaml b/conf/k8s/base/deployment.yaml index d0c792f1f..a4dd65d1f 100644 --- a/conf/k8s/base/deployment.yaml +++ b/conf/k8s/base/deployment.yaml @@ -5,19 +5,28 @@ metadata: spec: selector: matchLabels: - app: playground + app.kubernetes.io/component: http-server replicas: 1 template: + metadata: + labels: + app.kubernetes.io/component: http-server + annotations: + prometheus.io/scrape: "true" spec: serviceAccountName: default-service-account containers: - - image: gcr.io/substrateplayground-252112/jeluard/substrate-playground + - name: playground + image: gcr.io/substrateplayground-252112/jeluard/substrate-playground ports: - containerPort: 80 env: - - name: PLAYGROUND_HOST - value: ${PLAYGROUND_HOST} - - name: ROCKET_PORT - value: "${PLAYGROUND_PORT}" - - name: ROCKET_ENV - value: ${ENVIRONMENT} \ No newline at end of file + # See https://rocket.rs/v0.4/guide/configuration/ + - name: ROCKET_ENV + value: "staging" + - name: ROCKET_PORT + value: "80" + - name: ROCKET_LOG + value: "normal" + - name: ROCKET_ADDRESS + value: "0.0.0.0" \ No newline at end of file diff --git a/conf/k8s/base/ingress.yaml b/conf/k8s/base/ingress.yaml index bb48edc97..cd495abca 100644 --- a/conf/k8s/base/ingress.yaml +++ b/conf/k8s/base/ingress.yaml @@ -3,6 +3,7 @@ kind: Ingress metadata: name: ingress annotations: + kubernetes.io/ingress.global-static-ip-name: playground-staging kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/enable-cors: "true" nginx.ingress.kubernetes.io/cors-allow-methods: "GET, OPTIONS" diff --git a/conf/k8s/base/kustomization.yaml b/conf/k8s/base/kustomization.yaml index c2c6d35eb..735d46d7f 100644 --- a/conf/k8s/base/kustomization.yaml +++ b/conf/k8s/base/kustomization.yaml @@ -1,8 +1,11 @@ -commonLabels: - app: playground +#commonLabels: +# app.kubernetes.io/name: playground +# app.kubernetes.io/component: controller +# app.kubernetes.io/managed-by: kustomize resources: - cluster-role-binding.yaml - deployment.yaml - ingress.yaml + - nginx.yaml - service-account.yaml - service.yaml \ No newline at end of file diff --git a/conf/k8s/nginx.yaml b/conf/k8s/base/nginx.yaml similarity index 77% rename from conf/k8s/nginx.yaml rename to conf/k8s/base/nginx.yaml index 125d07523..988e6c915 100644 --- a/conf/k8s/nginx.yaml +++ b/conf/k8s/base/nginx.yaml @@ -1,9 +1,21 @@ -# From https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.27.1/deploy/static/mandatory.yaml +# See https://github.com/kubernetes/ingress-nginx/blob/master/docs/deploy/index.md#gce-gke +# Copied from https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.30.0/deploy/static/mandatory.yaml + +apiVersion: v1 +kind: Namespace +metadata: + name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + +--- kind: ConfigMap apiVersion: v1 metadata: - name: config + name: nginx-configuration + namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx @@ -12,7 +24,8 @@ metadata: kind: ConfigMap apiVersion: v1 metadata: - name: config-tcp-services + name: tcp-services + namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx @@ -21,15 +34,18 @@ metadata: kind: ConfigMap apiVersion: v1 metadata: - name: config-udp-services + name: udp-services + namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx + --- apiVersion: v1 kind: ServiceAccount metadata: - name: service-account + name: nginx-ingress-serviceaccount + namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx @@ -38,7 +54,7 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: - name: nginx-ingress-clusterrole-${K8S_NAMESPACE} + name: nginx-ingress-clusterrole labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx @@ -96,7 +112,8 @@ rules: apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role metadata: - name: role + name: nginx-ingress-role + namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx @@ -141,33 +158,35 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding metadata: name: nginx-ingress-role-nisa-binding + namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: role + name: nginx-ingress-role subjects: - kind: ServiceAccount - name: service-account - namespace: ${K8S_NAMESPACE} + name: nginx-ingress-serviceaccount + namespace: ingress-nginx + --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: - name: nginx-ingress-clusterrole-nisa-binding-${K8S_NAMESPACE} + name: nginx-ingress-clusterrole-nisa-binding labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: nginx-ingress-clusterrole-${K8S_NAMESPACE} + name: nginx-ingress-clusterrole subjects: - kind: ServiceAccount - name: service-account - namespace: ${K8S_NAMESPACE} + name: nginx-ingress-serviceaccount + namespace: ingress-nginx --- @@ -175,6 +194,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: nginx-ingress-controller + namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx @@ -195,17 +215,17 @@ spec: spec: # wait up to five minutes for the drain of connections terminationGracePeriodSeconds: 300 - serviceAccountName: service-account + serviceAccountName: nginx-ingress-serviceaccount nodeSelector: kubernetes.io/os: linux containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.27.1 + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0 args: - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/config - - --tcp-services-configmap=$(POD_NAMESPACE)/config-tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/config-udp-services + - --configmap=$(POD_NAMESPACE)/nginx-configuration + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - --publish-service=$(POD_NAMESPACE)/ingress-nginx - --annotations-prefix=nginx.ingress.kubernetes.io securityContext: @@ -257,36 +277,39 @@ spec: exec: command: - /wait-shutdown + --- - apiVersion: v1 - kind: LimitRange - metadata: - name: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - spec: - limits: - - default: - min: - memory: 90Mi - cpu: 100m - type: Container ---- -# From https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.27.1/deploy/static/provider/cloud-generic.yaml +apiVersion: v1 +kind: LimitRange +metadata: + name: ingress-nginx + namespace: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +spec: + limits: + - min: + memory: 90Mi + cpu: 100m + type: Container + +# From https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.30.0/deploy/static/provider/cloud-generic.yaml + +--- -kind: Service apiVersion: v1 +kind: Service metadata: - name: service + name: ingress-nginx + namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx spec: externalTrafficPolicy: Local type: LoadBalancer - loadBalancerIP: ${PLAYGROUND_STATIC_IP} selector: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx @@ -298,4 +321,4 @@ spec: - name: https port: 443 protocol: TCP - targetPort: https + targetPort: https \ No newline at end of file diff --git a/conf/k8s/base/nginx/config-tcp-services.yaml b/conf/k8s/base/nginx/config-tcp-services.yaml deleted file mode 100644 index 69dca33cb..000000000 --- a/conf/k8s/base/nginx/config-tcp-services.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: config-tcp-services - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx \ No newline at end of file diff --git a/conf/k8s/base/nginx/config-udp-services.yaml b/conf/k8s/base/nginx/config-udp-services.yaml deleted file mode 100644 index 6a1b12007..000000000 --- a/conf/k8s/base/nginx/config-udp-services.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: config-udp-services - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx \ No newline at end of file diff --git a/conf/k8s/base/nginx/config.yaml b/conf/k8s/base/nginx/config.yaml deleted file mode 100644 index 1d2fd2adf..000000000 --- a/conf/k8s/base/nginx/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: config - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx \ No newline at end of file diff --git a/conf/k8s/base/nginx/role.yaml b/conf/k8s/base/nginx/role.yaml deleted file mode 100644 index 53aed4ea7..000000000 --- a/conf/k8s/base/nginx/role.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: role - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get \ No newline at end of file diff --git a/conf/k8s/base/nginx/service-account.yaml b/conf/k8s/base/nginx/service-account.yaml deleted file mode 100644 index 1456610c7..000000000 --- a/conf/k8s/base/nginx/service-account.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: service-account - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx \ No newline at end of file diff --git a/conf/k8s/base/nginx/service.yaml b/conf/k8s/base/nginx/service.yaml deleted file mode 100644 index 9f845f442..000000000 --- a/conf/k8s/base/nginx/service.yaml +++ /dev/null @@ -1,23 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: nginx/service - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -spec: - externalTrafficPolicy: Local - type: LoadBalancer - loadBalancerIP: ${PLAYGROUND_STATIC_IP} - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ports: - - name: http - port: 80 - protocol: TCP - targetPort: http - - name: https - port: 443 - protocol: TCP - targetPort: https \ No newline at end of file diff --git a/conf/k8s/base/service.yaml b/conf/k8s/base/service.yaml index cd1ea309c..0cbfd7362 100644 --- a/conf/k8s/base/service.yaml +++ b/conf/k8s/base/service.yaml @@ -8,4 +8,4 @@ spec: - port: 80 targetPort: 80 selector: - app: deployment \ No newline at end of file + app.kubernetes.io/component: http-server \ No newline at end of file diff --git a/conf/k8s/deployment.yaml.tmpl b/conf/k8s/deployment.yaml.tmpl deleted file mode 100644 index 98a351b65..000000000 --- a/conf/k8s/deployment.yaml.tmpl +++ /dev/null @@ -1,98 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ${K8S_NAMESPACE} ---- -apiVersion: v1 -kind: ResourceQuota -metadata: - name: pod-quota -spec: - hard: - pods: "50" ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: default-service-account-${K8S_NAMESPACE} ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: default-playground-role -subjects: -- kind: ServiceAccount - name: default-service-account-${K8S_NAMESPACE} - namespace: ${K8S_NAMESPACE} -roleRef: - kind: ClusterRole - name: cluster-admin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: playground -spec: - selector: - matchLabels: - app: playground - replicas: 1 - template: - metadata: - labels: - app: playground - spec: - serviceAccountName: default-service-account-${K8S_NAMESPACE} - containers: - - name: playground - image: ${IMAGE} - ports: - - containerPort: ${PLAYGROUND_PORT} - env: - - name: ROCKET_PORT - value: "${PLAYGROUND_PORT}" - - name: ROCKET_ENV - value: ${ENVIRONMENT} ---- -apiVersion: v1 -kind: Service -metadata: - name: playground-http - labels: - app: playground -spec: - type: NodePort - ports: - - port: ${PLAYGROUND_PORT} - targetPort: ${PLAYGROUND_PORT} - selector: - app: playground ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: ingress - annotations: - kubernetes.io/ingress.class: "nginx" - nginx.ingress.kubernetes.io/enable-cors: "true" - nginx.ingress.kubernetes.io/cors-allow-methods: "GET, OPTIONS" - # nginx.ingress.kubernetes.io/configuration-snippet: | - # rewrite_log on; - # location ~ /front-end/.* { - # rewrite ^/front-end(/.*)?$ https://$host/$1 break; - # } - # nginx.ingress.kubernetes.io/cors-allow-origin: "https://${PLAYGROUND_HOST}" -spec: - tls: - - hosts: - - '*.${PLAYGROUND_HOST}' - secretName: playground-tls - rules: - - host: ${PLAYGROUND_HOST} - http: - paths: - - path: / - backend: - serviceName: playground-http - servicePort: ${PLAYGROUND_PORT} diff --git a/conf/k8s/overlays/dev/theia-images/template b/conf/k8s/overlays/dev/theia-images/template deleted file mode 100644 index 03e175b04..000000000 --- a/conf/k8s/overlays/dev/theia-images/template +++ /dev/null @@ -1 +0,0 @@ -jeluard/theia-substrate:latest \ No newline at end of file diff --git a/conf/k8s/overlays/production/custom-load-balancer-ip.yaml b/conf/k8s/overlays/production/custom-load-balancer-ip.yaml new file mode 100644 index 000000000..8f65f7141 --- /dev/null +++ b/conf/k8s/overlays/production/custom-load-balancer-ip.yaml @@ -0,0 +1,6 @@ +kind: Service +apiVersion: v1 +metadata: + name: ingress-nginx +spec: + loadBalancerIP: 35.202.224.243 \ No newline at end of file diff --git a/conf/k8s/overlays/production/ingress-patch.yaml b/conf/k8s/overlays/production/ingress-patch.yaml new file mode 100644 index 000000000..73165310a --- /dev/null +++ b/conf/k8s/overlays/production/ingress-patch.yaml @@ -0,0 +1,6 @@ +- op: replace + path: /spec/rules/0/host + value: playground.substrate.dev +- op: replace + path: /spec/tls/0/hosts/0 + value: '*.playground.substrate.dev' \ No newline at end of file diff --git a/conf/k8s/overlays/production/kustomization.yaml b/conf/k8s/overlays/production/kustomization.yaml index 667bc49e3..8f2470809 100644 --- a/conf/k8s/overlays/production/kustomization.yaml +++ b/conf/k8s/overlays/production/kustomization.yaml @@ -1,9 +1,30 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +commonLabels: + app.kubernetes.io/environment: production + app.kubernetes.io/version: "1.0" + bases: - - ../../base - - resource-quota.yaml +- ../../base + +resources: +- namespace.yaml +- resource-quota.yaml + +namespace: playground + +images: +- name: gcr.io/substrateplayground-252112/jeluard/substrate-playground + digest: sha256:b33808b4ecbb92fdad85aec909a1702600bebdd7aeb1f744c35f522d62be7a0b + +patches: +- custom-load-balancer-ip.yaml -patchesStrategicMerge: - - custom-env.yaml \ No newline at end of file +patchesJson6902: +- target: + group: extensions + version: v1beta1 + kind: Ingress + name: ingress + path: ingress-patch.yaml \ No newline at end of file diff --git a/conf/k8s/overlays/production/namespace.yaml b/conf/k8s/overlays/production/namespace.yaml new file mode 100644 index 000000000..fd897662d --- /dev/null +++ b/conf/k8s/overlays/production/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: playground \ No newline at end of file diff --git a/conf/k8s/overlays/production/theia-images/template b/conf/k8s/overlays/production/theia-images/template index 4ec004b96..2152b0af5 100644 --- a/conf/k8s/overlays/production/theia-images/template +++ b/conf/k8s/overlays/production/theia-images/template @@ -1 +1 @@ -gcr.io/substrateplayground-252112/jeluard/theia-substrate@sha256:db318d8f67d6cd9e5114f81b58eb8033f394bc2da478541a142f6aa6d3448ffd \ No newline at end of file +gcr.io/substrateplayground-252112/jeluard/theia-substrate@sha256:d0c004e8ac4c866e28cde82951aafd4c1d0a70debe119145d0ae4a908a16c4b3 \ No newline at end of file diff --git a/conf/k8s/overlays/staging/custom-load-balancer-ip.yaml b/conf/k8s/overlays/staging/custom-load-balancer-ip.yaml index 6ddefea99..8d043e9b0 100644 --- a/conf/k8s/overlays/staging/custom-load-balancer-ip.yaml +++ b/conf/k8s/overlays/staging/custom-load-balancer-ip.yaml @@ -1,6 +1,6 @@ kind: Service apiVersion: v1 metadata: - name: service + name: ingress-nginx spec: loadBalancerIP: 34.69.4.59 \ No newline at end of file diff --git a/conf/k8s/overlays/staging/kustomization.yaml b/conf/k8s/overlays/staging/kustomization.yaml index 521abe9de..e68245df6 100644 --- a/conf/k8s/overlays/staging/kustomization.yaml +++ b/conf/k8s/overlays/staging/kustomization.yaml @@ -1,15 +1,21 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +commonLabels: + app.kubernetes.io/environment: staging + app.kubernetes.io/version: "1.0" + bases: - ../../base +resources: +- namespace.yaml + namespace: playground-staging -nameSuffix: -staging images: - name: gcr.io/substrateplayground-252112/jeluard/substrate-playground - newTag: 3.4.5 + digest: sha256:b33808b4ecbb92fdad85aec909a1702600bebdd7aeb1f744c35f522d62be7a0b patches: - custom-load-balancer-ip.yaml diff --git a/conf/k8s/overlays/staging/namespace.yaml b/conf/k8s/overlays/staging/namespace.yaml new file mode 100644 index 000000000..bd4e86945 --- /dev/null +++ b/conf/k8s/overlays/staging/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: playground-staging \ No newline at end of file diff --git a/conf/k8s/overlays/staging/theia-images/template b/conf/k8s/overlays/staging/theia-images/template index 1c55d3298..2152b0af5 100644 --- a/conf/k8s/overlays/staging/theia-images/template +++ b/conf/k8s/overlays/staging/theia-images/template @@ -1 +1 @@ -gcr.io/substrateplayground-252112/jeluard/theia-substrate@sha256:0998ca43483db3f4f2a6b47150f047ec824d1f0200a188f581a8026fe7bbb34c \ No newline at end of file +gcr.io/substrateplayground-252112/jeluard/theia-substrate@sha256:d0c004e8ac4c866e28cde82951aafd4c1d0a70debe119145d0ae4a908a16c4b3 \ No newline at end of file