[Fixed #17] Moved to kustomize. Use dedicated cluster per environment

Makefile

@@ -94,7 +94,7 @@ k8s-gke-static-ip: k8s-assert
 
 # Deploy playground on kubernetes
 k8s-deploy-playground: k8s-assert
-	kubectl apply --validate=true --record -k conf/k8s/overlays/${ENVIRONMENT}
+	kubectl apply --record -k conf/k8s/overlays/${ENVIRONMENT}
 
 # Undeploy playground from kubernetes
 k8s-undeploy-playground: k8s-assert
@@ -106,4 +106,4 @@ k8s-undeploy-theia: k8s-assert
 
 # Creates or replaces the `images` config map from `conf/k8s/images/*.properties`
 k8s-update-images-config: k8s-assert
-	kubectl create configmap theia-images --namespace=${IDENTIFIER} --from-env-file=conf/k8s/images/${ENVIRONMENT}.properties --dry-run -o yaml | kubectl apply -f -
+	kubectl create configmap theia-images --namespace=${IDENTIFIER} --from-file=conf/k8s/overlays/${ENVIRONMENT}/theia-images/ --dry-run -o yaml | kubectl apply -f -

backend/src/kubernetes.rs

@@ -18,10 +18,12 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use std::{collections::BTreeMap, error::Error, time::SystemTime};
 use uuid::Uuid;
 
-const APP_LABEL: &str = "app";
-const APP_VALUE: &str = "theia-substrate";
-const USER_UUID_LABEL: &str = "user-uuid";
-const INSTANCE_UUID_LABEL: &str = "instance-uuid";
+const APP_LABEL: &str = "app.kubernetes.io/name";
+const APP_VALUE: &str = "playground";
+const COMPONENT_LABEL: &str = "app.kubernetes.io/component";
+const COMPONENT_VALUE: &str = "theia";
+const OWNER_LABEL: &str = "app.kubernetes.io/owner";
+const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
 const INGRESS_NAME: &str = "ingress";
 
 fn error_to_string<T: std::fmt::Display>(err: T) -> String {
@@ -45,17 +47,18 @@ async fn list_by_selector<K: Clone + DeserializeOwned + Meta>(
 fn create_pod(user_uuid: &str, instance_uuid: &str, image: &str) -> Pod {
     let mut labels = BTreeMap::new();
     labels.insert(APP_LABEL.to_string(), APP_VALUE.to_string());
-    labels.insert(USER_UUID_LABEL.to_string(), user_uuid.to_string());
-    labels.insert(INSTANCE_UUID_LABEL.to_string(), instance_uuid.to_string());
+    labels.insert(COMPONENT_LABEL.to_string(), COMPONENT_VALUE.to_string());
+    labels.insert(OWNER_LABEL.to_string(), user_uuid.to_string());
+    labels.insert(INSTANCE_LABEL.to_string(), instance_uuid.to_string());
     Pod {
         metadata: Some(ObjectMeta {
-            generate_name: Some(format!("{}-", APP_VALUE).to_string()),
+            generate_name: Some(format!("{}-", COMPONENT_VALUE).to_string()),
             labels: Some(labels),
             ..Default::default()
         }),
         spec: Some(PodSpec {
             containers: vec![Container {
-                name: format!("{}-container", APP_VALUE).to_string(),
+                name: format!("{}-container", COMPONENT_VALUE).to_string(),
                 image: Some(image.to_string()),
                 ..Default::default()
             }],
@@ -68,12 +71,13 @@ fn create_pod(user_uuid: &str, instance_uuid: &str, image: &str) -> Pod {
 fn create_service(instance_uuid: &str) -> Service {
     let mut labels = BTreeMap::new();
     labels.insert(APP_LABEL.to_string(), APP_VALUE.to_string());
-    labels.insert(INSTANCE_UUID_LABEL.to_string(), instance_uuid.to_string());
+    labels.insert(COMPONENT_LABEL.to_string(), COMPONENT_VALUE.to_string());
+    labels.insert(INSTANCE_LABEL.to_string(), instance_uuid.to_string());
     let mut selectors = BTreeMap::new();
-    selectors.insert(INSTANCE_UUID_LABEL.to_string(), instance_uuid.to_string());
+    selectors.insert(INSTANCE_LABEL.to_string(), instance_uuid.to_string());
     Service {
         metadata: Some(ObjectMeta {
-            generate_name: Some(format!("{}-http-", APP_VALUE).to_string()),
+            generate_name: Some(format!("{}-http-", COMPONENT_VALUE).to_string()),
             labels: Some(labels),
             ..Default::default()
         }),
@@ -149,7 +153,7 @@ pub struct Engine {
     namespace: String,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct InstanceDetails {
     pub user_uuid: String,
     pub instance_uuid: String,
@@ -207,12 +211,12 @@ impl Engine {
         Ok(Engine { host, namespace })
     }
 
-    fn user_selector(user_uuid: &str) -> String {
-        format!("{}={}", USER_UUID_LABEL, user_uuid)
+    fn owner_selector(user_uuid: &str) -> String {
+        format!("{}={}", OWNER_LABEL, user_uuid)
     }
 
     fn instance_selector(instance_uuid: &str) -> String {
-        format!("{}={}", INSTANCE_UUID_LABEL, instance_uuid)
+        format!("{}={}", INSTANCE_LABEL, instance_uuid)
     }
 
     fn pod_to_instance(self, pod: &Pod) -> Result<InstanceDetails, String> {
@@ -232,8 +236,8 @@ impl Engine {
             .as_ref()
             .and_then(|md| {
                 Some((
-                    md.labels.clone()?.get(USER_UUID_LABEL)?.to_string(),
-                    md.labels.clone()?.get(INSTANCE_UUID_LABEL)?.to_string(),
+                    md.labels.clone()?.get(OWNER_LABEL)?.to_string(),
+                    md.labels.clone()?.get(INSTANCE_LABEL)?.to_string(),
                 ))
             })
             .ok_or("Metadata unavailable")?;
@@ -264,13 +268,13 @@ impl Engine {
         let config = config().await?;
         let client = APIClient::new(config);
         let pod_api: Api<Pod> = Api::namespaced(client, &self.namespace);
-        let pods = list_by_selector(&pod_api, Engine::user_selector(user_uuid)).await?;
+        let pods = list_by_selector(&pod_api, Engine::owner_selector(user_uuid)).await?;
         let names: Vec<String> = pods
             .iter()
             .flat_map(|pod| {
                 pod.metadata
                     .as_ref()
-                    .and_then(|md| Some(md.labels.clone()?.get(INSTANCE_UUID_LABEL)?.to_string()))
+                    .and_then(|md| Some(md.labels.clone()?.get(INSTANCE_LABEL)?.to_string()))
             })
             .collect::<Vec<_>>();
 
@@ -282,14 +286,14 @@ impl Engine {
         let client = APIClient::new(config);
         let pod_api: Api<Pod> = Api::namespaced(client, &self.namespace);
         let pods =
-            list_by_selector(&pod_api, format!("{}={}", APP_LABEL, APP_VALUE).to_string()).await?;
+            list_by_selector(&pod_api, format!("{}={}", COMPONENT_LABEL, COMPONENT_VALUE).to_string()).await?;
         let names = pods
             .iter()
             .flat_map(|pod| {
                 self.clone()
                     .pod_to_instance(pod)
                     .ok()
-                    .map(|i| (/*i.user_uuid*/ "".to_string(), i))
+                    .map(|i| (i.clone().user_uuid, i))
             })
             .collect();
 

backend/src/main.rs

@@ -7,7 +7,7 @@ mod manager;
 mod metrics;
 
 use crate::manager::Manager;
-use rocket::{http::Method, routes};
+use rocket::{config::Environment, http::Method, routes};
 use rocket_contrib::serve::StaticFiles;
 use rocket_cors::{AllowedOrigins, CorsOptions};
 use rocket_prometheus::PrometheusMetrics;
@@ -24,7 +24,7 @@ pub struct Context {
 async fn main() -> Result<(), Box<dyn Error>> {
     // Initialize log configuration. Reads `RUST_LOG` if any, otherwise fallsback to `default`
     if env::var("RUST_LOG").is_err() {
-        env::set_var("RUST_LOG", "info,kube=info");
+        env::set_var("RUST_LOG", "warn");
     }
     env_logger::init();
 
@@ -40,6 +40,8 @@ async fn main() -> Result<(), Box<dyn Error>> {
     }
     .to_cors()?;
 
+    log::info!("Running in {:?} mode", Environment::active()?);
+
     let manager = Manager::new().await?;
     manager.clone().spawn_background_thread();
     let prometheus = PrometheusMetrics::with_registry(manager.clone().metrics.create_registry()?);

conf/Dockerfile

@@ -62,7 +62,7 @@ LABEL stage=builder
 FROM scratch
 
 ENV RUST_BACKTRACE=full\
-    RUST_LOG="error,$BINARY_NAME=info"
+    RUST_LOG="warn"
 
 COPY --from=builder-backend /opt/bin/$BINARY_NAME /
 COPY --from=builder-frontend /opt/dist/ /static

conf/k8s/base/deployment.yaml

@@ -5,19 +5,28 @@ metadata:
 spec:
   selector:
     matchLabels:
-      app: playground
+      app.kubernetes.io/component: http-server
   replicas: 1
   template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: http-server
+      annotations:
+        prometheus.io/scrape: "true"
     spec:
       serviceAccountName: default-service-account
       containers:
-      - image: gcr.io/substrateplayground-252112/jeluard/substrate-playground
+      - name: playground
+        image: gcr.io/substrateplayground-252112/jeluard/substrate-playground
         ports:
         - containerPort: 80
         env:
-        - name: PLAYGROUND_HOST
-          value: ${PLAYGROUND_HOST}
-        - name: ROCKET_PORT
-          value: "${PLAYGROUND_PORT}"
-        - name: ROCKET_ENV
-          value: ${ENVIRONMENT}
+          # See https://rocket.rs/v0.4/guide/configuration/
+          - name: ROCKET_ENV
+            value: "staging"
+          - name: ROCKET_PORT
+            value: "80"
+          - name: ROCKET_LOG
+            value: "normal"
+          - name: ROCKET_ADDRESS
+            value: "0.0.0.0"

conf/k8s/base/ingress.yaml

@@ -3,6 +3,7 @@ kind: Ingress
 metadata:
   name: ingress
   annotations:
+    kubernetes.io/ingress.global-static-ip-name: playground-staging
     kubernetes.io/ingress.class: "nginx"
     nginx.ingress.kubernetes.io/enable-cors: "true"
     nginx.ingress.kubernetes.io/cors-allow-methods: "GET, OPTIONS"

conf/k8s/base/kustomization.yaml

@@ -1,8 +1,11 @@
-commonLabels:
-    app: playground
+#commonLabels:
+#    app.kubernetes.io/name: playground
+#    app.kubernetes.io/component: controller
+#    app.kubernetes.io/managed-by: kustomize
 resources:
     - cluster-role-binding.yaml
     - deployment.yaml
     - ingress.yaml
+    - nginx.yaml
     - service-account.yaml
     - service.yaml