Refactor cook so that we generate both k8s and mesos pool names. (#1305)

* Split default pools into parallel k8s and mesos pools. Refactor preparation for doing a hybrid k8s+mesos simultaneously. config-k8s and config pass integration tests for mesos + k8s. * Added seed_k8s_pools command for generating k8s pools.
twosigma · Nov 11, 2019 · 70de0c5 · 70de0c5
1 parent fe619f3
commit 70de0c5
Show file tree

Hide file tree

Showing 8 changed files with 68 additions and 23 deletions.
diff --git a/integration/travis/run_integration.sh b/integration/travis/run_integration.sh
@@ -104,7 +104,7 @@ export COOK_KEYSTORE_PATH=${COOK_KEYSTORE_PATH}
 case "$COOK_POOLS" in
   on)
     echo "Pools are turned on for this run"
-    DEFAULT_POOL=gamma
+    DEFAULT_POOL=mesos-gamma
     cd ${SCHEDULER_DIR}
     lein exec -p datomic/data/seed_pools.clj ${COOK_DATOMIC_URI_1}
     ;;

diff --git a/scheduler/api-only-config.edn b/scheduler/api-only-config.edn
@@ -20,7 +20,7 @@
            :user-metrics-interval-seconds 60}
  :nrepl {:enabled? true
          :port 8888}
- :pools {:default "gamma"}
+ :pools {:default "mesos-gamma"}
  :port 12321
  :rate-limit {:user-limit-per-m 1000000}
  :unhandled-exceptions {:log-level :error}}
diff --git a/scheduler/bin/help-make-cluster b/scheduler/bin/help-make-cluster
@@ -20,7 +20,7 @@ gcloud="gcloud --project $PROJECT"
 echo "---- Building kubernetes cluster for project=$PROJECT zone=$ZONE clustername=$CLUSTERNAME kubeconfig=$COOK_KUBECONFIG"
 
 # This creates a cluster, with some nodes that are needed for kubernetes management.
-# Create 7 nodes, with three tainted for alpha, 3 untainted and 3 tainted with gamma pool.
+# Create 7 nodes, with three tainted for k8s-alpha, 3 untainted and 3 tainted with k8s-gamma pool.
 # The untainted ones are for GKE's own uses for its system pods. Also convenient to have around if I'm doing non-pool tests.
 # The second line of flags about upgrades and legacy endpoints is to suppress some warnings.
 echo "---- Creating new cluster (please wait 5 minutes)"
@@ -35,12 +35,12 @@ echo "---- Setting up gcloud credentials"
 $gcloud container clusters get-credentials "$CLUSTERNAME" --zone "$ZONE"
 
 # Make some node-pools --- this takes a while, but it helps guarantee that we have nodes with the appropriate cook-pool taints.
-echo "---- Making extra alpha and gamma nodepools for cook pools (please wait 5-10 minutes)"
-$gcloud container node-pools create cook-pool-gamma --zone "$ZONE" --cluster="$CLUSTERNAME" --disk-size=20gb --machine-type=g1-small \
-       --node-taints=cook-pool=gamma:NoSchedule \
+echo "---- Making extra k8s-alpha and k8s-gamma nodepools for cook pools (please wait 5-10 minutes)"
+$gcloud container node-pools create cook-pool-k8s-gamma --zone "$ZONE" --cluster="$CLUSTERNAME" --disk-size=20gb --machine-type=g1-small \
+       --node-taints=cook-pool=k8s-gamma:NoSchedule \
        --num-nodes=3
-$gcloud container node-pools create cook-pool-alpha --zone "$ZONE" --cluster="$CLUSTERNAME" --disk-size=20gb --machine-type=g1-small \
-       --node-taints=cook-pool=alpha:NoSchedule \
+$gcloud container node-pools create cook-pool-k8s-alpha --zone "$ZONE" --cluster="$CLUSTERNAME" --disk-size=20gb --machine-type=g1-small \
+       --node-taints=cook-pool=k8s-alpha:NoSchedule \
        --num-nodes=2
 
 echo "---- Setting up cook namespace in kubernetes"

diff --git a/scheduler/config.edn b/scheduler/config.edn
@@ -55,7 +55,7 @@
            :user-metrics-interval-seconds 60}
  :nrepl {:enabled? true
          :port #config/env-int "COOK_NREPL_PORT"}
- :pools {:default "gamma"}
+ :pools {:default "mesos-gamma"}
  :port #config/env-int "COOK_PORT"
  :ssl {:port #config/env-int "COOK_SSL_PORT"
        :keystore-path #config/env "COOK_KEYSTORE_PATH"

diff --git a/scheduler/datomic/data/seed_k8s_pools.clj b/scheduler/datomic/data/seed_k8s_pools.clj
@@ -0,0 +1,45 @@
+(ns data.seed-k8s-pools
+  (:require [cook.datomic :as datomic]
+            [cook.quota :as quota]
+            [datomic.api :as d]))
+
+(def uri (second *command-line-args*))
+(println "Datomic URI is" uri)
+
+(defn create-pool
+  [conn name state]
+  (println "Creating pool" name)
+  @(d/transact conn [{:db/id (d/tempid :db.part/user)
+                      :pool/name name
+                      :pool/purpose "This is a pool for testing purposes"
+                      :pool/state state
+                      :pool/dru-mode :pool.dru-mode/default}]))
+
+(defn pools
+  [db]
+  (->> (d/q '[:find [?p ...]
+              :in $ [?state ...]
+              :where
+              [?p :pool/state ?state]]
+            db [:pool.state/active :pool.state/inactive])
+       (map (partial d/entity db))
+       (map d/touch)))
+
+(try
+  (let [conn (datomic/create-connection {:settings {:mesos-datomic-uri uri}})]
+    (println "Connected to Datomic:" conn)
+    (create-pool conn "k8s-alpha" :pool.state/active)
+    (create-pool conn "k8s-beta" :pool.state/inactive)
+    (create-pool conn "k8s-gamma" :pool.state/active)
+    (create-pool conn "k8s-delta" :pool.state/inactive)
+    (quota/set-quota! conn "default" "k8s-alpha" "For quota-related testing." :cpus 8 :mem 1024)
+    (quota/set-quota! conn "default" "k8s-gamma" "For quota-related testing." :cpus 9 :mem 2048)
+    (println "Pools & Quotas:")
+    (run! (fn [{:keys [pool/name] :as p}]
+            (clojure.pprint/pprint p)
+            (clojure.pprint/pprint (quota/get-quota (d/db conn) "default" name)))
+          (pools (d/db conn)))
+    (System/exit 0))
+  (catch Throwable t
+    (println "Failed to seed pools:" t)
+    (System/exit 1)))
diff --git a/scheduler/datomic/data/seed_pools.clj b/scheduler/datomic/data/seed_pools.clj
@@ -28,12 +28,12 @@
 (try
   (let [conn (datomic/create-connection {:settings {:mesos-datomic-uri uri}})]
     (println "Connected to Datomic:" conn)
-    (create-pool conn "alpha" :pool.state/active)
-    (create-pool conn "beta" :pool.state/inactive)
-    (create-pool conn "gamma" :pool.state/active)
-    (create-pool conn "delta" :pool.state/inactive)
-    (quota/set-quota! conn "default" "alpha" "For quota-related testing." :cpus 8 :mem 1024)
-    (quota/set-quota! conn "default" "gamma" "For quota-related testing." :cpus 9 :mem 2048)
+    (create-pool conn "mesos-alpha" :pool.state/active)
+    (create-pool conn "mesos-beta" :pool.state/inactive)
+    (create-pool conn "mesos-gamma" :pool.state/active)
+    (create-pool conn "mesos-delta" :pool.state/inactive)
+    (quota/set-quota! conn "default" "mesos-alpha" "For quota-related testing." :cpus 8 :mem 1024)
+    (quota/set-quota! conn "default" "mesos-gamma" "For quota-related testing." :cpus 9 :mem 2048)
     (println "Pools & Quotas:")
     (run! (fn [{:keys [pool/name] :as p}]
             (clojure.pprint/pprint p)

diff --git a/scheduler/minimesosFile b/scheduler/minimesosFile
@@ -11,7 +11,7 @@ minimesos {
         imageTag = "1.0.0-0.1.0"
         loggingLevel = "# INHERIT FROM CLUSTER"
         portNumber = 5051
-        attributes = "cook-pool:gamma"
+        attributes = "cook-pool:mesos-gamma"
 
         resources {
             cpu {
@@ -57,7 +57,7 @@ minimesos {
         imageTag = "1.0.0-0.1.0"
         loggingLevel = "# INHERIT FROM CLUSTER"
         portNumber = 5051
-        attributes = "cook-pool:gamma"
+        attributes = "cook-pool:mesos-gamma"
 
         resources {
             cpu {
@@ -103,7 +103,7 @@ minimesos {
         imageTag = "1.0.0-0.1.0"
         loggingLevel = "# INHERIT FROM CLUSTER"
         portNumber = 5051
-        attributes = "cook-pool:alpha"
+        attributes = "cook-pool:mesos-alpha"
 
         resources {
             cpu {
@@ -149,7 +149,7 @@ minimesos {
         imageTag = "1.0.0-0.1.0"
         loggingLevel = "# INHERIT FROM CLUSTER"
         portNumber = 5051
-        attributes = "cook-pool:alpha"
+        attributes = "cook-pool:mesos-alpha"
 
         resources {
             cpu {

diff --git a/travis/minimesosFile b/travis/minimesosFile
@@ -11,7 +11,7 @@ minimesos {
         imageTag = "1.0.0-0.1.0"
         loggingLevel = "# INHERIT FROM CLUSTER"
         portNumber = 5051
-        attributes = "cook-pool:gamma"
+        attributes = "cook-pool:mesos-gamma"
 
         resources {
             cpu {
@@ -57,7 +57,7 @@ minimesos {
         imageTag = "1.0.0-0.1.0"
         loggingLevel = "# INHERIT FROM CLUSTER"
         portNumber = 5051
-        attributes = "cook-pool:gamma"
+        attributes = "cook-pool:mesos-gamma"
 
         resources {
             cpu {
@@ -103,7 +103,7 @@ minimesos {
         imageTag = "1.0.0-0.1.0"
         loggingLevel = "# INHERIT FROM CLUSTER"
         portNumber = 5051
-        attributes = "cook-pool:alpha"
+        attributes = "cook-pool:mesos-alpha"
 
         resources {
             cpu {
@@ -149,7 +149,7 @@ minimesos {
         imageTag = "1.0.0-0.1.0"
         loggingLevel = "# INHERIT FROM CLUSTER"
         portNumber = 5051
-        attributes = "cook-pool:alpha"
+        attributes = "cook-pool:mesos-alpha"
 
         resources {
             cpu {