Skip to content
Permalink
Browse files

Refactor cook so that we generate both k8s and mesos pool names. (#1305)

* Split default pools into parallel k8s and mesos pools.

Refactor preparation for doing a hybrid k8s+mesos simultaneously.
config-k8s and config pass integration tests for mesos + k8s.

* Added seed_k8s_pools command for generating k8s pools.
  • Loading branch information
scrosby authored and nsinkov committed Nov 11, 2019
1 parent fe619f3 commit 70de0c5a1bfcfe08d6a13221bd38a3c0efb5a942
@@ -104,7 +104,7 @@ export COOK_KEYSTORE_PATH=${COOK_KEYSTORE_PATH}
case "$COOK_POOLS" in
on)
echo "Pools are turned on for this run"
DEFAULT_POOL=gamma
DEFAULT_POOL=mesos-gamma
cd ${SCHEDULER_DIR}
lein exec -p datomic/data/seed_pools.clj ${COOK_DATOMIC_URI_1}
;;
@@ -20,7 +20,7 @@
:user-metrics-interval-seconds 60}
:nrepl {:enabled? true
:port 8888}
:pools {:default "gamma"}
:pools {:default "mesos-gamma"}
:port 12321
:rate-limit {:user-limit-per-m 1000000}
:unhandled-exceptions {:log-level :error}}
@@ -20,7 +20,7 @@ gcloud="gcloud --project $PROJECT"
echo "---- Building kubernetes cluster for project=$PROJECT zone=$ZONE clustername=$CLUSTERNAME kubeconfig=$COOK_KUBECONFIG"

# This creates a cluster, with some nodes that are needed for kubernetes management.
# Create 7 nodes, with three tainted for alpha, 3 untainted and 3 tainted with gamma pool.
# Create 7 nodes, with three tainted for k8s-alpha, 3 untainted and 3 tainted with k8s-gamma pool.
# The untainted ones are for GKE's own uses for its system pods. Also convenient to have around if I'm doing non-pool tests.
# The second line of flags about upgrades and legacy endpoints is to suppress some warnings.
echo "---- Creating new cluster (please wait 5 minutes)"
@@ -35,12 +35,12 @@ echo "---- Setting up gcloud credentials"
$gcloud container clusters get-credentials "$CLUSTERNAME" --zone "$ZONE"

# Make some node-pools --- this takes a while, but it helps guarantee that we have nodes with the appropriate cook-pool taints.
echo "---- Making extra alpha and gamma nodepools for cook pools (please wait 5-10 minutes)"
$gcloud container node-pools create cook-pool-gamma --zone "$ZONE" --cluster="$CLUSTERNAME" --disk-size=20gb --machine-type=g1-small \
--node-taints=cook-pool=gamma:NoSchedule \
echo "---- Making extra k8s-alpha and k8s-gamma nodepools for cook pools (please wait 5-10 minutes)"
$gcloud container node-pools create cook-pool-k8s-gamma --zone "$ZONE" --cluster="$CLUSTERNAME" --disk-size=20gb --machine-type=g1-small \
--node-taints=cook-pool=k8s-gamma:NoSchedule \
--num-nodes=3
$gcloud container node-pools create cook-pool-alpha --zone "$ZONE" --cluster="$CLUSTERNAME" --disk-size=20gb --machine-type=g1-small \
--node-taints=cook-pool=alpha:NoSchedule \
$gcloud container node-pools create cook-pool-k8s-alpha --zone "$ZONE" --cluster="$CLUSTERNAME" --disk-size=20gb --machine-type=g1-small \
--node-taints=cook-pool=k8s-alpha:NoSchedule \
--num-nodes=2

echo "---- Setting up cook namespace in kubernetes"
@@ -55,7 +55,7 @@
:user-metrics-interval-seconds 60}
:nrepl {:enabled? true
:port #config/env-int "COOK_NREPL_PORT"}
:pools {:default "gamma"}
:pools {:default "mesos-gamma"}
:port #config/env-int "COOK_PORT"
:ssl {:port #config/env-int "COOK_SSL_PORT"
:keystore-path #config/env "COOK_KEYSTORE_PATH"
@@ -0,0 +1,45 @@
(ns data.seed-k8s-pools
(:require [cook.datomic :as datomic]
[cook.quota :as quota]
[datomic.api :as d]))

(def uri (second *command-line-args*))
(println "Datomic URI is" uri)

(defn create-pool
[conn name state]
(println "Creating pool" name)
@(d/transact conn [{:db/id (d/tempid :db.part/user)
:pool/name name
:pool/purpose "This is a pool for testing purposes"
:pool/state state
:pool/dru-mode :pool.dru-mode/default}]))

(defn pools
[db]
(->> (d/q '[:find [?p ...]
:in $ [?state ...]
:where
[?p :pool/state ?state]]
db [:pool.state/active :pool.state/inactive])
(map (partial d/entity db))
(map d/touch)))

(try
(let [conn (datomic/create-connection {:settings {:mesos-datomic-uri uri}})]
(println "Connected to Datomic:" conn)
(create-pool conn "k8s-alpha" :pool.state/active)
(create-pool conn "k8s-beta" :pool.state/inactive)
(create-pool conn "k8s-gamma" :pool.state/active)
(create-pool conn "k8s-delta" :pool.state/inactive)
(quota/set-quota! conn "default" "k8s-alpha" "For quota-related testing." :cpus 8 :mem 1024)
(quota/set-quota! conn "default" "k8s-gamma" "For quota-related testing." :cpus 9 :mem 2048)
(println "Pools & Quotas:")
(run! (fn [{:keys [pool/name] :as p}]
(clojure.pprint/pprint p)
(clojure.pprint/pprint (quota/get-quota (d/db conn) "default" name)))
(pools (d/db conn)))
(System/exit 0))
(catch Throwable t
(println "Failed to seed pools:" t)
(System/exit 1)))
@@ -28,12 +28,12 @@
(try
(let [conn (datomic/create-connection {:settings {:mesos-datomic-uri uri}})]
(println "Connected to Datomic:" conn)
(create-pool conn "alpha" :pool.state/active)
(create-pool conn "beta" :pool.state/inactive)
(create-pool conn "gamma" :pool.state/active)
(create-pool conn "delta" :pool.state/inactive)
(quota/set-quota! conn "default" "alpha" "For quota-related testing." :cpus 8 :mem 1024)
(quota/set-quota! conn "default" "gamma" "For quota-related testing." :cpus 9 :mem 2048)
(create-pool conn "mesos-alpha" :pool.state/active)
(create-pool conn "mesos-beta" :pool.state/inactive)
(create-pool conn "mesos-gamma" :pool.state/active)
(create-pool conn "mesos-delta" :pool.state/inactive)
(quota/set-quota! conn "default" "mesos-alpha" "For quota-related testing." :cpus 8 :mem 1024)
(quota/set-quota! conn "default" "mesos-gamma" "For quota-related testing." :cpus 9 :mem 2048)
(println "Pools & Quotas:")
(run! (fn [{:keys [pool/name] :as p}]
(clojure.pprint/pprint p)
@@ -11,7 +11,7 @@ minimesos {
imageTag = "1.0.0-0.1.0"
loggingLevel = "# INHERIT FROM CLUSTER"
portNumber = 5051
attributes = "cook-pool:gamma"
attributes = "cook-pool:mesos-gamma"

resources {
cpu {
@@ -57,7 +57,7 @@ minimesos {
imageTag = "1.0.0-0.1.0"
loggingLevel = "# INHERIT FROM CLUSTER"
portNumber = 5051
attributes = "cook-pool:gamma"
attributes = "cook-pool:mesos-gamma"

resources {
cpu {
@@ -103,7 +103,7 @@ minimesos {
imageTag = "1.0.0-0.1.0"
loggingLevel = "# INHERIT FROM CLUSTER"
portNumber = 5051
attributes = "cook-pool:alpha"
attributes = "cook-pool:mesos-alpha"

resources {
cpu {
@@ -149,7 +149,7 @@ minimesos {
imageTag = "1.0.0-0.1.0"
loggingLevel = "# INHERIT FROM CLUSTER"
portNumber = 5051
attributes = "cook-pool:alpha"
attributes = "cook-pool:mesos-alpha"

resources {
cpu {
@@ -11,7 +11,7 @@ minimesos {
imageTag = "1.0.0-0.1.0"
loggingLevel = "# INHERIT FROM CLUSTER"
portNumber = 5051
attributes = "cook-pool:gamma"
attributes = "cook-pool:mesos-gamma"

resources {
cpu {
@@ -57,7 +57,7 @@ minimesos {
imageTag = "1.0.0-0.1.0"
loggingLevel = "# INHERIT FROM CLUSTER"
portNumber = 5051
attributes = "cook-pool:gamma"
attributes = "cook-pool:mesos-gamma"

resources {
cpu {
@@ -103,7 +103,7 @@ minimesos {
imageTag = "1.0.0-0.1.0"
loggingLevel = "# INHERIT FROM CLUSTER"
portNumber = 5051
attributes = "cook-pool:alpha"
attributes = "cook-pool:mesos-alpha"

resources {
cpu {
@@ -149,7 +149,7 @@ minimesos {
imageTag = "1.0.0-0.1.0"
loggingLevel = "# INHERIT FROM CLUSTER"
portNumber = 5051
attributes = "cook-pool:alpha"
attributes = "cook-pool:mesos-alpha"

resources {
cpu {

0 comments on commit 70de0c5

Please sign in to comment.
You can’t perform that action at this time.