From 7388496165b990dace4e32cb38f718ca8983ad95 Mon Sep 17 00:00:00 2001 From: Timo Kramer Date: Tue, 17 Mar 2020 20:40:09 +0100 Subject: [PATCH] Upgraded clj-docker-client to 0.5.1 - Refactored the interfacing functions to use data-driven client - Added different connections for different tasks to multiplex - Amended tests to adapt to new client functions - Refactored some naming to be more precise - Closes #68 - Refers #20 https://github.com/lispyclouds/clj-docker-client/issues/20 Most of the time it was sufficient to just change the function call to run docker/invoke with a map of options. The new client returns a message when there was a problem, so I needed to catch that to return a failjure. Tests had to be changed to keep up with the changed function calls. Problem when multiple connections on socket occur still persists. Currently it is solved by using a simple patch in the clj-docker-lib that is in use. There a new connection is made everytime invoke is called. That disobeys settings made by the user and connects bulleaded to the local socket on /var/run/docker.sock. Squashed commit of the following: commit e6b5a8bad70bbe331dcebbf1cc4fa053bbc6167e Author: Timo Kramer Date: Tue Mar 17 20:36:36 2020 +0100 using patched docker-lib to solve socket-problem commit 68b967459f9100f022b924cc5242dce09280c7d9 Author: Timo Kramer Date: Tue Mar 17 15:57:32 2020 +0100 Revert "trying different things to solve socket problems" This reverts commit 57a335e03e895f4a2835240b97780284e0090563. commit 57a335e03e895f4a2835240b97780284e0090563 Author: Timo Kramer Date: Sun Mar 8 15:36:52 2020 +0100 trying different things to solve socket problems commit 21ad5f90c8a019912d0f52ad337a81db1c15dd1f Author: Timo Kramer Date: Fri Mar 6 13:15:51 2020 +0100 Upgraded clj-docker-client to 0.5.1 - Refactored the interfacing functions to use data-driven client - Added different connections for different tasks to multiplex - Amended tests to adapt to new client functions - Refactored some naming to be more precise - Closes #68 Most of the time it was sufficient to just change the function call to run docker/invoke with a map of options. The new client returns a message when there was a problem, so I needed to catch that to return a failjure. Tests had to be changed to keep up with the changed function calls. Still there is a multiplex-problem when pulling multiple images. Squashed commit of the following: commit 64b6a322078559e8ec29a94423945dc09b8d7c0a Author: Timo Kramer Date: Thu Mar 5 20:23:18 2020 +0100 improved some tests commit 554fa1115972195131e171741f4f0ed1c176b426 Author: Timo Kramer Date: Tue Mar 3 15:06:08 2020 +0100 the errors were not printed correctly commit 0915bfe59776b2e67a41e3fb0e116d14c484a8ae Author: Timo Kramer Date: Tue Mar 3 15:05:45 2020 +0100 extra connection for a pull because experiencing Socket problems commit ab297d4f29e7ed809db5605850cfa769bed0b91c Author: Timo Kramer Date: Tue Mar 3 01:02:04 2020 +0100 less delay because pipeline is passed and cannot be stopped commit a6c5c3ab7aaa7f59129676e9ac904ae0d33db3ec Author: Timo Kramer Date: Mon Mar 2 23:32:13 2020 +0100 logging was not working because of special chars commit 5fd3b69f3cf0972d0bce5712167d9b06ea37b3d1 Author: Timo Kramer Date: Mon Mar 2 14:15:12 2020 +0100 TODO was superfluous commit 47fc68a40e6918ff60c20ff53236f8b738a966d8 Author: Timo Kramer Date: Mon Mar 2 12:24:55 2020 +0100 refactored run to start-container because more accurate commit 04d51c03a54a333b611b4c963658a6ffcca8a412 Author: Timo Kramer Date: Mon Mar 2 12:14:33 2020 +0100 cider-formatting commit 50ea2fc8bedb1db06f9df9802b5c78d9230eabbd Author: Timo Kramer Date: Mon Mar 2 12:11:08 2020 +0100 refactored function build to create-container because that is more accurate commit aec1d81a3347a2d7d876288bb6d9d8ddee549b5d Author: Timo Kramer Date: Mon Mar 2 11:34:15 2020 +0100 comment corrected commit f7eb5782ae4912a59ed9825287d9a4d788b614bc Author: Timo Kramer Date: Mon Mar 2 10:58:07 2020 +0100 stream to artifact-store works now commit 509cbc98f8a220979edb00d6a798cdf0312f61b1 Author: Timo Kramer Date: Fri Feb 28 18:32:24 2020 +0100 bootstrap_wendy.sh fails on extracting the tar commit aed0816dd469082e31959fd95d6ba7b221d3b240 Merge: 05e1d75 afa375b Author: Timo Kramer Date: Fri Feb 28 14:50:09 2020 +0100 Merge remote-tracking branch 'bob/master' into #68-upgrade-docker-client commit 05e1d7574162977d219e5a4d036f897ee8eade62 Author: Timo Kramer Date: Wed Feb 26 16:24:10 2020 +0100 refactored commit e9393274c754b991126ea0e743b3b6d29eaa3148 Author: Timo Kramer Date: Wed Feb 26 10:15:39 2020 +0100 mistake in health-check was always returning unhealthy or errored out on connection issue commit f9577df9dbd6a41fe370e4e1a20382d1ea24a37d Author: Timo Kramer Date: Tue Feb 25 15:38:05 2020 +0100 fixed malformed return of commit-image function commit 9d05e0d0ac7aafed8b4c27ba5218e15b68430477 Author: Timo Kramer Date: Tue Feb 25 12:41:14 2020 +0100 took functions to format CMD and ENV for passing to daemon commit 1217e313fcb2e3c909158ac660e254c9be1cb840 Author: Timo Kramer Date: Mon Feb 24 18:50:08 2020 +0100 TODO added commit 5c33c056b113303d6df9a52f5bd0c4452e923598 Author: Timo Kramer Date: Mon Feb 24 14:05:13 2020 +0100 log-streaming implemented commit bef9118a3dde1eb719409ce441135e31650e065a Author: Timo Kramer Date: Sun Feb 23 15:26:17 2020 +0100 updated docker-client commit 1bfddf12dbe13a4a7e5eb4be61d1c74ea08a726b Author: Timo Kramer Date: Sun Feb 23 15:26:03 2020 +0100 timeouts added for docker connection commit 180262acadd1e9a6142d5364a56fb8ef7e2fe2b2 Author: Timo Kramer Date: Sun Feb 23 15:25:42 2020 +0100 bug filed for clj-docker on timeout on putcontainerarchive commit 4298a4f7167ecdd25676f330b09a7dc9d278c4b4 Author: Timo Kramer Date: Tue Feb 18 17:15:36 2020 +0100 garbage collection repaired commit 2036b96d0cfc830ce6055259341978eaaea56290 Author: Timo Kramer Date: Tue Feb 18 17:15:12 2020 +0100 refactored execution internals commit 5b3b25b303fb0d091be91ed61275536e0d5afc4f Author: Timo Kramer Date: Tue Feb 18 14:06:07 2020 +0100 fixed some tests commit a550949e651c77c85084f7a4ec429dd2317a7836 Author: Timo Kramer Date: Tue Feb 18 11:36:02 2020 +0100 pipeline internals adapted commit fbf8a98ab802230dc1258964ab90e0f7d955f88e Author: Timo Kramer Date: Mon Feb 17 17:55:39 2020 +0100 refactored commit-image into execution/internals and adapted pipeline/internals next-step to new clj-docker-client commit e7084b3db88e8262421b3ea426ed8f7fc65174b2 Author: Timo Kramer Date: Sat Feb 15 18:21:15 2020 +0100 failing test; dont know how to mock file commit 44acdb9c70d51f67dd82a7015d835831312f3aae Author: Timo Kramer Date: Sat Feb 15 18:20:20 2020 +0100 made commit a state commit d9e003f721dc59e3f792a368f00c8a7ab31c40ac Author: Timo Kramer Date: Fri Feb 14 16:06:23 2020 +0100 resource internals adapted commit 74baf389b5d648a23f3c298dbe61dceff85b2ec2 Author: Timo Kramer Date: Mon Feb 10 16:01:08 2020 +0100 artifact/core adapted commit 6b81cce92b510b9d82930aa38a2a988c3a175030 Author: Timo Kramer Date: Mon Feb 10 12:44:52 2020 +0100 health check adapted commit 5c9f70b1f1f1df9dea345da112fc7eb27116fe05 Author: Timo Kramer Date: Thu Feb 6 17:50:31 2020 +0100 run function using new clj-docker commit 448df78fede46626272b697dc00ffbeb09921b18 Author: Timo Kramer Date: Thu Feb 6 16:56:54 2020 +0100 improved build to use failjure return commit 1f89b7595a7b9dfdc2ca505af437dc1f86177210 Author: Timo Kramer Date: Thu Feb 6 16:22:16 2020 +0100 status-of function now using new clj-docker lib commit 8de927372d9ca1aa0f033027fd5042cfc7c7ae58 Author: Timo Kramer Date: Thu Feb 6 15:06:03 2020 +0100 validation for image name improved and container build implemented commit b8eda9d2af00139058f2be756266e4aafe28dae3 Author: Timo Kramer Date: Tue Feb 4 15:01:02 2020 +0100 log-and-fail to avoid repetition commit d38ff6c8e5a674c03a12939ab0d8e8b8bb698874 Author: Timo Kramer Date: Sun Feb 2 18:36:38 2020 +0100 pull image function refactored commit c7a4c662b639b374ebd1158ccdd13020fc22d639 Author: Timo Kramer Date: Thu Jan 30 16:05:27 2020 +0100 switched to states commit 6d51bd2799eee2568522c3ac7f6e9ab0eac8625c Author: Timo Kramer Date: Wed Jan 29 13:16:42 2020 +0100 kill-container commit d944ebc62827e93647e53d496c4aa7dc223e8e5d Author: Timo Kramer Date: Wed Jan 29 10:51:55 2020 +0100 first function transitioned to clj-docker 0.4.0 --- build.boot | 2 +- integration-tests/bob-tests.strest.yaml | 15 +- src/bob/api/health.clj | 32 ++- src/bob/artifact/core.clj | 20 +- src/bob/execution/internals.clj | 326 +++++++++++++++++------- src/bob/pipeline/internals.clj | 154 +++++------ src/bob/resource/internals.clj | 97 ++++--- src/bob/states.clj | 17 +- src/bob/util.clj | 69 +++++ test/bob/api/health_test.clj | 12 +- test/bob/artifact/core_test.clj | 17 +- test/bob/execution/internals_test.clj | 181 +++++++++---- test/bob/pipeline/internals_test.clj | 125 ++++----- test/bob/resource/internals_test.clj | 19 ++ test/bob/util_test.clj | 22 ++ 15 files changed, 755 insertions(+), 353 deletions(-) diff --git a/build.boot b/build.boot index d96ba5ef..9c9f4f83 100644 --- a/build.boot +++ b/build.boot @@ -36,7 +36,7 @@ [com.layerware/hugsql "0.5.1"] [metosin/compojure-api "2.0.0-alpha30"] [prismatic/schema "1.1.12"] - [lispyclouds/clj-docker-client "0.3.2"] + [timokramer/clj-docker-client "0.5.2-SNAPSHOT"] [mount "0.1.16"] [environ "1.1.0"] [com.impossibl.pgjdbc-ng/pgjdbc-ng "0.8.3"] diff --git a/integration-tests/bob-tests.strest.yaml b/integration-tests/bob-tests.strest.yaml index debb4b51..5abfa43e 100644 --- a/integration-tests/bob-tests.strest.yaml +++ b/integration-tests/bob-tests.strest.yaml @@ -213,11 +213,22 @@ requests: - jsonpath: "content.message" expect: "Ok" + secondPipelineStatusRunning: + request: + url: "http://bob:7777/api/pipelines/status/groups/dev/names/test1/number/1" + method: "GET" + delay: 2000 + validate: + - jsonpath: "status" + expect: 200 + - jsonpath: "content.message" + expect: "running" + pipelineStop: request: url: "http://bob:7777/api/pipelines/stop/groups/dev/names/test1/number/1" method: "POST" - delay: 10000 + delay: 2000 validate: - jsonpath: "status" expect: 200 @@ -299,7 +310,7 @@ requests: - jsonpath: "content.message" expect: "passed" - secondPipelineStatus: + secondPipelineStatusStopped: request: url: "http://bob:7777/api/pipelines/status/groups/dev/names/test1/number/1" method: "GET" diff --git a/src/bob/api/health.clj b/src/bob/api/health.clj index e05415e9..0b9d36dd 100644 --- a/src/bob/api/health.clj +++ b/src/bob/api/health.clj @@ -41,21 +41,27 @@ "Check the systems we depend upon. Returns nil if everything is alright, else returns a sequence of strings naming the failing systems." [] - (let [docker (when (f/failed? (f/try* (docker/ping states/docker-conn))) - ["Docker"]) - postgres (when (f/failed? (f/try* (db-health-check states/db))) - ["Postgres"]) - extsys (when (nil? postgres) - (ping-external-systems))] + (let [ping-client (docker/client {:category :_ping + :conn (docker/connect {:uri "unix:///var/run/docker.sock" + :connect-timeout 1000 + :read-timeout 2000 + :write-timeout 2000 + :call-timeout 3000})}) + docker (when (not (= "OK" (docker/invoke ping-client {:op :SystemPing}))) + ["Docker"]) + postgres (when (f/failed? (f/try* (db-health-check states/db))) + ["Postgres"]) + extsys (when (nil? postgres) + (ping-external-systems))] (filter some? (concat docker postgres extsys)))) (defn respond-to-health-check "Endpoint for answering a health check" [] (d/let-flow [failures (health-check)] - (if (empty? failures) - (u/respond "Yes we can! \uD83D\uDD28 \uD83D\uDD28") - (u/service-unavailable (str "Health check failed: " (clojure.string/join " and " failures) " not healthy"))))) + (if (empty? failures) + (u/respond "Yes we can! \uD83D\uDD28 \uD83D\uDD28") + (u/service-unavailable (str "Health check failed: " (clojure.string/join " and " failures) " not healthy"))))) (defn log-health-check "Logs if any of the subsystems is unhealthy." @@ -83,6 +89,12 @@ (a/close! heartbeat)) (comment + (docker/categories) + (def ping (docker/client {:category :_ping :conn states/docker-conn})) + (docker/ops (docker/client {:category :_ping :conn states/docker-conn})) + (docker/invoke ping {:op :SystemPing}) + (f/failed? (f/try* (docker/invoke ping {:op :SystemPing}))) + (when (nil? nil) (concat (get-artifact-stores states/db) (get-external-resources states/db))) (map #(when (f/failed? (f/try* @(http/get (clojure.string/join "/" [(:url %) "ping"]) {:throw-exceptions false}))) (:name %)) @@ -90,7 +102,7 @@ (ping-external-systems) - (str "Health check failed: " (clojure.string/join " and " (health-check)) " not healthy") + (health-check) (log-health-check) diff --git a/src/bob/artifact/core.clj b/src/bob/artifact/core.clj index 842d7829..4e7be0b8 100644 --- a/src/bob/artifact/core.clj +++ b/src/bob/artifact/core.clj @@ -22,6 +22,7 @@ [taoensso.timbre :as log] [bob.util :as u] [bob.states :as states] + [bob.execution.internals :as int] [bob.artifact.db :as db])) (defn register-artifact-store @@ -95,7 +96,11 @@ Returns a Failure object if failed." [group name number artifact run-id path store-name] (if-let [{url :url} (db/get-artifact-store states/db {:name store-name})] - (f/try-all [stream (docker/stream-path states/docker-conn run-id path) + (f/try-all [_ (log/debugf "Streaming from container with id %s and path %s" run-id path) + stream (docker/invoke states/containers {:op :ContainerArchive + :params {:id run-id + :path path} + :as :stream}) upload-url (clojure.string/join "/" [url "bob_artifact" @@ -114,13 +119,22 @@ "Ok" (f/when-failed [err] (log/errorf "Error in uploading artifact: %s" (f/message err)) - (f/try* (docker/rm states/docker-conn run-id)) + (f/try* (int/delete-container run-id :force)) err)) (do (log/error "Error locating Artifact Store") - (f/try* (docker/rm states/docker-conn run-id)) + (f/try* (int/delete-container run-id :force)) (f/fail "No such artifact store registered")))) (comment + (docker/ops (docker/client {:category :containers :conn states/docker-conn})) + (let [foo (docker/invoke states/containers {:op :ContainerArchive + :params + {:id "clever_swartz" + :path "/root/my-source/target/default+uberjar/wendy"} + :as :stream})] + @(http/post "http://localhost:8001/bob_artifact/wendy/build/3/wendy" {:multipart [{:name "data" :content foo}]})) + (upload-artifact "dev" "test" 1 "poetry.lock" "0be7b883382b" "/opt" "s3") + (db/register-artifact-store states/db {:name "s3" :url "http://localhost:8001"}) diff --git a/src/bob/execution/internals.clj b/src/bob/execution/internals.clj index 59fb1b5c..8cfa300a 100644 --- a/src/bob/execution/internals.clj +++ b/src/bob/execution/internals.clj @@ -1,20 +1,21 @@ ; This file is part of Bob. ; ; Bob is free software: you can redistribute it and/or modify -; it under the terms of the GNU Affero General Public License as published by +; it under the terms of the GNU General Public License as published by ; the Free Software Foundation, either version 3 of the License, or ; (at your option) any later version. ; ; Bob is distributed in the hope that it will be useful, ; but WITHOUT ANY WARRANTY; without even the implied warranty of ; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -; GNU Affero General Public License for more details. +; GNU General Public License for more details. ; -; You should have received a copy of the GNU Affero General Public License +; You should have received a copy of the GNU General Public License ; along with Bob. If not, see . (ns bob.execution.internals - (:require [failjure.core :as f] + (:require [clojure.string :as cs] + [failjure.core :as f] [clj-docker-client.core :as docker] [taoensso.timbre :as log] [bob.util :as u] @@ -24,105 +25,246 @@ "Checks if an image is present locally. Returns the name or the error if any." [name] - (let [result (f/try* (filter #(= (:RepoTags %) [name]) - (docker/image-ls states/docker-conn)))] - (if (or (f/failed? result) (zero? (count result))) - (f/fail "Failed to find %s" name) - name))) + (let [result (filter #(= (:RepoTags %) [name]) + (docker/invoke states/images {:op :ImageList}))] + (if (zero? (count result)) + (u/log-and-fail "Failed to find" name "locally") + (do (log/debugf "Found image locally: %s" name) + name)))) -(defn kill-container - "Kills a running container using SIGKILL. - Returns the name or the error if any." - [name] - (if (f/failed? (f/try* (docker/kill states/docker-conn name))) - (do (log/errorf "Error killing container %s" name) - (f/fail "Could not kill %s" name)) - name)) - -(defn pull +(defn pull-image "Pulls in an image if it's not present locally. Returns the name or the error if any." [name] - (if (and (f/failed? (has-image name)) - (f/failed? (f/try* (do (log/debugf "Pulling image: %s" name) - (docker/pull states/docker-conn name) - (log/debugf "Pulled image: %s" name))))) - (do (log/errorf "Could not pull image: %s" name) - (f/fail "Cannot pull %s" name)) - name)) - -(defn build - "Builds a container. - Takes the base image and the entry point command. + (let [validate-name (fn [name] + (let [split-name (cs/split name #":")] + (if (= 2 (count split-name)) + {:repo (first split-name) + :tag (second split-name)} + (u/log-and-fail "Please provide a repository and a tag as image name:" split-name)))) + pull-invoke (fn [split-name] + (let [{:keys [repo tag]} split-name + _ (log/debugf "Pulling image with repo %s and tag %s" repo tag) + result (docker/invoke states/images {:op :ImageCreate + :params {:fromImage repo + :tag tag}})] + (if (contains? result :message) + (f/fail "Could not pull image %s:%s" repo tag) + result)))] + (if (and (f/failed? (has-image name)) + (f/failed? (pull-invoke (validate-name name)))) + (u/log-and-fail "Could not pull image:" name) + (do (log/debugf "Successfully pulled image: %s" name) + name)))) + +(defn commit-image + "Create a new image from a container + Defaults to repo containing of container-id a timestamp + and tag latest + + Returns image identifier" + [container-id cmd] + (let [repo (format "%s/%d" container-id (System/currentTimeMillis)) + result (docker/invoke states/commit {:op :ImageCommit + :params {:container container-id + :repo repo + :tag "latest" + :containerConfig {:Cmd [cmd]}}})] + (if (contains? result :message) + (u/log-and-fail "Could not commit image:" (:message result)) + (:Id result)))) + +(defn delete-image + "Remove an image, along with any untagged parent images that were referenced by that image. + Takes image id or repo-tag-combination, latest is appended if no tag is provided. + + Returns id of deleted image." + [image] + (let [_ (log/debugf "Deleting image with id %s" image) + result (docker/invoke states/images {:op :ImageDelete + :params {:name image}})] + (if (contains? result :message) + (u/log-and-fail "Could not delete image:" (:message result)) + result))) + +(defn create-container + "Creates a container. + Takes the base image and optionally the step and evars. Returns the id of the built container." - [image step evars] - (let [resource (:needs_resource step) - working-dir (when resource (str "/root/" resource)) - cmd (:cmd step)] - (log/debugf "Creating new container with: - image: %s - entry point: %s - environment: %s - working dir: %s" - image - cmd - evars - working-dir) - (f/try* (docker/create states/docker-conn - image - cmd - evars - {} - working-dir)))) + ([image] (let [_ (log/debugf "Creating new container with: + image: %s" image) + result (docker/invoke states/containers {:op :ContainerCreate + :params {:body {:Image image}}})] + (if (contains? result :message) + (u/log-and-fail "Could not build container with image:" (:message result)) + (:Id result)))) + ([image step] (create-container image step {})) + ([image step evars] (let [resource (:needs_resource step) + working-dir (when resource (str "/root/" resource)) + cmd (u/sh-tokenize! (:cmd step)) + formatted-evars (u/format-env-vars evars) + _ (log/debugf "Creating new container with: + image: %s + cmd: %s + evars: %s + working-dir: %s" + image + cmd + (vec formatted-evars) + working-dir) + result (docker/invoke states/containers {:op :ContainerCreate + :params {:body {:Image image + :Cmd cmd + :Env formatted-evars + :WorkingDir working-dir}}})] + (if (contains? result :message) + (u/log-and-fail "Could not build container with image:" (:message result)) + (:Id result))))) (defn status-of "Returns the status of a container by id." [^String id] - (let [result (f/try* (docker/container-state states/docker-conn id))] - (if (f/failed? result) - (do (log/errorf "Could not fetch container status: %s" - (f/message result)) - result) - {:running? (:Running result) - :exit-code (:ExitCode result)}))) - -(defn run - "Synchronously starts up a previously built container. - Returns the id when complete or and error in case on non-zero exit." + (let [result (docker/invoke states/containers {:op :ContainerInspect + :params {:id id}}) + running? (-> result :State :Running) + exit-code (-> result :State :ExitCode)] + (if (contains? result :message) + (u/log-and-fail (format "Could not fetch container status: %s" (:message result))) + {:running? running? :exit-code exit-code}))) + +(defn logs-live + "Get stdout and stderr logs from a container." + [id reaction-fn] + (let [log-client (docker/client {:category :containers + :conn (docker/connect {:uri "unix:///var/run/docker.sock" + :connect-timeout 1000 + :read-timeout 2000 + :write-timeout 2000 + :call-timeout 30000})}) + log-stream (docker/invoke log-client {:op :ContainerLogs + :params + {:id id + :follow true + :stdout true} + :as :stream})] + (future + (with-open [rdr (clojure.java.io/reader log-stream)] + (loop [r (java.io.BufferedReader. rdr)] + (when-let [line (.readLine r)] + (let [log-string (clojure.string/replace-first line #"^\W+" "")] + (when (not (cs/blank? log-string)) (reaction-fn log-string)) + (recur r)))))))) + +(defn start-container + "Synchronously starts up a previously built container by id. + Takes container-id and run-id which is the id of the build. + + Returns the id when complete or an error in case on non-zero exit." [id run-id] (f/try-all [_ (log/debugf "Starting container %s" id) - _ (docker/start states/docker-conn id) + _ (docker/invoke states/containers {:op :ContainerStart :params {:id id}}) _ (log/debugf "Attaching to container %s for logs" id) - _ (docker/logs-live states/docker-conn - id - #(u/log-to-db % run-id)) - status (-> (docker/inspect states/docker-conn id) - :State - :ExitCode)] - (if (zero? status) - (u/format-id id) - (do (log/debugf "Container %s exited with non-zero status: %s" - id - status) - (f/fail "Abnormal exit."))) - (f/when-failed [err] - (log/errorf "Error in running container %s: %s" - id - (f/message err)) - err))) + _ (logs-live id #(u/log-to-db % run-id)) + status (:StatusCode (docker/invoke states/containers {:op :ContainerWait :params {:id id}}))] + (do (f/when-failed [err] + (u/log-and-fail "Error in running container" + (str id ":") + (f/message err))) + (if (zero? status) + (u/format-id id) + (u/log-and-fail "Container with id" + id + "exited with non-zero status:" + status))))) + +(defn kill-container + "Kills a running container using SIGKILL. + Returns the name or the error if any." + [name] + (let [result (docker/invoke states/containers {:op :ContainerKill :params {:id name}})] + (if (cs/blank? result) + name + (u/log-and-fail "Could not kill" name)))) + +(defn delete-container + "Removes container by id. Takes optional keyword argument :force to force-remove container. + Defaults to force-remove. + + Returns a Failure object if failed." + ([id] + (delete-container id :force)) + ([id force-flag] + (let [id (str id) + force-flag (= force-flag :force) + _ (log/debugf "Deleting container with id %s" id) + result (docker/invoke states/containers + {:op :ContainerDelete + :params {:id id + :force force-flag}})] + (if-let [message (get result :message)] + (u/log-and-fail "Could not delete container:" message) + result)))) (comment - (build "busybox:musl" - {:needs_resource "source" - :cmd "ls"} - {}) - (log/infof "Creating new container with: - image: %s - entry point: %s - env vars: %s - working dir: %s" - "busybox:musl" - "echo Hello" - {:k1 "v1" - :k2 "v2"} - "/root")) + (docker/ops states/images) + (docker/ops states/containers) + (docker/doc states/images :ImageCreate) + (docker/invoke states/images {:op :ImageCreate :params {:fromImage "clojure" :tag "latest"}}) + (docker/invoke states/images {:op :ImageDelete :params {:name "clojure:latest"}}) + (def myid (:Id (docker/invoke states/containers {:op :ContainerCreate + :params {:body {:Image "oracle/graalvm-ce:19.3.0" + :Cmd "sh -c 'touch test.txt && echo $PATH >> test.txt'" + :Env "" + :WorkingDir nil}}}))) + (docker/invoke states/containers {:op :ContainerStart :params {:id myid}}) + (:StatusCode (docker/invoke states/containers {:op :ContainerWait :params {:id myid}})) + + (create-container "bobcd/bob:latest" + {:needs_resource "source" + :cmd "ls"}) + + (defn validate-name [name] + (let [split-name (cs/split name #":")] + (if (= 2 (count split-name)) + {:repo (first split-name) + :tag (second split-name)} + (u/log-and-fail "Please provide a repository and a tag as image name:" split-name)))) + + (defn pull-invoke [split-name] + (let [{:keys [repo tag]} split-name + _ (log/debugf "Pulling image with repo %s and tag %s" repo tag) + result (docker/invoke states/images {:op :ImageCreate + :params {:fromImage repo + :tag tag}})] + (if (contains? result :message) + (f/fail "Could not pull image %s:%s" repo tag) + result))) + + (def name "bobcd/bob:latest") + (has-image name) + (validate-name name) + (pull-invoke (validate-name name)) + (pull-invoke {:repo "oracle/graalvm-ce" :tags "19.3.0"}) + (pull-image "docker.io/bobcd/bob:latest") + (if (and (f/failed? (has-image name)) + (f/failed? (pull-invoke (validate-name name)))) + (u/log-and-fail "Could not pull image:" name) + (do (log/debugf "Successfully pulled image: %s" name) + name)) + + (docker/invoke states/images {:op :ImageCreate + :params {:fromImage "alpine" + :tag "3.11.3"}}) + (:ExitCode (:State (docker/invoke states/containers {:op :ContainerInspect :params {:id "82edce1bfdea"}}))) + + (docker/invoke states/containers {:op :ContainerStart :params {:id "interesting_swirles"}}) + (:StatusCode (docker/invoke states/containers {:op :ContainerWait :params {:id "interesting_swirles"}})) + + (docker/invoke states/containers {:op :ContainerDelete :params {:id "16a4b4e860b4" :force true}}) + (delete-container "3c12417d5a7e" true) + + (docker/invoke states/images {:op :ImageDelete + :params {:name "clojure:latest"}}) + + (delete-image "clojure") + (clojure.string/replace-first "bin usr" #"^\W+" "")) diff --git a/src/bob/pipeline/internals.clj b/src/bob/pipeline/internals.clj index a2dbe922..015e2b78 100644 --- a/src/bob/pipeline/internals.clj +++ b/src/bob/pipeline/internals.clj @@ -19,12 +19,12 @@ [taoensso.timbre :as log] [cheshire.core :as json] [mount.core :as m] - [bob.execution.internals :as e] [bob.util :as u] - [bob.resource.core :as r] - [bob.artifact.core :as artifact] [bob.states :as states] + [bob.artifact.core :as artifact] + [bob.execution.internals :as e] [bob.pipeline.db :as db] + [bob.resource.core :as r] [bob.resource.db :as rdb]) (:import (com.impossibl.postgres.jdbc PGDataSource) (com.impossibl.postgres.api.jdbc PGNotificationListener @@ -45,11 +45,12 @@ (if (nil? new-images) (list image) new-images)))) (defn gc-images + "garbage-collect images by run-id." [run-id] - (log/debugf "Deleting all images for run %s" run-id) - (f/try* (run! #(docker/image-rm states/docker-conn %) - (get @images-produced run-id))) - (swap! images-produced dissoc run-id)) + (let [_ (log/debugf "Deleting all images for run %s" run-id) + result (f/try* (run! #(e/delete-image %) + (get @images-produced run-id)))] + (swap! images-produced dissoc run-id))) (defn update-pid "Sets the current container id in the runs tables. @@ -62,10 +63,10 @@ _ (db/update-runs states/db {:pid pid :id run-id})] - pid - (f/when-failed [err] - (log/errorf "Failed to update pid: %s" (f/message err)) - err))) + pid + (f/when-failed [err] + (log/errorf "Failed to update pid: %s" (f/message err)) + err))) (defn resourceful-step "Create a resource mounted image for a step if it needs it." @@ -87,15 +88,12 @@ Returns the new container id or errors if any." [build-state step evars pipeline run-id] (f/try-all [_ (log/debugf "Committing container: %s" (:id build-state)) - image (docker/commit-container - states/docker-conn - (:id build-state) - (format "%s/%d" (:id build-state) (System/currentTimeMillis)) - "latest" - (:cmd step)) + image (e/commit-image + (:id build-state) + (:cmd step)) _ (mark-image-for-gc image run-id) _ (log/debug "Removing commited container") - _ (docker/rm states/docker-conn (:id build-state)) + _ (e/delete-container (:id build-state)) resource (:needs_resource step) mounted (:mounted build-state) mount-needed? (if (nil? resource) @@ -105,15 +103,15 @@ (resourceful-step step pipeline image run-id) image) _ (mark-image-for-gc image run-id) - result {:id (e/build image step evars) + result {:id (e/create-container image step evars) :mounted mounted} _ (log/debugf "Built a resourceful container: %s" (:id build-state))] - (if mount-needed? - (update-in result [:mounted] conj resource) - result) - (f/when-failed [err] - (log/errorf "Next step creation failed: %s" (f/message err)) - err))) + (if mount-needed? + (update-in result [:mounted] conj resource) + result) + (f/when-failed [err] + (log/errorf "Next step creation failed: %s" (f/message err)) + err))) (defn exec-step "Reducer function to implement the sequential execution of steps. @@ -135,10 +133,10 @@ pipeline) result (next-step build-state step evars pipeline run-id) id (update-pid (:id result) run-id) - id (let [result (e/run id run-id)] + id (let [result (e/start-container id run-id)] (when (f/failed? result) (log/debugf "Removing failed container %s" id) - (docker/rm states/docker-conn id)) + (e/delete-container id)) result) [group name] (clojure.string/split pipeline #":") _ (when-let [artifact (:produces_artifact step)] @@ -148,29 +146,31 @@ number artifact id - (str (get-in (docker/inspect states/docker-conn id) + (str (get-in (docker/invoke states/containers + {:op :ContainerInspect + :params {:id id}}) [:Config :WorkingDir]) "/" (:artifact_path step)) (:artifact_store step)))] - {:id id - :mounted (:mounted result)} - (f/when-failed [err] - (log/errorf "Failed to exec step: %s with error %s" step (f/message err)) - err)))) + {:id id + :mounted (:mounted result)} + (f/when-failed [err] + (log/errorf "Failed to exec step: %s with error %s" step (f/message err)) + err)))) (defn next-build-number-of "Generates a sequential build number for a pipeline." [name] (f/try-all [result (last (db/pipeline-runs states/db {:pipeline name}))] - (if (nil? result) - 1 - (inc (result :number))) - (f/when-failed [err] - (log/errorf "Error generating build number for %s: %s" - name - (f/message err)) - err))) + (if (nil? result) + 1 + (inc (result :number))) + (f/when-failed [err] + (log/errorf "Error generating build number for %s: %s" + name + (f/message err)) + err))) ;; TODO: Avoid doing the first step separately. Do it in the reduce like a normal person. (defn exec-steps @@ -192,16 +192,15 @@ :pipeline pipeline :status "running"}) _ (u/log-to-db (format "[bob] Pulling image %s" image) run-id) - _ (e/pull image) + _ (e/pull-image image) _ (mark-image-for-gc image run-id) image (resourceful-step first-step pipeline image run-id) _ (mark-image-for-gc image run-id) - id (e/build image first-step evars) + id (e/create-container image first-step evars) id (update-pid id run-id) - id (let [result (e/run id run-id)] + id (let [result (e/start-container id run-id)] (when (f/failed? result) - (log/debugf "Removing failed container %s" id) - (docker/rm states/docker-conn id)) + (e/delete-container id)) result) build-state (reduce (partial exec-step run-id evars pipeline number) {:id id @@ -210,33 +209,33 @@ [])} (rest steps)) _ (log/debug "Removing last successful container") - _ (docker/rm states/docker-conn (:id build-state)) + _ (e/delete-container (:id build-state)) _ (log/infof "Marking run %d for %s as passed" number pipeline) _ (gc-images run-id) _ (db/update-run states/db {:status "passed" :id run-id}) _ (u/log-to-db "[bob] Run successful" run-id)] - id - (f/when-failed [err] - (let [status (f/try* (:status (db/status-of states/db {:pipeline pipeline :number number})))] - (when-not (= status "stopped") - (log/infof "Marking run %d for %s as failed with reason: %s" - number - pipeline - (f/message err)) - (u/log-to-db (format "[bob] Run failed with reason: %s" (f/message err)) run-id) - (f/try* (db/update-run states/db - {:status "failed" - :id run-id})))) - (gc-images run-id) - err))))) + id + (f/when-failed [err] + (let [status (f/try* (:status (db/status-of states/db {:pipeline pipeline :number number})))] + (when-not (= status "stopped") + (log/infof "Marking run %d for %s as failed with reason: %s" + number + pipeline + (str err)) + (u/log-to-db (format "[bob] Run failed with reason: %s" (str err)) run-id) + (f/try* (db/update-run states/db + {:status "failed" + :id run-id})))) + (gc-images run-id) + err))))) (defn container-in-node "Checks if the container with `id` is running in the local Docker daemon." [id] (some #(clojure.string/starts-with? (:Id %) id) - (f/try* (docker/ps states/docker-conn)))) + (docker/invoke states/containers {:op :ContainerList}))) (defn sync-action "Dispatches either the action or signal based on external signal and container locality. @@ -282,11 +281,11 @@ _ (gc-images run-id)]) db-fn #(db/stop-run states/db criteria) _ (sync-action signalled? pid stopping-fn db-fn)] - "Ok" - (f/when-failed [err] - (let [message (f/message err)] - (log/errorf "Failed to stop pipeline: %s" message) - message))))))) + "Ok" + (f/when-failed [err] + (let [message (f/message err)] + (log/errorf "Failed to stop pipeline: %s" message) + message))))))) (defn pipeline-logs "Fetches all the logs from from a particular run-id split by lines. @@ -303,13 +302,13 @@ logs (:content (db/logs-of states/db {:run-id run-id})) _ (when (nil? logs) (f/fail "Unable to fetch logs for this run"))] - (->> logs - (clojure.string/split-lines) - (drop (dec offset)) - (take lines)) - (f/when-failed [err] - (log/errorf "Failed to fetch logs for pipeline %s: %s" name (f/message err)) - err))) + (->> logs + (clojure.string/split-lines) + (drop (dec offset)) + (take lines)) + (f/when-failed [err] + (log/errorf "Failed to fetch logs for pipeline %s: %s" name (f/message err)) + err))) (defn image-of "Returns the image associated with the pipeline." @@ -374,4 +373,9 @@ (container-in-node "118dadda9545") - (listen-on "stopped" (constantly true))) + (listen-on "stopped" (constantly true)) + + (str (get-in (docker/invoke states/containers {:op :ContainerInspect :params {:id "8778328a32b6"}}) + [:Config :WorkingDir])) + + (docker/invoke states/containers {:op :ContainerList})) diff --git a/src/bob/resource/internals.clj b/src/bob/resource/internals.clj index dc2aa459..83fd5052 100644 --- a/src/bob/resource/internals.clj +++ b/src/bob/resource/internals.clj @@ -19,8 +19,10 @@ [failjure.core :as f] [clj-docker-client.core :as docker] [taoensso.timbre :as log] + [bob.util :as u] [bob.states :as states] - [bob.resource.db :as db]) + [bob.resource.db :as db] + [bob.execution.internals :as e]) (:import (java.io BufferedOutputStream File FileInputStream @@ -40,9 +42,9 @@ (format "%s/bob_resource?%s" url (s/join - "&" - (map #(format "%s=%s" (:key %) (:value %)) - params))))) + "&" + (map #(format "%s=%s" (:key %) (:value %)) + params))))) (defn fetch-resource "Downloads a resource(tar file) and returns the stream." @@ -52,10 +54,10 @@ (:name resource) url)] ;; TODO: Potential out of memory issues here - (:body @(http/get url)) - (f/when-failed [err] - (log/errorf "Failed to fetch resource: %s" (f/message err)) - err))) + (:body @(http/get url)) + (f/when-failed [err] + (log/errorf "Failed to fetch resource: %s" (f/message err)) + err))) (defn prefix-dir-on-tar! "Adds a prefix to the tar entry paths to make a directory. @@ -79,8 +81,25 @@ (.close out-stream) (.getAbsolutePath archive))) +(defn put-container-archive + "Upload a tar archive to be extracted to a path in the filesystem of container id. + The tar archive needs to be compressed with one of the following algorithms: + identity (no compression), gzip, bzip2, xz. + Returns Id on success and failjure on failure" + [container-id archive-path extract-path] + (let [result (with-open [xin (-> archive-path + clojure.java.io/file + clojure.java.io/input-stream)] + (docker/invoke states/containers {:op :PutContainerArchive + :params {:id container-id + :path extract-path + :inputStream xin}}))] + (if (get result :message) + (u/log-and-fail "Could not put archive into container:" (:message result)) + result))) + (defn initial-image-of - "Takes a InputStream of the resource, name and image and builds the initial image. + "Takes an InputStream of the resource, name and image and builds the initial image. This image is used by Bob as the starting image which holds the initial state for the rest of the steps. @@ -95,27 +114,18 @@ TarInputStream. (prefix-dir-on-tar! resource-name)) _ (log/debug "Creating temp container for resource mount") - id (docker/create states/docker-conn image "" {} {}) + container-id (e/create-container image) _ (log/debug "Copying resources to container") - _ (-> states/docker-conn - (.copyArchiveToContainerCmd id) - (.withTarInputStream (-> archive FileInputStream.)) - (.withRemotePath "/root") - .exec) + _ (put-container-archive container-id archive "/root") _ (-> archive File. .delete) _ (log/debug "Committing resourceful container") - provisioned-image (docker/commit-container - states/docker-conn - id - (format "%s/%d" id (System/currentTimeMillis)) - "latest" - cmd) + provisioned-image (e/commit-image container-id cmd) _ (log/debug "Removing temp container") - _ (docker/rm states/docker-conn id)] - provisioned-image - (f/when-failed [err] - (log/errorf "Failed to create initial image: %s" (f/message err)) - err))) + _ (e/delete-container container-id :force)] + provisioned-image + (f/when-failed [err] + (log/errorf "Failed to create initial image: %s" (f/message err)) + err))) (defn add-params "Saves the map of GET params to be sent to the resource." @@ -139,12 +149,12 @@ "Fetches list of parameters associated with the resource" [pipeline name] (reduce - (fn [r {:keys [key value]}] - (assoc r (keyword key) value)) - {} - (db/resource-params-of states/db - {:name name - :pipeline pipeline}))) + (fn [r {:keys [key value]}] + (assoc r (keyword key) value)) + {} + (db/resource-params-of states/db + {:name name + :pipeline pipeline}))) (comment (def resource {:name "my-source" @@ -158,4 +168,25 @@ (->> (db/resource-params-of states/db {:name "my-source" - :pipeline "dev:test"}))) + :pipeline "dev:test"})) + + (with-open [xin (-> "/home/timo/projects/bob/test.tar" + clojure.java.io/file + clojure.java.io/input-stream)] + (docker/invoke states/containers {:op :PutContainerArchive + :params {:id "8778328a32b6" + :path "/root" + :inputStream xin}})) + + (put-container-archive "8778328a32b6" "/home/timo/projects/bob/test.tar" "/root") + (e/commit-image "8778328a32b6" "foo") + (e/delete-container "e206ec8f15d5") + + (docker/categories) + (def commit (docker/client {:category :commit :conn states/docker-conn})) + (docker/invoke commit {:op :ImageCommit + :params {:container "8778328a32b6" + :repo "local" + :tag "foobar" + :comment "foo so bar" + :author "memyselfandi"}})) diff --git a/src/bob/states.clj b/src/bob/states.clj index 825f9f16..d1744141 100644 --- a/src/bob/states.clj +++ b/src/bob/states.clj @@ -51,9 +51,20 @@ :start (repl/migrate migration-config)) (m/defstate docker-conn - :start (docker/connect) - :stop (do (log/info "Closing docker connection") - (docker/disconnect docker-conn))) + :start (docker/connect {:uri "unix:///var/run/docker.sock" + :connect-timeout 1000 + :read-timeout 60000 + :write-timeout 60000 + :call-timeout 90000})) + +(m/defstate images + :start (docker/client {:category :images :conn docker-conn})) + +(m/defstate containers + :start (docker/client {:category :containers :conn docker-conn})) + +(m/defstate commit + :start (docker/client {:category :commit :conn docker-conn})) (comment (m/start) diff --git a/src/bob/util.clj b/src/bob/util.clj index 78ad0f14..be7907b1 100644 --- a/src/bob/util.clj +++ b/src/bob/util.clj @@ -15,6 +15,8 @@ (ns bob.util (:require [ring.util.http-response :as res] + [taoensso.timbre :as log] + [failjure.core :as f] [bob.states :as states] [bob.pipeline.db :as db]) (:import (java.util UUID))) @@ -46,3 +48,70 @@ [data run-id] (db/upsert-log states/db {:run run-id :content data})) + +(defn log-and-fail + [& strings] + (let [errormessage (clojure.string/join " " strings)] + (do (log/errorf errormessage) + (f/fail errormessage)))) + +;; TODO: Optimize as mentioned in: +;; https://www.reddit.com/r/Clojure/comments/8zurv4/critical_code_review_and_feedback/ +(defn sh-tokenize! + "Tokenizes a shell command given as a string into the command and its args. + Either returns a list of tokens or throws an IllegalStateException. + Sample input: sh -c 'while sleep 1; do echo \\\"${RANDOM}\\\"; done' + Output: [sh, -c, while sleep 1; do echo \"${RANDOM}\"; done]" + [^String command] + (let [[escaped? + current-arg + args + state] (loop [cmd command + escaped? false + state :no-token + current-arg "" + args []] + (if (or (nil? cmd) + (zero? (count cmd))) + [escaped? current-arg args state] + (let [char ^Character (first cmd)] + (if escaped? + (recur (rest cmd) false state (str current-arg char) args) + (case state + :single-quote (if (= char \') + (recur (rest cmd) escaped? :normal current-arg args) + (recur (rest cmd) escaped? state (str current-arg char) args)) + :double-quote (case char + \" (recur (rest cmd) escaped? :normal current-arg args) + \\ (let [next (second cmd)] + (if (or (= next \") + (= next \\)) + (recur (drop 2 cmd) escaped? state (str current-arg next) args) + (recur (drop 2 cmd) escaped? state (str current-arg char next) args))) + (recur (rest cmd) escaped? state (str current-arg char) args)) + (:no-token :normal) (case char + \\ (recur (rest cmd) true :normal current-arg args) + \' (recur (rest cmd) escaped? :single-quote current-arg args) + \" (recur (rest cmd) escaped? :double-quote current-arg args) + (if-not (Character/isWhitespace char) + (recur (rest cmd) escaped? :normal (str current-arg char) args) + (if (= state :normal) + (recur (rest cmd) escaped? :no-token "" (conj args current-arg)) + (recur (rest cmd) escaped? state current-arg args)))) + (throw (IllegalStateException. + (format "Invalid shell command: %s, unexpected token %s found." command state))))))))] + (if escaped? + (conj args (str current-arg \\)) + (if (not= state :no-token) + (conj args current-arg) + args)))) + +(defn format-env-vars + [env-vars] + (map #(format "%s=%s" (name (first %)) (last %)) + env-vars)) + +(comment + (log-and-fail "foo" :bar) + + (db/logs-of states/db {:run-id "1"})) diff --git a/test/bob/api/health_test.clj b/test/bob/api/health_test.clj index 15a64fe9..de95c257 100644 --- a/test/bob/api/health_test.clj +++ b/test/bob/api/health_test.clj @@ -27,40 +27,40 @@ (deftest health-check-various-conditions-test (testing "all systems operational" - (with-redefs-fn {#'docker/ping (constantly "OK") + (with-redefs-fn {#'docker/invoke (constantly "OK") #'db-health-check (constantly {:?column? true}) #'ping-external-systems (constantly '())} #(is (= [] (health-check))))) (testing "failing docker daemon" - (with-redefs-fn {#'docker/ping (constantly (f/fail "Docker Failed")) + (with-redefs-fn {#'docker/invoke (constantly (java.io.IOException.)) #'db-health-check (constantly {:?column? true}) #'ping-external-systems (constantly '())} #(is (= ["Docker"] (health-check))))) (testing "failing postgres db" - (with-redefs-fn {#'docker/ping (constantly "OK") + (with-redefs-fn {#'docker/invoke (constantly "OK") #'db-health-check (constantly (f/fail "Postgres Failed"))} #(is (= ["Postgres"] (health-check))))) (testing "failing docker and postgres" - (with-redefs-fn {#'docker/ping (constantly (f/fail "Docker Failed")) + (with-redefs-fn {#'docker/invoke (constantly (java.io.IOException.)) #'db-health-check (constantly (f/fail "Postgres Failed"))} #(is (= ["Docker" "Postgres"] (health-check))))) (testing "failing external resource" - (with-redefs-fn {#'docker/ping (constantly "OK") + (with-redefs-fn {#'docker/invoke (constantly "OK") #'db-health-check (constantly {:?column? true}) #'ping-external-systems (constantly '("failed"))} #(is (= ["failed"] (health-check))))) (testing "failing external resources" - (with-redefs-fn {#'docker/ping (constantly "OK") + (with-redefs-fn {#'docker/invoke (constantly "OK") #'db-health-check (constantly {:?column? true}) #'ping-external-systems (constantly '("failed" "failed"))} #(is (= ["failed" "failed"] diff --git a/test/bob/artifact/core_test.clj b/test/bob/artifact/core_test.clj index 608c2e7a..477bd621 100644 --- a/test/bob/artifact/core_test.clj +++ b/test/bob/artifact/core_test.clj @@ -105,23 +105,16 @@ (= {:message "Cannot reach artifact store: Shizz"} (:body result)))))))) +; TODO reimplement the stream tests (deftest artifact-upload (testing "successful artifact upload" (with-redefs-fn {#'db/get-artifact-store (constantly {:url "bob-url"}) - #'docker/stream-path (fn [_ id path] - (tu/check-and-fail - #(and (= "1" id) - (= "/path" path))) - :stream) + #'docker/invoke (constantly "foo") #'http/post (fn [url options] (future (tu/check-and-fail - #(and (= "bob-url/bob_artifact/dev/test/1/afile" - url) - (= "data" - (get-in options [:multipart 0 :name])) - (= :stream - (get-in options [:multipart 0 :content]))))))} + #(= "bob-url/bob_artifact/dev/test/1/afile" + url))))} #(is (= "Ok" (upload-artifact "dev" "test" 1 "afile" "1" "/path" "s3"))))) @@ -134,7 +127,7 @@ (testing "unsuccessful artifact upload" (with-redefs-fn {#'db/get-artifact-store (constantly {:url "bob-url"}) - #'docker/stream-path (constantly :stream) + #'docker/invoke (constantly :stream) #'http/post (constantly (future (throw (Exception. "bad call"))))} #(let [result (upload-artifact "dev" "test" 1 "afile" "1" "/path" "s3")] (is (f/failed? result)))))) diff --git a/test/bob/execution/internals_test.clj b/test/bob/execution/internals_test.clj index 392fa9ac..77de8f97 100644 --- a/test/bob/execution/internals_test.clj +++ b/test/bob/execution/internals_test.clj @@ -22,25 +22,30 @@ (deftest docker-image-presence (testing "image present" - (with-redefs-fn {#'docker/image-ls (constantly [{:RepoTags ["img"]}])} - #(is (= "img" (has-image "img"))))) + (with-redefs-fn {#'docker/invoke (constantly [{:RepoTags ["clojure:latest"]}])} + #(is (= "clojure:latest" (has-image "clojure:latest"))))) + + (testing "image present no tag provided" + (with-redefs-fn {#'docker/invoke (constantly [{:RepoTags ["clojure:latest"]}])} + #(let [result (has-image "clojure")] + (is (and (f/failed? result) + (= "Failed to find clojure locally" + (f/message result))))))) (testing "image absent" - (with-redefs-fn {#'docker/image-ls (constantly [])} - #(let [result (has-image "img")] + (with-redefs-fn {#'docker/invoke (constantly [{:RepoTags ["clojure:foo"]}])} + #(let [result (has-image "clojure:latest")] (is (and (f/failed? result) - (= "Failed to find img" + (= "Failed to find clojure:latest locally" (f/message result)))))))) (deftest container-kill (testing "successful kill" - (with-redefs-fn {#'docker/kill (fn [_ name] - (tu/check-and-fail - #(= "c1" name)))} + (with-redefs-fn {#'docker/invoke (constantly "")} #(is (= "c1" (kill-container "c1"))))) (testing "unsuccessful kill" - (with-redefs-fn {#'docker/kill (constantly (f/fail "Failed"))} + (with-redefs-fn {#'docker/invoke (constantly "{:message \"Cannot kill container\"}")} #(let [result (kill-container "c1")] (is (and (f/failed? result) (= "Could not kill c1" @@ -49,62 +54,130 @@ (deftest image-pull (testing "successful pull" (with-redefs-fn {#'has-image (constantly true) - #'docker/pull (fn [_ name] - (tu/check-and-fail - #(= "img" name)))} - #(is (= "img" (pull "img"))))) + #'docker/invoke (fn [_ name] + (tu/check-and-fail + #(= "img" name)))} + #(is (= "img" (pull-image "img"))))) (testing "unsuccessful pull" (with-redefs-fn {#'has-image (constantly (f/fail "Failed")) - #'docker/pull (fn [_ _] (throw (Exception. "Failed")))} - #(let [result (pull "img")] + #'docker/invoke (fn [_ _] {:message "failed"})} + #(let [result (pull-image "clojure:foo")] (is (and (f/failed? result) - (= "Cannot pull img" - (f/message result)))))))) + (= "Could not pull image: clojure:foo" + (f/message result))))))) + + (testing "missing tag in image name" + (with-redefs-fn {#'docker/invoke (constantly {:message "failed"})} + #(let [result (pull-image "clojure")] + (is (f/failed? result))))) + + (testing "image already locally available" + (with-redefs-fn {#'has-image (constantly true)} + #(is (= "img" (pull-image "img")))))) + +(deftest container-build + (testing "successful build" + (with-redefs-fn {#'docker/invoke (constantly {:Id "83996a1d", :Warnings []})} + #(is (= "83996a1d" (create-container "foo:bar" + {:needs_resource "source" + :cmd "ls"} + {:FOO "bar"}))))) + + (testing "successful build single param" + (with-redefs-fn {#'docker/invoke (constantly {:Id "83996a1d", :Warnings []})} + #(is (= "83996a1d" (create-container "foo:bar"))))) + + (testing "successful build double param" + (with-redefs-fn {#'docker/invoke (constantly {:Id "83996a1d", :Warnings []})} + #(is (= "83996a1d" (create-container "foo:bar" {:cmd "ls"}))))) + + (testing "failing build" + (with-redefs-fn {#'docker/invoke (constantly {:message "failed"})} + #(is (f/failed? (create-container "foo:bar" + {:needs_resource "source" + :cmd "ls"} + {:FOO "bar"})))))) (deftest container-status (testing "successful status fetch" - (with-redefs-fn {#'docker/container-state (fn [_ id] - (tu/check-and-fail - #(= "id" id)) - {:Running false - :ExitCode 0})} + (with-redefs-fn {#'docker/invoke (constantly {:State {:Running false + :ExitCode 0}})} #(is (= {:running? false :exit-code 0} (status-of "id"))))) (testing "unsuccessful status fetch" - (with-redefs-fn {#'docker/container-state (fn [_ _] (throw (Exception. "Failed")))} + (with-redefs-fn {#'docker/invoke (fn [_ id] {:message (format "No such container: %s" id)})} #(is (f/failed? (status-of "id")))))) -(deftest container-runs - (testing "successful run" +;(deftest logs-live-test +; (testing "successfully pulling logs from container" +; (with-redefs-fn {#'docker/client (constantly true) +; #'docker/invoke (constantly (char-array "foobar"))} +; #(is (= (future "foobar") (logs-live "12121212" (fn [log] (str log)))))))) + +(deftest container-starts + (testing "successful start" (let [id "11235813213455"] - (with-redefs-fn {#'docker/start (fn [_ cid] - (tu/check-and-fail - #(= id cid))) - #'docker/logs-live (constantly true) - #'docker/inspect (fn [_ cid] - (tu/check-and-fail - #(= id cid)) - {:State {:ExitCode 0}})} - #(is (= "112358132134" (run id "run-id")))))) - - (testing "successful run, non-zero exit" + (with-redefs-fn {#'docker/invoke (fn [_ cid] + (tu/check-and-fail + #(= id (-> cid + :params + :id))) + {:StatusCode 0}) + #'logs-live (constantly true)} + #(is (= "112358132134" (start-container id "run-id")))))) + + (testing "successful start, non-zero exit" (let [id "11235813213455"] - (with-redefs-fn {#'docker/start (fn [_ cid] - (tu/check-and-fail - #(= id cid))) - #'docker/logs-live (constantly true) - #'docker/inspect (fn [_ cid] - (tu/check-and-fail - #(= id cid)) - {:State {:ExitCode 1}})} - #(let [result (run id "run-id")] - (is (and (f/failed? result) - (= "Abnormal exit." - (f/message result)))))))) - - (testing "unsuccessful run" - (with-redefs-fn {#'docker/start (constantly nil) - #'docker/logs-live (constantly true) - #'docker/inspect (fn [_ _] (throw (Exception. "Failed")))} - #(is (f/failed? (run "id" "run-id")))))) + (with-redefs-fn {#'docker/invoke (fn [_ cid] + (tu/check-and-fail + #(= id (-> cid + :params + :id))) + {:StatusCode 1}) + #'logs-live (constantly true)} + #(let [result (start-container id "run-id")] + (is (and (f/failed? result))))))) + + (testing "unsuccessful start" + (with-redefs-fn {#'logs-live (constantly true) + #'docker/invoke (fn [_ _] (throw (Exception. "Failed")))} + #(is (f/failed? (start-container "id" "run-id")))))) + +(deftest container-deletion + (testing "successful deletion of container" + (with-redefs-fn {#'docker/invoke (constantly "")} + #(is (= "" (delete-container 1))))) + + (testing "successful force deletion of running container" + (with-redefs-fn {#'docker/invoke (constantly "")} + #(is (= "" (delete-container 1 :force))))) + + (testing "failed deletion of running container" + (with-redefs-fn {#'docker/invoke (constantly {:message "Failed"})} + #(is (f/failed? (delete-container 1)))))) + +(deftest test-commit-image + (testing "Successful creation of new image from container" + (with-redefs-fn {#'docker/invoke (constantly {:Id "sha256:f2763f84"})} + #(is (= "sha256:f2763f84" (commit-image "12121212" "foo"))))) + + (testing "Failed creation of new image from container" + (with-redefs-fn {#'docker/invoke (constantly {:message "foobar"})} + #(is (f/failed? (commit-image "12121212" "foo")))))) + +(deftest test-delete-image + (testing "Successful deletion of image" + (let [image "foobar:1.1.0"] + (with-redefs-fn {#'docker/invoke (fn [_ opmap] + (tu/check-and-fail + #(= image (-> opmap + :params + :name))) + [{:Untagged image} {:Deleted image}])} + #(is (= (delete-image image) [{:Untagged image} {:Deleted image}]))))) + + (testing "Failed deletion of image" + (let [image "foobar:1.1.0"] + (with-redefs-fn {#'docker/invoke (constantly (f/fail "No such image: foobar:1.1.0"))} + #(is (= (delete-image image) (f/fail "Could not delete image: No such image: foobar:1.1.0"))))))) diff --git a/test/bob/pipeline/internals_test.clj b/test/bob/pipeline/internals_test.clj index b6885924..77cfa7e9 100644 --- a/test/bob/pipeline/internals_test.clj +++ b/test/bob/pipeline/internals_test.clj @@ -30,28 +30,35 @@ (deftest image-mark-and-sweep-test (log/merge-config! {:level :report}) (testing "marking an image for the first time" - (is (= {"build1" (list "image1")} - (mark-image-for-gc "image1" "build1")))) + (with-redefs [bob.pipeline.internals/images-produced (atom {})] + (is (= {"build1" (list "image1")} + (mark-image-for-gc "image1" "build1"))))) (testing "marking the same image is idempotent" - (is (= {"build1" (list "image1")} - (mark-image-for-gc "image1" "build1")))) + (with-redefs [bob.pipeline.internals/images-produced (atom {})] + (is (= {"build1" (list "image1")} + (mark-image-for-gc "image1" "build1"))))) (testing "marking a new image for the same build" - (is (= {"build1" (list "image2" "image1")} - (mark-image-for-gc "image2" "build1")))) + (with-redefs [bob.pipeline.internals/images-produced (atom {"build1" (list "image1")})] + (is (= {"build1" (list "image2" "image1")} + (mark-image-for-gc "image2" "build1"))))) (testing "marking a new image for another same build" - (is (= {"build1" (list "image2" "image1") - "build2" (list "image1")} - (mark-image-for-gc "image1" "build2")))) + (with-redefs [bob.pipeline.internals/images-produced (atom {"build1" (list "image2" "image1")})] + (is (= {"build1" (list "image2" "image1") + "build2" (list "image1")} + (mark-image-for-gc "image1" "build2")))))) +(deftest gc-images-test (testing "sweep images for build1" - (with-redefs-fn {#'docker/image-rm (fn [_ image] - (tu/check-and-fail - #(some #{image} ["image1" "image2"])))} - #(is (= {"build2" (list "image1")} - (gc-images "build1")))))) + (with-redefs [bob.pipeline.internals/images-produced (atom {"build1" (list "image2" "image1") + "build2" (list "image1")})] + (with-redefs-fn {#'e/delete-image (fn [image] + (tu/check-and-fail + #(some #{image} ["image1" "image2"])))} + #(is (= {"build2" (list "image1")} + (gc-images "build1"))))))) (deftest pid-updates (testing "successful container id update" @@ -92,17 +99,14 @@ (deftest next-step-execution (testing "next images generation with resource mount" - (with-redefs-fn {#'docker/commit-container (fn [_ id repo tag cmd] - (tu/check-and-fail - #(and (= "id" id) - (clojure.string/starts-with? repo id) - (= "latest" tag) - (= "hello" cmd))) - "img") - #'resourceful-step (constantly "img") - #'e/build (constantly "id") - #'docker/rm (constantly nil) - #'mark-image-for-gc (constantly nil)} + (with-redefs-fn {#'e/commit-image (fn [id cmd] + (tu/check-and-fail + #(and (= "id" id) + (= "hello" cmd)))) + #'resourceful-step (constantly "img") + #'e/create-container (constantly "id") + #'e/delete-container (constantly nil) + #'mark-image-for-gc (constantly nil)} #(is (= {:id "id" :mounted ["source"]} (next-step {:id "id" :mounted []} @@ -113,17 +117,14 @@ "run-id"))))) (testing "next image generation without resource mount" - (with-redefs-fn {#'docker/commit-container (fn [_ id repo tag cmd] - (tu/check-and-fail - #(and (= "id" id) - (clojure.string/starts-with? repo id) - (= "latest" tag) - (= "hello" cmd))) - "img") - #'resourceful-step (constantly "img") - #'e/build (constantly "id") - #'docker/rm (constantly nil) - #'mark-image-for-gc (constantly nil)} + (with-redefs-fn {#'e/commit-image (fn [id cmd] + (tu/check-and-fail + #(and (= "id" id) + (= "hello" cmd)))) + #'resourceful-step (constantly "img") + #'e/create-container (constantly "id") + #'e/delete-container (constantly nil) + #'mark-image-for-gc (constantly nil)} #(is (= {:id "id" :mounted []} (next-step {:id "id" :mounted []} @@ -133,10 +134,10 @@ "run-id"))))) (testing "failed next image generation" - (with-redefs-fn {#'docker/commit-container #(throw (Exception. "nope")) - #'resourceful-step (constantly "img") - #'e/build (constantly "id") - #'mark-image-for-gc (constantly nil)} + (with-redefs-fn {#'e/commit-image #(throw (Exception. "nope")) + #'resourceful-step (constantly "img") + #'e/create-container (constantly "id") + #'mark-image-for-gc (constantly nil)} #(is (f/failed? (next-step {:id "id" :mounted []} {:cmd "hello"} {} @@ -162,14 +163,14 @@ #(and (= "id" id) (= "id" run-id))) "id") - #'e/run (fn [id run-id] + #'e/start-container (fn [id run-id] (tu/check-and-fail #(and (= "id" id) (= "id" run-id))) "id") - #'docker/inspect (fn [_ id] + #'docker/invoke (fn [_ params] (tu/check-and-fail - #(= "id" id)) + #(= "id" (:id (:params params)))) {:Config {:WorkingDir "/some"}}) #'artifact/upload-artifact (fn [group name number artifact id path store-name] (tu/check-and-fail @@ -201,12 +202,12 @@ #(and (= "id" id) (= "id" run-id))) "id") - #'e/run (fn [id run-id] + #'e/start-container (fn [id run-id] (tu/check-and-fail #(and (= "id" id) (= "id" run-id))) "id") - #'docker/inspect (fn [_ id] + #'docker/invoke (fn [_ id] (tu/check-and-fail #(= "id" id)) {:Config {:WorkingDir "/some"}}) @@ -220,8 +221,8 @@ nein (fn [& _] (throw (Exception. "shouldn't be called")))] (with-redefs-fn {#'next-step nein #'update-pid nein - #'e/run nein - #'docker/inspect nein + #'e/start-container nein + #'docker/invoke nein #'artifact/upload-artifact nein} #(is (reduced? (exec-step "id" {} "dev:test" 1 (f/fail "shizz") test-step)))))) @@ -230,8 +231,8 @@ nein (fn [& _] (throw (Exception. "shouldn't be called")))] (with-redefs-fn {#'next-step (constantly (f/fail "shizz")) #'update-pid nein - #'e/run nein - #'docker/inspect nein + #'e/start-container nein + #'docker/invoke nein #'artifact/upload-artifact nein} #(is (f/failed? (exec-step "id" {} "dev:test" 1 {:id "id"} test-step))))))) @@ -271,7 +272,7 @@ :status "running"} args))) #'u/log-to-db (constantly nil) - #'e/pull (fn [img] + #'e/pull-image (fn [img] (tu/check-and-fail #(= "img" img)) "img") @@ -283,7 +284,7 @@ (= "img" img) (= "run-id" run-id))) "img") - #'e/build (fn [img step evars] + #'e/create-container (fn [img step evars] (tu/check-and-fail #(and (= "img" img) (= first-step @@ -295,7 +296,7 @@ #(and (= "id" id) (= "run-id" run-id))) "id") - #'e/run (fn [id run-id] + #'e/start-container (fn [id run-id] (tu/check-and-fail #(and (= "id" id) (= "run-id" run-id))) @@ -314,7 +315,7 @@ #(= {:status "passed" :id "run-id"} args))) - #'docker/rm (constantly nil) + #'e/delete-container (constantly nil) #'mark-image-for-gc (constantly nil)} #(is (= "id" @(exec-steps "img" [first-step {:cmd "hello2"}] @@ -337,7 +338,7 @@ :status "running"} args))) #'u/log-to-db (constantly nil) - #'e/pull (fn [img] + #'e/pull-image (fn [img] (tu/check-and-fail #(= "img" img)) "img") @@ -349,7 +350,7 @@ (= "img" img) (= "run-id" run-id))) "img") - #'e/build (fn [img step evars] + #'e/create-container (fn [img step evars] (tu/check-and-fail #(and (= "img" img) (= first-step @@ -361,7 +362,7 @@ #(and (= "id" id) (= "run-id" run-id))) "id") - #'e/run (fn [id run-id] + #'e/start-container (fn [id run-id] (tu/check-and-fail #(and (= "id" id) (= "run-id" run-id))) @@ -380,7 +381,7 @@ #(= {:status "passed" :id "run-id"} args))) - #'docker/rm (constantly nil) + #'e/delete-container (constantly nil) #'mark-image-for-gc (constantly nil)} #(is (= "id" @(exec-steps "img" [first-step {:cmd "hello2"}] @@ -404,12 +405,12 @@ :status "running"} args))) #'u/log-to-db (constantly nil) - #'e/pull (constantly (f/fail "shizz")) + #'e/pull-image (constantly (f/fail "shizz")) #'db/status-of (constantly {:status "running"}) #'resourceful-step nein - #'e/build nein + #'e/create-container nein #'update-pid nein - #'e/run nein + #'e/start-container nein #'reduce nein #'db/update-run nein #'mark-image-for-gc nein} @@ -593,11 +594,11 @@ (deftest container-locality-test (testing "container is found locally" - (with-redefs-fn {#'docker/ps (constantly [{:Id "id1"}])} + (with-redefs-fn {#'docker/invoke (constantly [{:Id "id1"}])} #(is (container-in-node "id")))) (testing "container is not found locally" - (with-redefs-fn {#'docker/ps (constantly [{:Id "crappy-id"}])} + (with-redefs-fn {#'docker/invoke (constantly [{:Id "crappy-id"}])} #(is (nil? (container-in-node "id")))))) (deftest sync-action-test diff --git a/test/bob/resource/internals_test.clj b/test/bob/resource/internals_test.clj index ba63ba37..46dd0e4b 100644 --- a/test/bob/resource/internals_test.clj +++ b/test/bob/resource/internals_test.clj @@ -15,8 +15,11 @@ (ns bob.resource.internals-test (:require [clojure.test :refer :all] + [failjure.core :as f] + [clj-docker-client.core :as docker] [bob.test-utils :as tu] [bob.resource.db :as db] + [bob.execution.internals :as e] [bob.resource.internals :refer :all])) ;; TODO: Test the rest of the fns too? @@ -43,6 +46,7 @@ :name "r1"} "test")))))) + (deftest get-resource-params-test (testing "Testing if result from DB is mapped correctly" (with-redefs-fn {#'db/resource-params-of (fn [pipeline name] @@ -57,3 +61,18 @@ #(is (= (get-resource-params "res" "dev:test") {:env "dev" :database "jdbc:mysql://db.example.com/test"}))))) + +;; TODO how to mock a file? +;(deftest test-put-container-archive +; (testing "Upload of a tar archive into container" +; (with-redefs-fn {#'docker/invoke (constantly "") +; #'clojure.java.io/input-stream (constantly nil)} +; #(is (= "" (put-container-archive "121212121" "/path/foo/bar.tar" "/root"))))) +; +; (testing "Failed upload of a tar archive into container" +; (with-redefs-fn {#'docker/invoke (constantly {:message "failed"}) +; #'clojure.java.io/input-stream (constantly nil)} +; #(is (f/failed? (put-container-archive "121212121" "/path/foo/bar.tar" "/root")))))) +;; TODO how to mock a stream? +;(deftest test-initial-image-of) +;(deftest test-fetch-resource) diff --git a/test/bob/util_test.clj b/test/bob/util_test.clj index 51b2c8dc..a3494095 100644 --- a/test/bob/util_test.clj +++ b/test/bob/util_test.clj @@ -19,6 +19,7 @@ [clojure.test.check.properties :as prop] [clojure.spec.alpha :as s] [clojure.test :refer [deftest testing is]] + [failjure.core :as f] [bob.util :refer :all])) (defspec respond-returns-a-ring-response @@ -52,3 +53,24 @@ (let [id1 (get-id) id2 (get-id)] (is (not= id1 id2))))) + +(deftest log-and-fail-test + (testing "variadic arguments" + (is (f/failed? (log-and-fail (gen/generate gen/string-alphanumeric) + (gen/generate gen/string-alphanumeric) + (gen/generate gen/string-alphanumeric) + (gen/generate gen/string-alphanumeric)))))) + +(deftest shell-arg-tokenize-test + (testing "tokenizing a Shell command" + (is (= (sh-tokenize! "sh -c \"while sleep 1; do echo ${RANDOM}; done\"") + ["sh" "-c" "while sleep 1; do echo ${RANDOM}; done"]))) + (testing "tokenizing a Shell command with escaped double quotes" + (is (= (sh-tokenize! "sort -t \"\t\" -k2 test > test-sorted") + ["sort" "-t" "\t" "-k2" "test" ">" "test-sorted"])))) + +(deftest format-env-vars-test + (testing "format a map of env vars for Docker" + (is (= (format-env-vars {:var1 "val1" + :var2 "val2"}) + ["var1=val1" "var2=val2"]))))