Skip to content

Commit

Permalink
cider-formatting
Browse files Browse the repository at this point in the history
  • Loading branch information
@TimoKramer
TimoKramer committed Mar 2, 2020
1 parent 50ea2fc commit 04d51c0
Show file tree
Hide file tree
Showing 5 changed files with 103 additions and 102 deletions.
13 changes: 7 additions & 6 deletions src/bob/api/health.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,20 +44,20 @@
(let [ping-client (docker/client {:category :_ping
:conn (docker/connect {:uri "unix:///var/run/docker.sock"})})
docker (when (not (= "OK" (docker/invoke ping-client {:op :SystemPing})))
["Docker"])
["Docker"])
postgres (when (f/failed? (f/try* (db-health-check states/db)))
["Postgres"])
["Postgres"])
extsys (when (nil? postgres)
(ping-external-systems))]
(ping-external-systems))]
(filter some? (concat docker postgres extsys))))

(defn respond-to-health-check
"Endpoint for answering a health check"
[]
(d/let-flow [failures (health-check)]
(if (empty? failures)
(u/respond "Yes we can! \uD83D\uDD28 \uD83D\uDD28")
(u/service-unavailable (str "Health check failed: " (clojure.string/join " and " failures) " not healthy")))))
(if (empty? failures)
(u/respond "Yes we can! \uD83D\uDD28 \uD83D\uDD28")
(u/service-unavailable (str "Health check failed: " (clojure.string/join " and " failures) " not healthy")))))

(defn log-health-check
"Logs if any of the subsystems is unhealthy."
Expand All @@ -67,6 +67,7 @@
(log/debugf (str "Health check succeeded!"))
(log/warn (str "Health check failed: " (clojure.string/join " and " failures) " not healthy")))))

(put-container-archive "8778328a32b6" "/home/timo/projects/bob/test.tar" "/root")
(defn start-heartbeat
"Starts a periodic heatbeat which performs a health check."
[]
Expand Down
30 changes: 15 additions & 15 deletions src/bob/execution/internals.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,23 +100,23 @@
(:Id result))))
([image step] (create-container image step {}))
([image step evars] (let [resource (:needs_resource step)
working-dir (when resource (str "/root/" resource))
cmd (u/sh-tokenize! (:cmd step))
formatted-evars (u/format-env-vars evars)
_ (log/debugf "Creating new container with:
working-dir (when resource (str "/root/" resource))
cmd (u/sh-tokenize! (:cmd step))
formatted-evars (u/format-env-vars evars)
_ (log/debugf "Creating new container with:
image: %s
cmd: %s
evars: %s
working-dir: %s"
image
cmd
(vec formatted-evars)
working-dir)
result (docker/invoke states/containers {:op :ContainerCreate
:params {:body {:Image image
:Cmd cmd
:Env formatted-evars
:WorkingDir working-dir}}})]
image
cmd
(vec formatted-evars)
working-dir)
result (docker/invoke states/containers {:op :ContainerCreate
:params {:body {:Image image
:Cmd cmd
:Env formatted-evars
:WorkingDir working-dir}}})]
(if (contains? result :message)
(u/log-and-fail "Could not build container with image:" (:message result))
(:Id result)))))
Expand Down Expand Up @@ -215,8 +215,8 @@
(:StatusCode (docker/invoke states/containers {:op :ContainerWait :params {:id myid}}))

(create-container "bobcd/bob:latest"
{:needs_resource "source"
:cmd "ls"})
{:needs_resource "source"
:cmd "ls"})

(defn validate-name [name]
(let [split-name (cs/split name #":")]
Expand Down
102 changes: 51 additions & 51 deletions src/bob/pipeline/internals.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,10 @@
_ (db/update-runs states/db
{:pid pid
:id run-id})]
pid
(f/when-failed [err]
(log/errorf "Failed to update pid: %s" (f/message err))
err)))
pid
(f/when-failed [err]
(log/errorf "Failed to update pid: %s" (f/message err))
err)))

(defn resourceful-step
"Create a resource mounted image for a step if it needs it."
Expand All @@ -89,8 +89,8 @@
[build-state step evars pipeline run-id]
(f/try-all [_ (log/debugf "Committing container: %s" (:id build-state))
image (e/commit-image
(:id build-state)
(:cmd step))
(:id build-state)
(:cmd step))
_ (mark-image-for-gc image run-id)
_ (log/debug "Removing commited container")
_ (e/delete-container (:id build-state))
Expand All @@ -106,12 +106,12 @@
result {:id (e/create-container image step evars)
:mounted mounted}
_ (log/debugf "Built a resourceful container: %s" (:id build-state))]
(if mount-needed?
(update-in result [:mounted] conj resource)
result)
(f/when-failed [err]
(log/errorf "Next step creation failed: %s" (f/message err))
err)))
(if mount-needed?
(update-in result [:mounted] conj resource)
result)
(f/when-failed [err]
(log/errorf "Next step creation failed: %s" (f/message err))
err)))

(defn exec-step
"Reducer function to implement the sequential execution of steps.
Expand Down Expand Up @@ -153,24 +153,24 @@
"/"
(:artifact_path step))
(:artifact_store step)))]
{:id id
:mounted (:mounted result)}
(f/when-failed [err]
(log/errorf "Failed to exec step: %s with error %s" step (f/message err))
err))))
{:id id
:mounted (:mounted result)}
(f/when-failed [err]
(log/errorf "Failed to exec step: %s with error %s" step (f/message err))
err))))

(defn next-build-number-of
"Generates a sequential build number for a pipeline."
[name]
(f/try-all [result (last (db/pipeline-runs states/db {:pipeline name}))]
(if (nil? result)
1
(inc (result :number)))
(f/when-failed [err]
(log/errorf "Error generating build number for %s: %s"
name
(f/message err))
err)))
(if (nil? result)
1
(inc (result :number)))
(f/when-failed [err]
(log/errorf "Error generating build number for %s: %s"
name
(f/message err))
err)))

;; TODO: Avoid doing the first step separately. Do it in the reduce like a normal person.
(defn exec-steps
Expand Down Expand Up @@ -216,20 +216,20 @@
{:status "passed"
:id run-id})
_ (u/log-to-db "[bob] Run successful" run-id)]
id
(f/when-failed [err]
(let [status (f/try* (:status (db/status-of states/db {:pipeline pipeline :number number})))]
(when-not (= status "stopped")
(log/infof "Marking run %d for %s as failed with reason: %s"
number
pipeline
(f/message err))
(u/log-to-db (format "[bob] Run failed with reason: %s" (f/message err)) run-id)
(f/try* (db/update-run states/db
{:status "failed"
:id run-id}))))
(gc-images run-id)
err)))))
id
(f/when-failed [err]
(let [status (f/try* (:status (db/status-of states/db {:pipeline pipeline :number number})))]
(when-not (= status "stopped")
(log/infof "Marking run %d for %s as failed with reason: %s"
number
pipeline
(f/message err))
(u/log-to-db (format "[bob] Run failed with reason: %s" (f/message err)) run-id)
(f/try* (db/update-run states/db
{:status "failed"
:id run-id}))))
(gc-images run-id)
err)))))

(defn container-in-node
"Checks if the container with `id` is running in the local Docker daemon."
Expand Down Expand Up @@ -281,11 +281,11 @@
_ (gc-images run-id)])
db-fn #(db/stop-run states/db criteria)
_ (sync-action signalled? pid stopping-fn db-fn)]
"Ok"
(f/when-failed [err]
(let [message (f/message err)]
(log/errorf "Failed to stop pipeline: %s" message)
message)))))))
"Ok"
(f/when-failed [err]
(let [message (f/message err)]
(log/errorf "Failed to stop pipeline: %s" message)
message)))))))

(defn pipeline-logs
"Fetches all the logs from from a particular run-id split by lines.
Expand All @@ -302,13 +302,13 @@
logs (:content (db/logs-of states/db {:run-id run-id}))
_ (when (nil? logs)
(f/fail "Unable to fetch logs for this run"))]
(->> logs
(clojure.string/split-lines)
(drop (dec offset))
(take lines))
(f/when-failed [err]
(log/errorf "Failed to fetch logs for pipeline %s: %s" name (f/message err))
err)))
(->> logs
(clojure.string/split-lines)
(drop (dec offset))
(take lines))
(f/when-failed [err]
(log/errorf "Failed to fetch logs for pipeline %s: %s" name (f/message err))
err)))

(defn image-of
"Returns the image associated with the pipeline."
Expand Down
32 changes: 16 additions & 16 deletions src/bob/resource/internals.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@
(format "%s/bob_resource?%s"
url
(s/join
"&"
(map #(format "%s=%s" (:key %) (:value %))
params)))))
"&"
(map #(format "%s=%s" (:key %) (:value %))
params)))))

(defn fetch-resource
"Downloads a resource(tar file) and returns the stream."
Expand All @@ -54,10 +54,10 @@
(:name resource)
url)]
;; TODO: Potential out of memory issues here
(:body @(http/get url))
(f/when-failed [err]
(log/errorf "Failed to fetch resource: %s" (f/message err))
err)))
(:body @(http/get url))
(f/when-failed [err]
(log/errorf "Failed to fetch resource: %s" (f/message err))
err)))

(defn prefix-dir-on-tar!
"Adds a prefix to the tar entry paths to make a directory.
Expand Down Expand Up @@ -123,9 +123,9 @@
_ (log/debug "Removing temp container")
_ (e/delete-container container-id :force)]
provisioned-image
(f/when-failed [err]
(log/errorf "Failed to create initial image: %s" (f/message err))
err)))
(f/when-failed [err]
(log/errorf "Failed to create initial image: %s" (f/message err))
err)))

(defn add-params
"Saves the map of GET params to be sent to the resource."
Expand All @@ -149,12 +149,12 @@
"Fetches list of parameters associated with the resource"
[pipeline name]
(reduce
(fn [r {:keys [key value]}]
(assoc r (keyword key) value))
{}
(db/resource-params-of states/db
{:name name
:pipeline pipeline})))
(fn [r {:keys [key value]}]
(assoc r (keyword key) value))
{}
(db/resource-params-of states/db
{:name name
:pipeline pipeline})))

(comment
(def resource {:name "my-source"
Expand Down
28 changes: 14 additions & 14 deletions test/bob/execution/internals_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,9 @@
(testing "successful build"
(with-redefs-fn {#'docker/invoke (constantly {:Id "83996a1d", :Warnings []})}
#(is (= "83996a1d" (create-container "foo:bar"
{:needs_resource "source"
:cmd "ls"}
{:FOO "bar"})))))
{:needs_resource "source"
:cmd "ls"}
{:FOO "bar"})))))

(testing "successful build single param"
(with-redefs-fn {#'docker/invoke (constantly {:Id "83996a1d", :Warnings []})}
Expand All @@ -97,9 +97,9 @@
(testing "failing build"
(with-redefs-fn {#'docker/invoke (constantly {:message "failed"})}
#(is (f/failed? (create-container "foo:bar"
{:needs_resource "source"
:cmd "ls"}
{:FOO "bar"}))))))
{:needs_resource "source"
:cmd "ls"}
{:FOO "bar"}))))))

(deftest container-status
(testing "successful status fetch"
Expand All @@ -121,21 +121,21 @@
(testing "successful run"
(let [id "11235813213455"]
(with-redefs-fn {#'docker/invoke (fn [_ cid]
(tu/check-and-fail
#(= id (-> cid
:params
:id)))
(tu/check-and-fail
#(= id (-> cid
:params
:id)))
{:StatusCode 0})
#'logs-live (constantly true)}
#(is (= "112358132134" (run id "run-id"))))))

(testing "successful run, non-zero exit"
(let [id "11235813213455"]
(with-redefs-fn {#'docker/invoke (fn [_ cid]
(tu/check-and-fail
#(= id (-> cid
:params
:id)))
(tu/check-and-fail
#(= id (-> cid
:params
:id)))
{:StatusCode 1})
#'logs-live (constantly true)}
#(let [result (run id "run-id")]
Expand Down

0 comments on commit 04d51c0

Please sign in to comment.