-
Notifications
You must be signed in to change notification settings - Fork 0
/
config.clj
274 lines (238 loc) · 9.04 KB
/
config.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
(ns monkey.ci.config
"Configuration functionality. This reads the application configuration from various
sources, like environment vars or command-line args. The configuration is structured
in a hierarchy and optionally some values are converted. Then this configuration is
used to add any 'constructor functions', that are then used to create new functions to
do some actual work. This allows us to change the behaviour of the application with
configuration, but also makes it possible to inject dummy functions for testing
purposes."
(:require [camel-snake-kebab.core :as csk]
[cheshire.core :as json]
[clojure
[string :as cs]
[walk :as cw]]
[clojure.java.io :as io]
[clojure.tools.logging :as log]
[medley.core :as mc]
[monkey.ci
[pem :as pem]
[utils :as u]]))
(def ^:dynamic *global-config-file* "/etc/monkeyci/config.edn")
(def ^:dynamic *home-config-file* (-> (System/getProperty "user.home")
(io/file ".monkeyci" "config.edn")
(.getCanonicalPath)))
(def env-prefix "monkeyci")
;; Determine version at compile time
(defmacro version []
(or (System/getenv (csk/->SCREAMING_SNAKE_CASE (str env-prefix "-version"))) "0.1.0-SNAPSHOT"))
(defn- key-filter [prefix]
(let [exp (str (name prefix) "-")]
#(.startsWith (name %) exp)))
(defn- strip-prefix [prefix]
(fn [k]
(keyword (subs (name k) (inc (count (name prefix)))))))
(defn- filter-and-strip-keys [prefix m]
(->> m
(mc/filter-keys (key-filter prefix))
(mc/map-keys (strip-prefix prefix))))
(defn strip-env-prefix [e]
(filter-and-strip-keys env-prefix e))
(defn group-keys
"Takes all keys in given map `m` that start with `:prefix-` and
moves them to a submap with the prefix name, and the prefix
stripped from the keys. E.g. `{:test-key 100}` with prefix `:test`
would become `{:test {:key 100}}`"
[m prefix]
(let [s (filter-and-strip-keys prefix m)
r (mc/remove-keys (key-filter prefix) m)]
(cond-> r
(not-empty s)
(update prefix merge s))))
(defn group-and-merge-from-env
"Given a map, takes all keys in `:env` that start with the given prefix
(using `group-keys`) and merges them with the existing submap with same
key.
For example, `{:env {:test-key \"value\"} :test {:other-key \"other-value\"}}`
would become `{:test {:key \"value\" :other-key \"other-value\"}}`.
The newly grouped values overwrite any existing values."
[m prefix]
(let [em (filter-and-strip-keys prefix (:env m))]
(-> m
(update prefix merge em)
(as-> x (mc/remove-vals nil? x)))))
(defn keywordize-type [v]
(if (map? v)
(mc/update-existing v :type keyword)
v))
(defn- parse-edn
"Parses the input file as `edn` and converts keys to kebab-case."
[p]
(with-open [r (io/reader p)]
(->> (u/parse-edn r)
(cw/prewalk (fn [x]
(if (map-entry? x)
(let [[k v] x]
[(csk/->kebab-case-keyword (name k)) v])
x))))))
(defn- parse-json
"Parses the file as `json`, converting keys to kebab-case."
[p]
(with-open [r (io/reader p)]
(json/parse-stream r csk/->kebab-case-keyword)))
(defn load-config-file
"Loads configuration from given file. This supports json and edn and converts
keys always to kebab-case."
[f]
(when-let [p (some-> f
u/abs-path
io/file)]
(when (.exists p)
(log/debug "Reading configuration file:" p)
(letfn [(has-ext? [ext s]
(cs/ends-with? s ext))]
(condp has-ext? f
".edn" (parse-edn p)
".json" (parse-json p))))))
(def default-app-config
"Default configuration for the application, without env vars or args applied."
{:http
{:port 3000}
:events
{:type :manifold}
:runner
{:type :child}
:storage
{:type :memory}
:containers
{:type :podman}
:reporter
{:type :print}
:logging
{:type :inherit}
:workspace
{:type :disk :dir "tmp/workspace"}
:artifacts
{:type :disk :dir "tmp/artifacts"}
:cache
{:type :disk :dir "tmp/cache"}})
(defn- merge-configs [configs]
(reduce u/deep-merge default-app-config configs))
(defn load-raw-config
"Loads raw (not normalized) configuration from its various sources"
[extra-files]
(-> (map load-config-file (concat [*global-config-file*
*home-config-file*]
extra-files))
(merge-configs)
(u/prune-tree)))
(defmulti normalize-key
"Normalizes the config as read from files and env, for the specific key.
The method receives the entire config, that also holds the env and args
and should return the updated config."
(fn [k _] k))
(defmethod normalize-key :default [k c]
(mc/update-existing c k keywordize-type))
(defmethod normalize-key :dev-mode [_ conf]
(let [r (mc/assoc-some conf :dev-mode (get-in conf [:args :dev-mode]))]
(cond-> r
(not (boolean? (:dev-mode r))) (dissoc :dev-mode))))
(defn abs-work-dir [conf]
(u/abs-path (or (get-in conf [:args :workdir])
(:work-dir conf)
(u/cwd))))
(defmethod normalize-key :work-dir [_ conf]
(assoc conf :work-dir (abs-work-dir conf)))
(defmethod normalize-key :account [_ {:keys [args] :as conf}]
(let [c (update conf :account merge (-> args
(select-keys [:customer-id :project-id :repo-id])
(mc/assoc-some :url (:server args))))]
(cond-> c
(empty? (:account c)) (dissoc :account))))
(defn- dir-or-work-sub [conf k d]
(update conf k #(or (u/abs-path %) (u/combine (abs-work-dir conf) d))))
(defmethod normalize-key :checkout-base-dir [k conf]
(dir-or-work-sub conf k "checkout"))
(defmethod normalize-key :ssh-keys-dir [k conf]
(dir-or-work-sub conf k "ssh-keys"))
(defmethod normalize-key :api [_ conf]
conf)
(defmethod normalize-key :build [_ conf]
(update conf :build (fn [b]
(-> b
(group-keys :git)
(group-keys :script)
(mc/update-existing :sid u/parse-sid)
(mc/update-existing :git group-keys :author)))))
(defn normalize-config
"Given a configuration map loaded from file, environment variables and command-line
args, applies all registered normalizers to it and returns the result. Since the
order of normalizers is undefined, they should not be dependent on each other."
[conf env args]
(letfn [(merge-if-map [d m]
(if (map? d)
(merge d m)
(or m d)))
(nil-if-empty [x]
(when (or (not (seqable? x))
(and (seqable? x) (not-empty x)))
x))]
(-> (methods normalize-key)
(keys)
(as-> keys-to-normalize
(reduce (fn [r k]
(->> (or (get env k)
(filter-and-strip-keys k env))
(nil-if-empty)
(merge-if-map (get conf k))
(mc/assoc-some r k)
(u/prune-tree)
(normalize-key k)))
(assoc conf :env env :args args)
keys-to-normalize))
(dissoc :default :env))))
(defn app-config
"Combines app environment with command-line args into a unified
configuration structure. Args have precedence over env vars,
which in turn override config loaded from files and default values."
[env args]
(-> (load-raw-config (:config-file args))
(normalize-config (strip-env-prefix env) args)))
(defn- flatten-nested
"Recursively flattens a map of maps. Each key in the resulting map is a
combination of the path of the parent keys."
[path c]
(letfn [(make-key [k]
(->> (conj path k)
(map name)
(cs/join "-")
(keyword)))]
(reduce-kv (fn [r k v]
(if (map? v)
(merge r (flatten-nested (conj path k) v))
(assoc r (make-key k) v)))
{}
c)))
(defmulti serialize-config class)
(defmethod serialize-config :default [x]
(str x))
(defmethod serialize-config clojure.lang.Keyword [k]
(name k))
(defmethod serialize-config java.security.PrivateKey [pk]
(pem/private-key->pem pk))
(defn config->env
"Creates a map of env vars from the config. This is done by flattening
the entries and prepending them with `monkeyci-`. Values are converted
to string."
[c]
(->> c
(flatten-nested [])
(mc/map-keys (fn [k]
(keyword (str env-prefix "-" (name k)))))
(mc/map-vals serialize-config)))
(defn normalize-typed
"Convenience function that converts the `:type` of an entry into a keyword and
then invokes `f` on it."
[k conf f]
(-> conf
(update k keywordize-type)
(f)))