This repository has been archived by the owner on Apr 29, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
migrations.clj
233 lines (194 loc) · 8.41 KB
/
migrations.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
;; Very loosely patterned after https://github.com/yogthos/migratus
(ns oc.lib.db.migrations
"Migrate RethinkDB data."
(:require [clojure.string :as s]
[clj-time.core :as t]
[clj-time.coerce :as coerce]
[rethinkdb.query :as r]
[oc.lib.db.common :as db-common]
[oc.lib.slugify :as slug]))
;; ----- Utility functions for migrations -----
(defn- create-database
"Create a RethinkDB database if it doesn't already exist."
[conn db-name]
(if-let [db-list (r/run (r/db-list) conn)]
(if (some #(= db-name %) db-list)
true ; already exists, return truthy
(r/run (r/db-create db-name) conn))))
(defn table-list
"Return a sequence of the table names in the RethinkDB."
[conn]
(r/run (r/table-list) conn))
(defn migration-file-name [migrations-dir migration-name]
(str (s/join java.io.File/separator [migrations-dir migration-name]) ".edn"))
(defn- store-migration
"Keep the migration name in the migration table so we don't run it again."
[conn migration-name]
(assert (= 1 (:inserted (-> (r/table "migrations")
(r/insert {:name migration-name})
(r/run conn))))))
(defn run-migration
"Run the migration specified by the migration name."
[conn migrations-dir migration-name]
(println "\nRunning migration:" migration-name)
(let [file-name (migration-file-name migrations-dir migration-name)
bare-name (s/join "-" (rest (s/split migration-name #"_"))) ; strip the timestamp
function-ns (s/join "." (-> file-name
(s/replace #"_" "-") ; swap _ for -
(s/split #"\/")
(rest) ; remove ./
(rest) ; remove /src/
(vec)
(drop-last))) ; remove file name portion
function-name (str function-ns "." bare-name "/up")] ; function name
(println "Loading name:" file-name)
(load-file file-name)
(println "Running function:" function-name)
((ns-resolve *ns* (symbol function-name)) conn))) ; run the migration
(defn- run-migrations
"Given a list of migrations that haven't been run, run them. Abort if any doesn't succeed."
[conn migrations-dir migration-names]
(doseq [migration-name migration-names]
(assert (true? (run-migration conn migrations-dir migration-name)))
(store-migration conn migration-name))
:ok)
(defn- report-migrations
"Report on how many migrations need to be run."
[migrations]
(let [m-count (count migrations)]
(cond
(zero? m-count) (println "No new migrations to run.")
(= 1 m-count) (println "1 new migration to run.")
:else (println (str m-count " new migrations to run."))))
migrations)
(defn- new-migrations
"Given a list of migrations that exist, return just the ones that haven't been run on this DB."
[conn migrations]
(let [migration-slugs (set (map #(second (re-matches #".*\/(.*).edn$" %)) (map str migrations))) ; from the filesystem
existing-slugs (set (map :name (r/run (r/table "migrations") conn)))] ; from the DB
(sort (clojure.set/difference migration-slugs existing-slugs))))
;; ----- Utility functions for migration helpers -----
(defn- index-list
"Return a sequence of the index names for a table in the RethinkDB."
[conn table-name]
(-> (r/table table-name) (r/index-list) (r/run conn)))
(defn- wait-for-index
"Pause until an index with the specified name is finished being created."
[conn table-name index-name]
(-> (r/table table-name)
(r/index-wait index-name)
(r/run conn)))
;; ----- Helper functions for implementing migrations -----
(defn create-index
"Create RethinkDB table index for the specified field if it doesn't exist."
([conn table-name index-name] (create-index conn table-name index-name {}))
([conn table-name index-name options]
(when (not-any? #(= index-name %) (index-list conn table-name))
(-> (r/table table-name)
(r/index-create index-name nil options)
(r/run conn))
(wait-for-index conn table-name index-name))))
(defn create-compound-index
"Create RethinkDB table compound index for the specified fields if it doesn't exist."
([conn table-name index-name index-function]
(create-compound-index conn table-name index-name index-function nil))
([conn table-name index-name index-function options]
(when (not-any? #(= index-name %) (index-list conn table-name))
(-> (r/table table-name)
(r/index-create index-name index-function options)
(r/run conn))
(wait-for-index conn table-name index-name))))
(defn remove-index
"Remove a RethinkDB table index for the specified field if it exists."
[conn table-name index-name]
(when (some #(= index-name %) (index-list conn table-name))
(-> (r/table table-name)
(r/index-drop index-name)
(r/run conn))))
(defn create-table
"Create a RethinkDB table with the specified primary key if it doesn't exist."
[conn db-name table-name primary-key]
(when (not-any? #(= table-name %) (table-list conn))
(-> (r/db db-name)
(r/table-create table-name {:primary-key primary-key :durability "hard"})
(r/run conn))))
(defn rename-table
"Rename a RethinkDB table from the old name to the new name if it exists."
[conn db-name old-table-name new-table-name]
(let [table-list (table-list conn)]
(when (and (some #(= old-table-name %) table-list)
(not-any? #(= new-table-name %) table-list))
(-> (r/db "rethinkdb")
(r/table "table_config")
(r/filter (r/fn [table] {:db db-name :name old-table-name}))
(r/update {:name new-table-name})
(r/run conn)))))
(defn delete-table
"Delete the specified RethinkDB table."
[conn table-name]
(r/run (r/table-drop table-name) conn))
(defn remove-properties
"Remove the specified properties (sequence of strings or keywords) from all docs in the specified table."
[conn table-name props]
{:pre [(sequential? props)
(every? db-common/s-or-k? props)]}
(-> (r/table table-name)
(r/replace (r/fn [resource]
(r/without resource props)))
(r/run conn)))
;; ----- Main entry-point functions for creating and running migrations -----
(defn migrate
"
Create the database (if needed) and the migration table (if needed) and run any
migrations that haven't already been run on this DB.
"
([db-map migrations-dir]
{:pre [(map? db-map)
(string? migrations-dir)]}
(with-open [conn (apply r/connect (flatten (vec db-map)))]
(migrate conn (:db db-map) migrations-dir)))
([conn db-name migrations-dir]
{:pre [(instance? clojure.lang.IDeref conn)
(string? db-name)
(string? migrations-dir)]}
(println (str "\nInitializing database:" db-name))
;; create DB (if it doesn't exist)
(when (create-database conn db-name)
;; create migration table (if it doesn't exist)
(create-table conn db-name "migrations" "name")
;; Run the migrations
(println "\nRunning migrations.")
(->> (filter #(s/ends-with? % ".edn") (file-seq (clojure.java.io/file migrations-dir)))
(new-migrations conn)
report-migrations
(run-migrations conn migrations-dir))
(println "Migrations complete."))
(println "\nDatabase initialization complete.\n")))
(defn create
"Create a new migration with the current date and the name."
[migrations-dir migration-template provided-name]
(let [timestamp (str (coerce/to-long (t/now)))
migration-name (slug/slugify provided-name 256)
full-name (str timestamp "-" migration-name)
file-name (migration-file-name migrations-dir (s/replace full-name #"-" "_"))
template (slurp migration-template)
contents (s/replace template #"MIGRATION-NAME" migration-name)]
(spit file-name contents)))
(comment
;; REPL testing
(require '[rethinkdb.query :as r])
(require '[oc.lib.db.migrations :as m] :reload)
(def conn [:host "127.0.0.1" :port 28015 :db "open_company_storage_dev"])
(m/create "test-it")
(m/migrate)
(with-open [c (apply r/connect conn)]
(m/table-list c))
(with-open [c (apply r/connect conn)]
(m/create-table c "open_company_storage_dev" "foo" "bar"))
(with-open [c (apply r/connect conn)]
(m/create-index c "foo" "blat"))
(with-open [c (apply r/connect conn)]
(m/remove-index c "foo" "blat"))
(with-open [c (apply r/connect conn)]
(m/delete-table c "foo"))
)