-
Notifications
You must be signed in to change notification settings - Fork 4
/
metadata.clj
459 lines (400 loc) · 20.6 KB
/
metadata.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
(ns clj-jargon.metadata
(:use [clj-jargon.validations]
[clj-jargon.item-info :only [object-type]])
(:require [clojure.string :as string]
[otel.otel :as otel]
[slingshot.slingshot :refer [throw+ try+]]
[clojure-commons.error-codes :refer [ERR_NOT_WRITEABLE]])
(:import [org.irods.jargon.core.exception CatNoAccessException]
[org.irods.jargon.core.pub DataObjectAO
CollectionAO
IRODSGenQueryExecutor]
[org.irods.jargon.core.pub.domain AvuData
Collection]
[org.irods.jargon.core.query IRODSGenQueryBuilder
IRODSQueryResultRow
QueryConditionOperators
RodsGenQueryEnum
AVUQueryElement
AVUQueryElement$AVUQueryPart
AVUQueryOperatorEnum
MetaDataAndDomainData]))
(def max-gen-query-results 50000)
(defn map2avu
"Converts an avu map into an AvuData instance."
[avu-map]
(AvuData/instance (:attr avu-map) (:value avu-map) (:unit avu-map)))
(defn- avu2map
[^MetaDataAndDomainData avu]
(hash-map :attr (.getAvuAttribute avu)
:value (.getAvuValue avu)
:unit (.getAvuUnit avu)))
(defn get-metadata
"Returns all of the metadata associated with a path."
[{^DataObjectAO data-ao :dataObjectAO
^CollectionAO collection-ao :collectionAO
:as cm}
^String dir-path & {:keys [known-type] :or {known-type nil}}]
(otel/with-span [s ["get-metadata"]]
(validate-path-lengths dir-path)
(mapv avu2map
(case (or known-type (object-type cm dir-path))
:dir (.findMetadataValuesForCollection collection-ao dir-path)
:file (.findMetadataValuesForDataObject data-ao dir-path)))))
(defn- get-metadata-by-query
[{^DataObjectAO data-ao :dataObjectAO
^CollectionAO collection-ao :collectionAO
:as cm} path query & {:keys [known-type] :or {known-type nil}}]
(otel/with-span [s ["get-metadata-by-query"]]
(validate-path-lengths path)
(mapv avu2map
(case (or known-type (object-type cm path))
:dir (.findMetadataValuesByMetadataQueryForCollection collection-ao query path)
:file (.findMetadataValuesForDataObjectUsingAVUQuery data-ao query path)))))
(defn get-attribute
"Returns a list of avu maps for a specific attribute associated with dir-path"
[{^DataObjectAO data-ao :dataObjectAO
^CollectionAO collection-ao :collectionAO
:as cm} dir-path attr & {:keys [known-type] :or {known-type nil}}]
(otel/with-span [s ["get-attribute" {:attributes {"path" dir-path
"attribute" attr}}]]
(let [query [(AVUQueryElement/instanceForValueQuery
AVUQueryElement$AVUQueryPart/ATTRIBUTE
AVUQueryOperatorEnum/EQUAL
attr)]]
(get-metadata-by-query cm dir-path query :known-type known-type))))
(defn get-attribute-value
[{^DataObjectAO data-ao :dataObjectAO
^CollectionAO collection-ao :collectionAO
:as cm} apath attr val & {:keys [known-type] :or {known-type nil}}]
(otel/with-span [s ["get-attribute-value" {:attributes {"path" apath
"attribute" attr
"value" (str val)}}]]
(let [query [(AVUQueryElement/instanceForValueQuery
AVUQueryElement$AVUQueryPart/ATTRIBUTE
AVUQueryOperatorEnum/EQUAL
attr) (AVUQueryElement/instanceForValueQuery
AVUQueryElement$AVUQueryPart/VALUE
AVUQueryOperatorEnum/EQUAL
(str val))]]
(get-metadata-by-query cm apath query :known-type known-type))))
(defn attribute?
"Returns true if the path has the associated attribute."
[cm dir-path attr & {:keys [known-type] :or {known-type nil}}]
(pos? (count (get-attribute cm dir-path attr :known-type known-type))))
(defn attr-value?
"Returns a truthy value if path has metadata that has an attribute of attr and
a value of val."
([cm path attr val & {:keys [known-type] :or {known-type nil}}]
(pos? (count (get-attribute-value cm path attr val :known-type known-type))))
([metadata attr val]
(-> (filter
#(and (= (:attr %1) attr)
(= (:value %1) val))
metadata)
count
pos?)))
(defmulti add-avu
(fn [ao-obj dir-path avu] (type ao-obj)))
(defmethod add-avu CollectionAO
[^CollectionAO ao-obj ^String dir-path ^AvuData avu]
(.addAVUMetadata ao-obj dir-path avu))
(defmethod add-avu DataObjectAO
[^DataObjectAO ao-obj ^String dir-path ^AvuData avu]
(.addAVUMetadata ao-obj dir-path avu))
(defmulti modify-avu
(fn [ao-obj dir-path old-avu avu] (type ao-obj)))
(defmethod modify-avu CollectionAO
[^CollectionAO ao-obj ^String dir-path ^AvuData old-avu ^AvuData avu]
(.modifyAVUMetadata ao-obj dir-path old-avu avu))
(defmethod modify-avu DataObjectAO
[^DataObjectAO ao-obj ^String dir-path ^AvuData old-avu ^AvuData avu]
(.modifyAVUMetadata ao-obj dir-path old-avu avu))
(defmulti delete-avu
(fn [ao-obj dir-path avu] (type ao-obj)))
(defmethod delete-avu CollectionAO
[^CollectionAO ao-obj ^String dir-path ^AvuData avu]
(.deleteAVUMetadata ao-obj dir-path avu))
(defmethod delete-avu DataObjectAO
[^DataObjectAO ao-obj ^String dir-path ^AvuData avu]
(.deleteAVUMetadata ao-obj dir-path avu))
(defn add-metadata
[cm dir-path attr value unit & {:keys [known-type] :or {known-type nil}}]
(validate-path-lengths dir-path)
(try+
(let [ao-obj (case (or known-type (object-type cm dir-path))
:dir (:collectionAO cm)
:file (:dataObjectAO cm))]
(add-avu ao-obj dir-path (AvuData/instance attr value unit)))
(catch CatNoAccessException _
(throw+ {:error_code ERR_NOT_WRITEABLE :path dir-path}))))
(defn set-metadata
"Sets an avu for dir-path."
[cm dir-path attr value unit & {:keys [known-type] :or {known-type nil}}]
(validate-path-lengths dir-path)
(let [avu (AvuData/instance attr value unit)
ao-obj (case (or known-type (object-type cm dir-path))
:dir (:collectionAO cm)
:file (:dataObjectAO cm))]
(if (zero? (count (get-attribute cm dir-path attr)))
(add-avu ao-obj dir-path avu)
(let [old-avu (map2avu (first (get-attribute cm dir-path attr)))]
(modify-avu ao-obj dir-path old-avu avu)))))
(defn- delete-meta
[cm dir-path attr-func & {:keys [known-type] :or {known-type nil}}]
(validate-path-lengths dir-path)
(let [fattr (first (attr-func))
avu (map2avu fattr)
ao-obj (case (or known-type (object-type cm dir-path))
:dir (:collectionAO cm)
:file (:dataObjectAO cm))]
(delete-avu ao-obj dir-path avu)))
(defn delete-metadata
([cm dir-path attr]
(delete-meta cm dir-path #(get-attribute cm dir-path attr)))
([cm dir-path attr val]
(delete-meta cm dir-path #(get-attribute-value cm dir-path attr val))))
(defn delete-avus
[cm dir-path avu-maps & {:keys [known-type] :or {known-type nil}}]
(validate-path-lengths dir-path)
(let [ao (case (or known-type (object-type cm dir-path))
:dir (:collectionAO cm)
:file (:dataObjectAO cm))]
(doseq [avu-map avu-maps]
(when (attr-value? cm dir-path (:attr avu-map) (:value avu-map))
(delete-avu ao dir-path (map2avu avu-map))))))
(defn- ^QueryConditionOperators op->constant
[op]
(or ({:between QueryConditionOperators/BETWEEN
:= QueryConditionOperators/EQUAL
:> QueryConditionOperators/GREATER_THAN
:>= QueryConditionOperators/GREATER_THAN_OR_EQUAL_TO
:in QueryConditionOperators/IN
:< QueryConditionOperators/LESS_THAN
:<= QueryConditionOperators/LESS_THAN_OR_EQUAL_TO
:like QueryConditionOperators/LIKE
:not= QueryConditionOperators/NOT_EQUAL
:not-like QueryConditionOperators/NOT_LIKE
:num= QueryConditionOperators/NUMERIC_EQUAL
:num> QueryConditionOperators/NUMERIC_GREATER_THAN
:num>= QueryConditionOperators/NUMERIC_GREATER_THAN_OR_EQUAL_TO
:num< QueryConditionOperators/NUMERIC_LESS_THAN
:num<= QueryConditionOperators/NUMERIC_LESS_THAN_OR_EQUAL_TO
:sounds-like QueryConditionOperators/SOUNDS_LIKE
:sounds-not-like QueryConditionOperators/SOUNDS_NOT_LIKE
:table QueryConditionOperators/TABLE} op)
(throw (Exception. (str "unknown operator: " op)))))
(defn- build-file-avu-query
[^String name op value]
(-> (IRODSGenQueryBuilder. true nil)
(.addSelectAsGenQueryValue RodsGenQueryEnum/COL_COLL_NAME)
(.addSelectAsGenQueryValue RodsGenQueryEnum/COL_DATA_NAME)
(.addConditionAsGenQueryField RodsGenQueryEnum/COL_META_DATA_ATTR_NAME
QueryConditionOperators/EQUAL name)
(.addConditionAsGenQueryField RodsGenQueryEnum/COL_META_DATA_ATTR_VALUE
(op->constant op)
(str value))
(.exportIRODSQueryFromBuilder max-gen-query-results)))
(defn- build-file-attr-query
[^String name]
(-> (IRODSGenQueryBuilder. true nil)
(.addSelectAsGenQueryValue RodsGenQueryEnum/COL_COLL_NAME)
(.addSelectAsGenQueryValue RodsGenQueryEnum/COL_DATA_NAME)
(.addConditionAsGenQueryField RodsGenQueryEnum/COL_META_DATA_ATTR_NAME
QueryConditionOperators/EQUAL name)
(.exportIRODSQueryFromBuilder max-gen-query-results)))
(defn- format-result
[^IRODSQueryResultRow rr]
(string/join "/" (.getColumnsAsList rr)))
(defn list-files-with-attr
[{^IRODSGenQueryExecutor executor :executor} attr]
(let [query (build-file-attr-query attr)
rs (.executeIRODSQueryAndCloseResult executor query 0)]
(map format-result (.getResults rs))))
(defn list-files-with-avu
[{^IRODSGenQueryExecutor executor :executor} n op value]
(otel/with-span [s ["list-files-with-avu" {:attributes {"name" n "op" (name op) "value" (str value)}}]]
(let [query (build-file-avu-query n op value)
rs (.executeIRODSQueryAndCloseResult executor query 0)]
(map format-result (.getResults rs)))))
(def ^:private file-avu-query-columns
{:name RodsGenQueryEnum/COL_META_DATA_ATTR_NAME
:value RodsGenQueryEnum/COL_META_DATA_ATTR_VALUE
:unit RodsGenQueryEnum/COL_META_DATA_ATTR_UNITS})
(def ^:private dir-avu-query-columns
{:name RodsGenQueryEnum/COL_META_COLL_ATTR_NAME
:value RodsGenQueryEnum/COL_META_COLL_ATTR_VALUE
:unit RodsGenQueryEnum/COL_META_COLL_ATTR_UNITS})
(defn- add-conditions-from-avu-spec
"Adds conditions from an AVU specification to a general query builder. The query specification
is a map in the following format:
{:name \"name\"
:value \"value\"
:unit \"unit\"}
The values in the map are strings indicating the name, value or unit of the AVUs to match. Each
entry in the map is optional, so that the caller can search for any combination of name and
value. For example, to search for AVUs named 'foo', the AVU specification would simply be
{:name \"foo\"}. Unrecognized keys in the AVU specification are currently ignored and conditions
are not added for null values."
[cols ^IRODSGenQueryBuilder builder avu-spec]
(->> (remove (comp nil? last) avu-spec)
(map (fn [[k v]] [(cols k) v]))
(remove (comp nil? first))
(map
(fn [[^RodsGenQueryEnum col ^String v]]
(.addConditionAsGenQueryField builder col QueryConditionOperators/EQUAL v)))
(dorun)))
(defn- build-subtree-query-from-avu-spec
"Builds a subtree query from a path and an AVU specification. The AVU specification is a map
in the following format:
{:name \"name\"
:value \"value\"
:unit \"unit\"}
The values in the map are strings indicating the name, value or unit of the AVUs to match. Each
entry in the map is optional, so that the caller can search for any combination of name and
value. For example, to search for AVUs named 'foo', the AVU specification would simply be
{:name \"foo\"}. Unrecognized keys in the AVU specification are currently ignored and conditions
are not added for null values.
The path is the absolute path to the root of the subtree to search. Items that are not in this
directory or any of its descendants will not be matched. The root of the subtree is included
in the search."
[select-columns condition-columns path avu-spec]
(let [builder (IRODSGenQueryBuilder. true nil)]
(dorun (map #(.addSelectAsGenQueryValue builder %) select-columns))
(when path
(.addConditionAsGenQueryField builder
RodsGenQueryEnum/COL_COLL_NAME
QueryConditionOperators/LIKE
(str path \%)))
(add-conditions-from-avu-spec condition-columns builder avu-spec)
(.exportIRODSQueryFromBuilder builder max-gen-query-results)))
(defn- list-items-in-tree-with-attr
"Lists either files or directories in a subtree given the path to the root of the subtree and an
AVU specification. The AVU specification is a map in the following format:
{:name \"name\"
:value \"value\"
:unit \"unit\"}
The values in the map are strings indicating the name, value or unit of the AVUs to match. Each
entry in the map is optional, so that the caller can search for any combination of name and
value. For example, to search for AVUs named 'foo', the AVU specification would simply be
{:name \"foo\"}. Unrecognized keys in the AVU specification are currently ignored and conditions
are not added for null values.
The path is the absolute path to the root of the subtree to search. Items that are not in this
directory or any of its descendants will not be matched. The root of the subtree is included
in the search.
The select-columns parameter indicates which columns should be selected from the query. The
condition-columns parameter is a map indicating which constants to use in the query for the
:name, :value, and :unit elements of the AVU specification. The format-row parameter is a
function that can be used to format each row in the result set. The single parameter to this
function is an instance of IRODSQueryResultRow."
[select-columns condition-columns format-row {^IRODSGenQueryExecutor executor :executor} path avu-spec]
(let [query (build-subtree-query-from-avu-spec select-columns condition-columns path avu-spec)]
(->> (.executeIRODSQueryAndCloseResult executor query 0)
(.getResults)
(mapv format-row))))
(def list-files-in-tree-with-attr
"Lists the paths to files in a subtree given the path to the root of the subtree and an AVU
specification. The AVU specification is a map in the following format:
{:name \"name\"
:value \"value\"
:unit \"unit\"}
The values in the map are strings indicating the name, value or unit of the AVUs to match. Each
entry in the map is optional, so that the caller can search for any combination of name and
value. For example, to search for AVUs named 'foo', the AVU specification would simply be
{:name \"foo\"}. Unrecognized keys in the AVU specification are currently ignored and conditions
are not added for null values.
The path is the absolute path to the root of the subtree to search. Items that are not in this
directory or any of its descendants will not be matched. The root of the subtree is included
in the search."
(partial list-items-in-tree-with-attr
[RodsGenQueryEnum/COL_COLL_NAME RodsGenQueryEnum/COL_DATA_NAME]
file-avu-query-columns
format-result))
(def list-collections-in-tree-with-attr
"Lists the paths to directories in a subtree given the path to the root of the subtree and an
AVU specification. The AVU specification is a map in the following format:
{:name \"name\"
:value \"value\"
:unit \"unit\"}
The values in the map are strings indicating the name, value or unit of the AVUs to match. Each
entry in the map is optional, so that the caller can search for any combination of name and
value. For example, to search for AVUs named 'foo', the AVU specification would simply be
{:name \"foo\"}. Unrecognized keys in the AVU specification are currently ignored and conditions
are not added for null values.
The path is the absolute path to the root of the subtree to search. Items that are not in this
directory or any of its descendants will not be matched. The root of the subtree is included
in the search."
(partial list-items-in-tree-with-attr
[RodsGenQueryEnum/COL_COLL_NAME]
dir-avu-query-columns
(fn [^IRODSQueryResultRow rr] (str (first (.getColumnsAsList rr))))))
(defn list-everything-in-tree-with-attr
"Lists the paths to both files and directories in a subtree given the path to the root of the
subtree and an AVU specification. The AVU specification is a map in the following format:
{:name \"name\"
:value \"value\"
:unit \"unit\"}
The values in the map are strings indicating the name, value or unit of the AVUs to match. Each
entry in the map is optional, so that the caller can search for any combination of name and
value. For example, to search for AVUs named 'foo', the AVU specification would simply be
{:name \"foo\"}. Unrecognized keys in the AVU specification are currently ignored and conditions
are not added for null values.
The path is the absolute path to the root of the subtree to search. Items that are not in this
directory or any of its descendants will not be matched. The root of the subtree is included
in the search."
[cm path avu-spec]
(doall (mapcat #(% cm path avu-spec)
[list-collections-in-tree-with-attr list-files-in-tree-with-attr])))
(defn get-avus-by-collection
"Returns AVUs associated with a collection that have the given attribute and value."
[{^CollectionAO collection-ao :collectionAO} file-path attr units]
(let [query [(AVUQueryElement/instanceForValueQuery
AVUQueryElement$AVUQueryPart/UNITS
AVUQueryOperatorEnum/EQUAL
units)
(AVUQueryElement/instanceForValueQuery
AVUQueryElement$AVUQueryPart/ATTRIBUTE
AVUQueryOperatorEnum/EQUAL
attr)]]
(mapv avu2map
(.findMetadataValuesByMetadataQueryForCollection collection-ao query file-path))))
(defn- get-coll-name
[^Collection coll]
(.getCollectionName coll))
(defn list-collections-with-attr-units
[{^CollectionAO collection-ao :collectionAO} attr units]
(let [query [(AVUQueryElement/instanceForValueQuery
AVUQueryElement$AVUQueryPart/UNITS
AVUQueryOperatorEnum/EQUAL
units)
(AVUQueryElement/instanceForValueQuery
AVUQueryElement$AVUQueryPart/ATTRIBUTE
AVUQueryOperatorEnum/EQUAL
attr)]]
(mapv get-coll-name
(.findDomainByMetadataQuery collection-ao query))))
(defn list-collections-with-attr-value
[{^CollectionAO collection-ao :collectionAO} attr value]
(otel/with-span [s ["list-collections-with-attr-value" {:attributes {"attribute" attr "value" (str value)}}]]
(let [query [(AVUQueryElement/instanceForValueQuery
AVUQueryElement$AVUQueryPart/VALUE
AVUQueryOperatorEnum/EQUAL
(str value))
(AVUQueryElement/instanceForValueQuery
AVUQueryElement$AVUQueryPart/ATTRIBUTE
AVUQueryOperatorEnum/EQUAL
attr)]]
(mapv get-coll-name
(.findDomainByMetadataQuery collection-ao query)))))
(defn list-everything-with-attr-value
"Generates a sequence of all collections and data objects with a given attribute having a given
value.
Parameters:
cm - the connected jargon context
attr - the name of the attribute
value - the value of the attribute
Returns:
It returns a sequence of collections and data object paths."
[cm attr value]
(otel/with-span [s ["list-everything-with-attr-value" {:attributes {"attribute" attr "value" (str value)}}]]
(concat (list-collections-with-attr-value cm attr value) (list-files-with-avu cm attr := value))))