-
Notifications
You must be signed in to change notification settings - Fork 3
/
evaluation_handler.clj
137 lines (109 loc) · 4.32 KB
/
evaluation_handler.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
(ns scicloj.metamorph.ml.evaluation-handler
(:require
[scicloj.metamorph.ml.tools :refer [dissoc-in pp-str multi-dissoc-in]]
[clojure.tools.reader :as tr]
[clojure.tools.reader.reader-types :as rts]
[clojure.java.classpath]
[clojure.repl :as repl]
[taoensso.nippy :as nippy])
(:import [java.io File]))
(defn file->topforms-with-metadata [path]
(->> path
slurp
rts/source-logging-push-back-reader
repeat
(map #(tr/read % false :EOF))
(take-while (partial not= :EOF))))
(defn resolve-keyword
"Interpret keyword or list as a symbol and try to resolve it."
[k pipe-ns]
(if (or (sequential? k) (map? k))
k
(-> (if-let [n (namespace k)] ;; namespaced?
(let [sn (symbol n)
n (str (get (ns-aliases pipe-ns) sn sn))] ;; try to find namespace in aliases
(symbol n (name k))) ;; create proper symbol with fixed namespace
(symbol (str pipe-ns) (name k))) ;; no namespace case
(resolve))))
(defn fns->code-list [pipeline-source-file]
(->>
(file->topforms-with-metadata pipeline-source-file)
(map (fn [form]
(when (sequential? form)
(hash-map :top-level (first form)
:form form
:form-str (pp-str form)))))
(filter some?)
(filter #(= 'defn (:top-level %)))
(map #(assoc % :fn-name (second (:form %))))))
(defn fns->code-map [pipeline-source-file]
(let [fns-code-list (fns->code-list pipeline-source-file)]
(zipmap
(map :fn-name fns-code-list)
fns-code-list)))
(defn fn-symbol->code [fn-symbol pipe-ns pipeline-source-file]
(:form-str
(get (fns->code-map pipeline-source-file)
(:name
(meta
(resolve-keyword fn-symbol pipe-ns))))))
(defn get-code [symbol pipe-ns pipeline-source-file]
;; (def symbol symbol)
(let [source (repl/source-fn symbol)
orig-source (some-> symbol
resolve
meta
:orig
clojure.repl/source-fn)]
;; resolve
;; #(.toSymbol %)
{:code-source (if orig-source orig-source source)
:code-local-source (fn-symbol->code symbol pipe-ns pipeline-source-file)}))
(defn get-classpath []
(->>
(clojure.java.classpath/classpath)
(map #(.getPath %))))
(defn get-fn-sources [qualified-pipe-decl pipe-ns pipeline-source-file]
(let [codes (atom {})]
(clojure.walk/postwalk (fn [keyword]
(when (keyword? keyword)
(let [symbol (symbol keyword)]
(swap! codes #(assoc % symbol
(get-code symbol pipe-ns pipeline-source-file)))))
keyword)
qualified-pipe-decl)
@codes))
(defn get-source-information [qualified-pipe-decl pipe-ns pipeline-source-file]
{:fn-sources (get-fn-sources qualified-pipe-decl pipe-ns pipeline-source-file)
:classpath (get-classpath)})
(defn example-nippy-handler [files output-dir result-reduce-fn]
(fn [result]
(let [
freezable-result
(-> result
(multi-dissoc-in
[
[:pipe-fn]
[:metric-fn]]))
temp-file (str output-dir "/" ( java.util.UUID/randomUUID) ".nippy")
_ (swap! files #(conj % temp-file))]
(nippy/freeze-to-file temp-file freezable-result)
(result-reduce-fn result))))
(defn qualify-keywords [pipe-decl pipe-ns]
(clojure.walk/postwalk (fn [form]
;; (println form)
(if-let [resolved (resolve-keyword form pipe-ns)]
(do
(if (var? resolved)
(let [v (var-get resolved)
k (keyword
(-> resolved meta :ns str)
(-> resolved meta :name str))]
k)
resolved))
form))
pipe-decl))
(defn qualify-pipelines [pipe-decls pipe-ns]
(mapv #(qualify-keywords % pipe-ns) pipe-decls))
(comment
(def x (nippy/thaw-from-file "/tmp/f29648a6-9a73-4cb4-a7df-27e868b5bccf.nippy")))