-
Notifications
You must be signed in to change notification settings - Fork 1.4k
/
reducers.clj
334 lines (294 loc) · 9.43 KB
/
reducers.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
; Copyright (c) Rich Hickey. All rights reserved.
; The use and distribution terms for this software are covered by the
; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
; which can be found in the file epl-v10.html at the root of this distribution.
; By using this software in any fashion, you are agreeing to be bound by
; the terms of this license.
; You must not remove this notice, or any other, from this software.
(ns ^{:doc
"A library for reduction and parallel folding. Alpha and subject
to change."
:author "Rich Hickey"}
clojure.core.reducers
(:refer-clojure :exclude [reduce map mapcat filter remove take take-while drop flatten cat])
(:require [clojure.walk :as walk]))
(alias 'core 'clojure.core)
(set! *warn-on-reflection* true)
;;;;;;;;;;;;;; some fj stuff ;;;;;;;;;;
(def pool (delay (java.util.concurrent.ForkJoinPool.)))
(defn fjtask [^Callable f]
(java.util.concurrent.ForkJoinTask/adapt f))
(defn- fjinvoke [f]
(if (java.util.concurrent.ForkJoinTask/inForkJoinPool)
(f)
(.invoke ^java.util.concurrent.ForkJoinPool @pool ^java.util.concurrent.ForkJoinTask (fjtask f))))
(defn- fjfork [task] (.fork ^java.util.concurrent.ForkJoinTask task))
(defn- fjjoin [task] (.join ^java.util.concurrent.ForkJoinTask task))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn reduce
"Like core/reduce except:
When init is not provided, (f) is used.
Maps are reduced with reduce-kv"
([f coll] (reduce f (f) coll))
([f init coll]
(if (instance? java.util.Map coll)
(clojure.core.protocols/kv-reduce coll f init)
(clojure.core.protocols/coll-reduce coll f init))))
(defprotocol CollFold
(coll-fold [coll n combinef reducef]))
(defn fold
"Reduces a collection using a (potentially parallel) reduce-combine
strategy. The collection is partitioned into groups of approximately
n (default 512), each of which is reduced with reducef (with a seed
value obtained by calling (combinef) with no arguments). The results
of these reductions are then reduced with combinef (default
reducef). combinef must be associative, and, when called with no
arguments, (combinef) must produce its identity element. These
operations may be performed in parallel, but the results will
preserve order."
{:added "1.5"}
([reducef coll] (fold reducef reducef coll))
([combinef reducef coll] (fold 512 combinef reducef coll))
([n combinef reducef coll]
(coll-fold coll n combinef reducef)))
(defn reducer
"Given a reducible collection, and a transformation function xf,
returns a reducible collection, where any supplied reducing
fn will be transformed by xf. xf is a function of reducing fn to
reducing fn."
{:added "1.5"}
([coll xf]
(reify
clojure.core.protocols/CollReduce
(coll-reduce [this f1]
(clojure.core.protocols/coll-reduce this f1 (f1)))
(coll-reduce [_ f1 init]
(clojure.core.protocols/coll-reduce coll (xf f1) init)))))
(defn folder
"Given a foldable collection, and a transformation function xf,
returns a foldable collection, where any supplied reducing
fn will be transformed by xf. xf is a function of reducing fn to
reducing fn."
{:added "1.5"}
([coll xf]
(reify
clojure.core.protocols/CollReduce
(coll-reduce [_ f1]
(clojure.core.protocols/coll-reduce coll (xf f1) (f1)))
(coll-reduce [_ f1 init]
(clojure.core.protocols/coll-reduce coll (xf f1) init))
CollFold
(coll-fold [_ n combinef reducef]
(coll-fold coll n combinef (xf reducef))))))
(defn- do-curried
[name doc meta args body]
(let [cargs (vec (butlast args))]
`(defn ~name ~doc ~meta
(~cargs (fn [x#] (~name ~@cargs x#)))
(~args ~@body))))
(defmacro ^:private defcurried
"Builds another arity of the fn that returns a fn awaiting the last
param"
[name doc meta args & body]
(do-curried name doc meta args body))
(defn- do-rfn [f1 k fkv]
`(fn
([] (~f1))
~(clojure.walk/postwalk
#(if (sequential? %)
((if (vector? %) vec identity)
(core/remove #{k} %))
%)
fkv)
~fkv))
(defmacro ^:private rfn
"Builds 3-arity reducing fn given names of wrapped fn and key, and k/v impl."
[[f1 k] fkv]
(do-rfn f1 k fkv))
(defcurried map
"Applies f to every value in the reduction of coll. Foldable."
{:added "1.5"}
[f coll]
(folder coll
(fn [f1]
(rfn [f1 k]
([ret k v]
(f1 ret (f k v)))))))
(defcurried mapcat
"Applies f to every value in the reduction of coll, concatenating the result
colls of (f val). Foldable."
{:added "1.5"}
[f coll]
(folder coll
(fn [f1]
(let [f1 (fn
([ret v]
(let [x (f1 ret v)] (if (reduced? x) (reduced x) x)))
([ret k v]
(let [x (f1 ret k v)] (if (reduced? x) (reduced x) x))))]
(rfn [f1 k]
([ret k v]
(reduce f1 ret (f k v))))))))
(defcurried filter
"Retains values in the reduction of coll for which (pred val)
returns logical true. Foldable."
{:added "1.5"}
[pred coll]
(folder coll
(fn [f1]
(rfn [f1 k]
([ret k v]
(if (pred k v)
(f1 ret k v)
ret))))))
(defcurried remove
"Removes values in the reduction of coll for which (pred val)
returns logical true. Foldable."
{:added "1.5"}
[pred coll]
(filter (complement pred) coll))
(defcurried flatten
"Takes any nested combination of sequential things (lists, vectors,
etc.) and returns their contents as a single, flat foldable
collection."
{:added "1.5"}
[coll]
(folder coll
(fn [f1]
(fn
([] (f1))
([ret v]
(if (sequential? v)
(clojure.core.protocols/coll-reduce (flatten v) f1 ret)
(f1 ret v)))))))
(defcurried take-while
"Ends the reduction of coll when (pred val) returns logical false."
{:added "1.5"}
[pred coll]
(reducer coll
(fn [f1]
(rfn [f1 k]
([ret k v]
(if (pred k v)
(f1 ret k v)
(reduced ret)))))))
(defcurried take
"Ends the reduction of coll after consuming n values."
{:added "1.5"}
[n coll]
(reducer coll
(fn [f1]
(let [cnt (atom n)]
(rfn [f1 k]
([ret k v]
(swap! cnt dec)
(if (neg? @cnt)
(reduced ret)
(f1 ret k v))))))))
(defcurried drop
"Elides the first n values from the reduction of coll."
{:added "1.5"}
[n coll]
(reducer coll
(fn [f1]
(let [cnt (atom n)]
(rfn [f1 k]
([ret k v]
(swap! cnt dec)
(if (neg? @cnt)
(f1 ret k v)
ret)))))))
;;do not construct this directly, use cat
(deftype Cat [cnt left right]
clojure.lang.Counted
(count [_] cnt)
clojure.lang.Seqable
(seq [_] (concat (seq left) (seq right)))
clojure.core.protocols/CollReduce
(coll-reduce [this f1] (clojure.core.protocols/coll-reduce this f1 (f1)))
(coll-reduce
[_ f1 init]
(clojure.core.protocols/coll-reduce
right f1
(clojure.core.protocols/coll-reduce left f1 init)))
CollFold
(coll-fold
[_ n combinef reducef]
(fjinvoke
(fn []
(let [rt (fjfork (fjtask #(coll-fold right n combinef reducef)))]
(combinef
(coll-fold left n combinef reducef)
(fjjoin rt)))))))
(defn cat
"A high-performance combining fn that yields the catenation of the
reduced values. The result is reducible, foldable, seqable and
counted, providing the identity collections are reducible, seqable
and counted. The single argument version will build a combining fn
with the supplied identity constructor. Tests for identity
with (zero? (count x)). See also foldcat."
{:added "1.5"}
([] (java.util.ArrayList.))
([ctor]
(fn
([] (ctor))
([left right] (cat left right))))
([left right]
(cond
(zero? (count left)) right
(zero? (count right)) left
:else
(Cat. (+ (count left) (count right)) left right))))
(defn append!
".adds x to acc and returns acc"
{:added "1.5"}
[^java.util.Collection acc x]
(doto acc (.add x)))
(defn foldcat
"Equivalent to (fold cat append! coll)"
{:added "1.5"}
[coll]
(fold cat append! coll))
(defn monoid
"Builds a combining fn out of the supplied operator and identity
constructor. op must be associative and ctor called with no args
must return an identity value for it."
{:added "1.5"}
[op ctor]
(fn m
([] (ctor))
([a b] (op a b))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; fold impls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn- foldvec
[v n combinef reducef]
(cond
(empty? v) (combinef)
(<= (count v) n) (reduce reducef (combinef) v)
:else
(let [split (quot (count v) 2)
v1 (subvec v 0 split)
v2 (subvec v split (count v))
fc (fn [child] #(foldvec child n combinef reducef))]
(fjinvoke
#(let [f1 (fc v1)
t2 (fjtask (fc v2))]
(fjfork t2)
(combinef (f1) (fjjoin t2)))))))
(extend-protocol CollFold
nil
(coll-fold
[coll n combinef reducef]
(combinef))
Object
(coll-fold
[coll n combinef reducef]
;;can't fold, single reduce
(reduce reducef (combinef) coll))
clojure.lang.IPersistentVector
(coll-fold
[v n combinef reducef]
(foldvec v n combinef reducef))
clojure.lang.PersistentHashMap
(coll-fold
[m n combinef reducef]
(.fold m n combinef reducef fjinvoke fjtask fjfork fjjoin)))