Permalink
Newer
Older
100644 91 lines (73 sloc) 3.1 KB
1
; Copyright (c) Chris Houser, April 2008. All rights reserved.
2
; The use and distribution terms for this software are covered by the
3
; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
4
; which can be found in the file epl-v10.html at the root of this distribution.
5
; By using this software in any fashion, you are agreeing to be bound by
6
; the terms of this license.
7
; You must not remove this notice, or any other, from this software.
8
9
; Functions for memory-mapping files, plus some functions that use a
10
; mmaped file for "normal" activies -- slurp, load-file, etc.
11
12
(ns
13
#^{:author "Chris Houser",
14
:doc "Functions for memory-mapping files, plus some functions that use a
15
mmaped file for \"normal\" activies -- slurp, load-file, etc."}
16
clojure.contrib.mmap
17
(:refer-clojure :exclude (slurp load-file))
18
(:import (java.nio ByteBuffer CharBuffer)
19
(java.io PushbackReader InputStream InputStreamReader
20
FileInputStream)))
21
22
;(set! *warn-on-reflection* true)
23
24
(def READ_ONLY #^{:private true}
25
(java.nio.channels.FileChannel$MapMode/READ_ONLY))
26
27
(defn mmap
28
"Memory-map the file named f. Returns a ByteBuffer."
29
[f]
30
(let [channel (.getChannel (FileInputStream. f))]
31
(.map channel READ_ONLY 0 (.size channel))))
32
33
(defn slurp
34
"Reads the file named by f and returns it as a string."
35
[#^String f]
36
(.. java.nio.charset.Charset (forName "UTF-8")
37
(newDecoder) (decode (mmap f))))
38
39
(defn buffer-stream
40
"Returns an InputStream for a ByteBuffer, such as returned by mmap."
41
[#^ByteBuffer buf]
42
(proxy [InputStream] []
43
(available [] (.remaining buf))
44
(read
45
([] (if (.hasRemaining buf) (.get buf) -1))
46
([dst offset len] (let [actlen (min (.remaining buf) len)]
47
(.get buf dst offset actlen)
48
(if (< actlen 1) -1 actlen))))))
49
50
(defn load-file [f]
51
"Like clojure.lang/load-file, but uses mmap internally."
52
(with-open [rdr (-> f mmap buffer-stream InputStreamReader. PushbackReader.)]
53
(load-reader rdr)))
54
55
56
(comment
57
58
(alias 'mmap 'clojure.contrib.mmap)
59
(alias 'core 'clojure.core)
60
61
;---
62
; zip_filter.clj is 95KB
63
(def tf "/home/chouser/build/clojure/src/clj/clojure/core.clj")
64
(println "\nload-file" tf)
65
(time (dotimes [_ 5] (core/load-file tf))) ; 5420.177813 msecs
66
(time (dotimes [_ 5] (mmap/load-file tf))) ; 7946.854434 msecs -- not so good
67
68
;---
69
; kern.log.0 is 961KB
70
(def tf "/var/log/kern.log.0")
71
(println "\nslurp" tf)
72
(time (dotimes [_ 10] (.length (core/slurp tf)))) ; 435.767226 msecs
73
(time (dotimes [_ 10] (.length (mmap/slurp tf)))) ; 93.176858 msecs
74
75
;---
76
; kern.log.0 is 961KB
77
(def tf "/var/log/kern.log.0")
78
(println "\nregex slurp large" tf)
79
(time (dotimes [_ 10] (count (re-seq #"EXT3.*" (core/slurp tf))))) ; 416
80
(time (dotimes [_ 10] (count (re-seq #"EXT3.*" (mmap/slurp tf))))) ; 101
81
82
;---
83
; mmap.clj is about 3.1KB
84
(def tf "/home/chouser/proj/clojure-contrib/src/clojure/contrib/mmap.clj")
85
(println "\nregex slurp small" tf)
86
87
(time (dotimes [_ 1000] (count (re-seq #"defn \S*" (core/slurp tf))))) ; 308
88
(time (dotimes [_ 1000] (count (re-seq #"defn \S*" (mmap/slurp tf))))) ; 198
89
90
)