Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

file 90 lines (73 sloc) 3.178 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
; Copyright (c) Chris Houser, April 2008. All rights reserved.
; The use and distribution terms for this software are covered by the
; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
; which can be found in the file epl-v10.html at the root of this distribution.
; By using this software in any fashion, you are agreeing to be bound by
; the terms of this license.
; You must not remove this notice, or any other, from this software.

; Functions for memory-mapping files, plus some functions that use a
; mmaped file for "normal" activies -- slurp, load-file, etc.

(ns
  #^{:author "Chris Houser",
     :doc "Functions for memory-mapping files, plus some functions that use a
mmaped file for \"normal\" activies -- slurp, load-file, etc."}
  clojure.contrib.mmap
    (:refer-clojure :exclude (slurp load-file))
    (:import (java.nio ByteBuffer CharBuffer)
             (java.io PushbackReader InputStream InputStreamReader
                      FileInputStream)))

;(set! *warn-on-reflection* true)

(def READ_ONLY #^{:private true}
  (java.nio.channels.FileChannel$MapMode/READ_ONLY))

(defn mmap
  "Memory-map the file named f. Returns a ByteBuffer."
  [f]
  (let [channel (.getChannel (FileInputStream. f))]
    (.map channel READ_ONLY 0 (.size channel))))

(defn slurp
  "Reads the file named by f and returns it as a string."
  [#^String f]
  (.. java.nio.charset.Charset (forName "UTF-8")
      (newDecoder) (decode (mmap f))))

(defn buffer-stream
  "Returns an InputStream for a ByteBuffer, such as returned by mmap."
  [#^ByteBuffer buf]
  (proxy [InputStream] []
    (available [] (.remaining buf))
    (read
      ([] (if (.hasRemaining buf) (.get buf) -1))
      ([dst offset len] (let [actlen (min (.remaining buf) len)]
                          (.get buf dst offset actlen)
                          (if (< actlen 1) -1 actlen))))))

(defn load-file [f]
  "Like clojure.lang/load-file, but uses mmap internally."
  (with-open [rdr (-> f mmap buffer-stream InputStreamReader. PushbackReader.)]
    (load-reader rdr)))


(comment

(alias 'mmap 'clojure.contrib.mmap)
(alias 'core 'clojure.core)

;---
; zip_filter.clj is 95KB
(def tf "/home/chouser/build/clojure/src/clj/clojure/core.clj")
(println "\nload-file" tf)
(time (dotimes [_ 5] (core/load-file tf))) ; 5420.177813 msecs
(time (dotimes [_ 5] (mmap/load-file tf))) ; 7946.854434 msecs -- not so good

;---
; kern.log.0 is 961KB
(def tf "/var/log/kern.log.0")
(println "\nslurp" tf)
(time (dotimes [_ 10] (.length (core/slurp tf)))) ; 435.767226 msecs
(time (dotimes [_ 10] (.length (mmap/slurp tf)))) ; 93.176858 msecs

;---
; kern.log.0 is 961KB
(def tf "/var/log/kern.log.0")
(println "\nregex slurp large" tf)
(time (dotimes [_ 10] (count (re-seq #"EXT3.*" (core/slurp tf))))) ; 416
(time (dotimes [_ 10] (count (re-seq #"EXT3.*" (mmap/slurp tf))))) ; 101

;---
; mmap.clj is about 3.1KB
(def tf "/home/chouser/proj/clojure-contrib/src/clojure/contrib/mmap.clj")
(println "\nregex slurp small" tf)

(time (dotimes [_ 1000] (count (re-seq #"defn \S*" (core/slurp tf))))) ; 308
(time (dotimes [_ 1000] (count (re-seq #"defn \S*" (mmap/slurp tf))))) ; 198

)
Something went wrong with that request. Please try again.