From f7b74080eb172fe60644a1e7cb949edfc463cf96 Mon Sep 17 00:00:00 2001 From: Jeff Palmucci Date: Fri, 23 Jul 2010 16:46:15 -0400 Subject: [PATCH] Initial commit --- .gitignore | 4 ++ README | 31 +++++++++++ project.clj | 5 ++ src/LICENSE | 22 ++++++++ src/yield.clj | 103 +++++++++++++++++++++++++++++++++++ test/clj_yield/core_test.clj | 39 +++++++++++++ 6 files changed, 204 insertions(+) create mode 100644 .gitignore create mode 100644 README create mode 100644 project.clj create mode 100644 src/LICENSE create mode 100644 src/yield.clj create mode 100644 test/clj_yield/core_test.clj diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d9148e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +pom.xml +*jar +lib +classes \ No newline at end of file diff --git a/README b/README new file mode 100644 index 0000000..fd2874f --- /dev/null +++ b/README @@ -0,0 +1,31 @@ +Clj-yield provides functionality similar to Python's yield +statement. For example: + +(def *sequence* + (with-yielding [out 5] + (loop [x 10] + (if (pos? x) + (do (yield out x) + (recur (dec x))))))) + +*sequence* => (10 9 8 7 6 5 4 3 2 1) + +With-yielding will create a background thread in which to run its +body, and return a lazy sequence of the yielded results. You can +specify how far the producer thread can get ahead of the consumers (5 +above). The yield function will block if the producer gets too far ahead. + +If the lazy sequence ever becomes garbage collectable, the yield +function will throw a java.lang.InterruptedException. The body of the +with-yielding should return. + +I wrote 'yield' for a couple different reasons: + +1) I wanted some code to generate a lazy sequence, but it was +cumbersome to write it using lazy-seq. 'Yield' allows me to write it +as a simple loop. + +2) I wanted to use transient data structures while calculating a lazy +sequence. Since 'yield' computes its output using a (single) background +thread, this isn't a problem. (Note: you can still spawn new threads +inside the body of a with-yielding.) diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..0d8e993 --- /dev/null +++ b/project.clj @@ -0,0 +1,5 @@ +(defproject clj-yield "1.0.0-SNAPSHOT" + :description "A function like python's yield statement that can push items to a lazy sequence from arbitrary (non-lazy) code." + :dependencies [[org.clojure/clojure "1.2.0-beta1"] + [org.clojure/clojure-contrib "1.2.0-beta1"]]) + diff --git a/src/LICENSE b/src/LICENSE new file mode 100644 index 0000000..8194305 --- /dev/null +++ b/src/LICENSE @@ -0,0 +1,22 @@ +clj-yield --- Copyright (c) 2010, Jeff Palmucci + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/yield.clj b/src/yield.clj new file mode 100644 index 0000000..4ccade9 --- /dev/null +++ b/src/yield.clj @@ -0,0 +1,103 @@ +(ns yield + (:use clojure.test)) + +;; ******************************************************************************** +;; + +(deftype garbage-monitor + [obj f] + Object + (finalize [this] (f obj)) + clojure.lang.IDeref + (deref [this] (.obj this))) + +(defmacro upon-gc [obj & body] + "Returns a derefable object that contains 'obj'. The body will run +'body' when the returned object becomes garbage collectable." + `(garbage-monitor. + ~obj + (fn [o#] + ~@body))) + +;; ******************************************************************************** + +;; can't put a nil into a linked blocking queue, so use this object to mark them +(defonce *nil-marker* (Object.)) +;; marker in the BlockingQueue that signals an exception in the +;; generating thread. The next element will be the exception that +;; occurred +(defonce *exception-marker* (Object.)) +;; end of the sequence +(defonce *end-marker* (Object.)) + +(defn yield [yseq x] + "Append an element to the sequence 'yseq', created with +'with-yielding'. May block if there is no capacity in yseq. + +If, while blocking, the output sequence is garbage collected, yield +will throw a java.lang.InterruptedException. The body of the +with-yielding should return." + (try + (.offer ^java.util.concurrent.LinkedBlockingQueue @yseq (if (nil? x) *nil-marker* x) + 10 java.util.concurrent.TimeUnit/DAYS) + (catch NullPointerException e + (if (nil? @yseq) + (throw (java.lang.InterruptedException. "Computing Garbage")) + (throw e))))) + +(defmacro with-yielding [[name n] & body] + "Construct and return a sequence that is filled using 'yield' from + within the body. The body can get up to 'n' elements ahead of the + sequence consumer before blocking on 'yield'. For example: + +user> (def *test-sequence* + (yield/with-yielding [out 5] + (loop [n 10] + (if (pos? n) + (do (println \"Yielding\" n) + (yield out n) + (recur (dec n))))))) +Yielding 10 +Yielding 9 +Yielding 8 +Yielding 7 +Yielding 6 +Yielding 5 +#'user/*test-sequence* +user> (first *test-sequence*) +Yielding 4 +10 +" + + `(with-yielding* ~n + (bound-fn [~name] + ~@body))) + +(defn with-yielding* [n f] + (let [queue (atom (java.util.concurrent.LinkedBlockingQueue. (int n))) + ft (future + (try + (f queue) + (catch Exception e + (.offer @queue *exception-marker* 10 java.util.concurrent.TimeUnit/DAYS) + (.offer @queue e 10 java.util.concurrent.TimeUnit/DAYS)) + (finally (.offer @queue *end-marker* 10 java.util.concurrent.TimeUnit/DAYS)))) + get-ele (fn get-ele [guard] + (let [ele (.take ^java.util.concurrent.LinkedBlockingQueue @queue) ] + (cond (= ele *end-marker*) () + + (= ele *exception-marker*) + (throw (RuntimeException. + (.take @queue))) + + true + (cons (if (= ele *nil-marker*) nil ele) + (lazy-seq (get-ele guard))))))] + (let [guard (upon-gc queue + (try + (let [q @queue] + (swap! queue (fn [x] nil)) + (.clear q)) + (catch Exception e (.printStackTrace e))))] + (lazy-seq (get-ele guard))))) + diff --git a/test/clj_yield/core_test.clj b/test/clj_yield/core_test.clj new file mode 100644 index 0000000..1d9281b --- /dev/null +++ b/test/clj_yield/core_test.clj @@ -0,0 +1,39 @@ +(ns clj-yield.core-test + (:use [yield] :reload-all) + (:use [clojure.test])) + +(deftest with-yielding-test + ;; check that the sequence is correctly generated + (is (= (apply + (with-yielding [out 10] + (loop [x 1000] + (if (pos? x) + (do + (yield out x) + (recur (dec x))))))) + 500500)) + + ;; Throw an exception, but don't read far enough to get it. + ;; checks lazy semantics + (is (= (first (with-yielding [out 10] + (loop [x 1000] + (if (pos? x) + (do + (yield out x) + (recur (dec x))))) + (throw (Exception. "exception")))) + 1000)) + + ;; hit the exception to test the exception throwing mechanism + (is (= (try + (count + (with-yielding [out 10] + (loop [x 1000] + (if (pos? x) + (do + (yield out x) + (recur (dec x))))) + (throw (Exception. "exception")))) + (catch Exception e :exception)) + :exception))) + +