Permalink
Browse files

Moved iteratee library to the top level of the collection.

  • Loading branch information...
1 parent 6c0d8ac commit a951c46c45bb1ecdc3f953b4f7883d6f2ef57ea3 @RayRacine committed Mar 27, 2013
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,42 @@
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Ray Racine's TR Library
+;; Copyright (C) 2007-2013 Raymond Paul Racine
+;;
+;; This program is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+;;
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with this program. If not, see <http://www.gnu.org/licenses/>.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+#lang typed/racket/base
+
+(require
+ (only-in "iteratee.rkt"
+ EnumerateeC Iteratee)
+ (only-in "enumeratees.rkt"
+ enumeratee/c-groupby enumeratee/c-flatmap)
+ (only-in "iteratees.rkt"
+ list-sink))
+
+(provide:
+ [reduce (All (D E) ((D D -> Boolean) (D -> (Option E)) -> (Iteratee D (Listof E))))])
+
+(: reduce (All (D E) ((D D -> Boolean) (D -> (Option E)) -> (Iteratee D (Listof E)))))
+(define (reduce is-same? reduce-fn)
+ (define: groupbyT : (EnumerateeC D (Listof D) (Listof E))
+ (enumeratee/c-groupby is-same?))
+ (define: mergeT : (EnumerateeC (Listof D) E (Listof E))
+ (enumeratee/c-flatmap reduce-fn))
+ (define: sink : (Iteratee E (Listof E))
+ (list-sink))
+ (define: list-iteratee : (Iteratee D (Listof E))
+ (groupbyT (mergeT sink)))
+ list-iteratee)
File renamed without changes.
@@ -0,0 +1,60 @@
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Ray Racine's TR Library
+;; Copyright (C) 2007-2013 Raymond Paul Racine
+;;
+;; This program is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+;;
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with this program. If not, see <http://www.gnu.org/licenses/>.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+#lang typed/racket/base
+
+;; (provide
+;; iseq icomplete
+;; Iteratee Stream
+;; (struct-out Continue)
+;; (struct-out Done)
+;; Enumerator eseq
+;; Enumeratee)
+
+(require racket/match)
+
+;;(define-type (Stream D) (U D 'Nothing 'EOS))
+
+;;(define-type (Iteratee D A) (U (Done D A) ((Stream D) -> (Iteratee D A))))
+
+
+;; (struct: (D A) Done ([stream : (Stream D)]
+;; [accum : A]))
+
+;; (struct: (D A) Continue ([step : ((Stream D) -> (Iteratee D A))]))
+
+;; (: icomplete (All (D A) (Iteratee D A) -> A))
+;; (define (icomplete iter)
+;; (match iter
+;; [(Done _ accum) accum]
+;; [(Continue step) (icomplete (step 'EOS))]))
+
+;; (: iseq (All (D A B) ((Iteratee D A) (A -> (Iteratee D B)) -> (Iteratee D B))))
+;; (define (iseq iter fn)
+;; (match iter
+;; [(Done d a) (fn a)]
+;; [(Continue step) (Continue (λ: ((d : (Stream D)))
+;; (iseq (step d) fn)))]))
+
+;; (define-type (Enumerator D A) ((Iteratee D A) -> (Iteratee D A)))
+
+;; (: eseq (All (D A) (Enumerator D A) (Enumerator D A) -> (Enumerator D A)))
+;; (define (eseq e1 e2)
+;; (λ (iter) (e2 (e1 iter))))
+
+;; (define-type (Enumeratee O I A) ((Iteratee I A) -> (Iteratee O (Iteratee I A))))
File renamed without changes.
File renamed without changes.
View
@@ -0,0 +1,15 @@
+#lang typed/racket/base
+
+(provide
+ (all-from-out "iteratee.rkt")
+ (all-from-out "iteratees.rkt")
+ (all-from-out "enumerators.rkt")
+ (all-from-out "enumeratees.rkt")
+ (all-from-out "iterfile.rkt"))
+
+(require
+ "iteratee.rkt"
+ "iteratees.rkt"
+ "enumerators.rkt"
+ "enumeratees.rkt"
+ "iterfile.rkt")
File renamed without changes.
@@ -26,7 +26,7 @@
Uri)
(only-in httpclient/uri/filescheme
local-path->uri)
- (only-in io/iteratee/iteratee
+ (only-in iteratee
icomplete Iteratee Continue Done Stream)
(only-in "types.rkt"
Range Block BlockSet Write))
View
@@ -32,7 +32,7 @@
(require
racket/match
- (only-in io/iteratee/iteratee
+ (only-in iteratee
Iteratee Stream Continue Done)
(only-in "types.rkt"
Text Record Record-key Record-value
View
@@ -40,7 +40,7 @@
Write TextParse Partition GroupCompare
BlockSet BlockSet-blocks BlockSet-uri
Map)
- (only-in io/iteratee/iteratee
+ (only-in iteratee
icomplete
Enumerator Iteratee)
(only-in "blockset.rkt"
@@ -22,7 +22,7 @@
[partition-iteratee (All (D) (Write D) (Partition D) (BlockSet D) -> (Iteratee D (BlockSet D)))])
(require
- (only-in io/iteratee/iteratee
+ (only-in iteratee
Iteratee Stream Continue Done)
(only-in httpclient/uri
parse-uri)
@@ -29,7 +29,7 @@
racket/match
(only-in prelude/std/opt
opt-map)
- (only-in io/iteratee/iteratee
+ (only-in iteratee
Enumerator Iteratee
Done Continue)
(only-in "../types.rkt"
@@ -10,23 +10,19 @@
(require
(only-in "../types.rkt"
RDD)
- (only-in "../../io/iteratee/iteratee.rkt"
+ (only-in iteratee
Enumerator
- Enumeratee)
- (only-in "../../io/iteratee/iterfile.rkt"
- IOResult)
- (only-in "../../io/iteratee/enumeratees.rkt"
- enumeratee-filter
- enumeratee-map)
- (only-in "../../io/iteratee/enumerators.rkt"
- enumerator/list)
- (only-in "../../io/iteratee/iterfile.rkt"
- TextPortIteratee
- iter-text-port)
+ Enumeratee
+ IOResult
+ enumeratee-filter
+ enumeratee-map
+ enumerator/list
+ TextPortIteratee
+ iter-text-port)
(only-in "rdd.rkt"
- RDDPrint
- RDDList RDDList-xs
- RDDMap RDDMap-cvt))
+ RDDPrint
+ RDDList RDDList-xs
+ RDDMap RDDMap-cvt))
(define-type RDDResult IOResult)
View
@@ -1,18 +1,15 @@
#lang typed/racket/base
(provide
- n-rdd
- n-blockset
;; ;; (struct-out RDDList)
-
(struct-out RDDSeq)
(struct-out RDDList)
;;(struct-out RDDFilter)
(struct-out RDDMap)
(struct-out RDDPrint)
generate-rdd-block-filename
rddfile-block-count
- rdd-text)
+ )
(require
racket/pretty
@@ -56,61 +53,13 @@
;; (RDDSeq A B)
;; (RDDPrint A B)))
-;; Build RDD from an input path
-(: rdd-text (case-> (Path -> (RDD Text))
- (Path Natural -> (RDD Text))))
-(define (rdd-text base-dir-path [block-size DEFAULT-BLOCK-SIZE])
- (RDD (list (BlockSet (local-path->uri base-dir-path)
- (apply append (map (λ: ((file-name : Path))
- (let ((full-path (path->complete-path file-name base-dir-path)))
- (n-block (path->string file-name) (file-size full-path) block-size)))
- (directory-list base-dir-path)))))))
-
(: rddfile-block-count (RDD -> Integer))
(define (rddfile-block-count rdd)
(define: total : Natural 0)
(for ((blockset (RDD-blocksets rdd)))
- (set! total (+ total (blockset-count blockset))))
+ (set! total (+ total (blockset-count blockset))))
total)
-(: n-block (String Nonnegative-Integer Nonnegative-Integer -> (Listof Block)))
-(define (n-block loc n sz)
- (let-values (((bs lb-sz) (quotient/remainder n sz)))
- (let: ((full-blocks : (Listof Block)
- (for/list ((b (in-range bs))
- #:when (>= b 0)) ;; for type-checker
- (let ((sod (* b sz))
- (eod (* (add1 b) sz)))
- (Block loc (Range sod eod))))))
- (if (> lb-sz 0) ;; partial block
- (let* ((sod (* bs sz))
- (eod (+ sod lb-sz)))
- (cons (Block loc (Range sod eod)) full-blocks))
- full-blocks))))
-
(: generate-rdd-block-filename (-> Path))
(define (generate-rdd-block-filename)
(make-temporary-file "rdd-~a.block" #f rdd-materialization-directory))
-
-;; Split a BlockSet into smaller blocksets no larger than N Blocks in size.
-;; Does NOT split a Block.
-(: n-blockset (BlockSet Natural -> (Listof BlockSet)))
-(define (n-blockset blockset n)
- (define uri (BlockSet-uri blockset))
- (let: loop : (Listof BlockSet) ((blocks : (Listof Block)'())
- (blocksets : (Listof BlockSet) '())
- (counter : Natural n)
- (all-blocks : (Listof Block) (BlockSet-blocks blockset)))
- (if (null? all-blocks)
- (cons (BlockSet uri blocks) blocksets)
- (if (zero? counter)
- (loop '() (cons (BlockSet uri blocks) blocksets) n all-blocks)
- (loop (cons (car all-blocks) blocks) blocksets (sub1 counter) (cdr all-blocks))))))
-
-;; Split an RDD into smaller BlockSets no larger than N of them.
-(: n-rdd (RDD Natural -> RDD))
-(define (n-rdd rdd n)
- (RDD (apply append (map (λ: ((blockset : BlockSet))
- (n-blockset blockset n))
- (RDD-blocksets rdd)))))
-
View
@@ -30,16 +30,13 @@
log-mr-info log-mr-error)
(only-in aws/s3/invoke
S3Response)
- (only-in io/iteratee/iteratee
+ (only-in iteratee
icomplete
- Iteratee Enumerator Enumeratee)
- (only-in io/iteratee/enumeratees
- enumeratee-groupby
- enumeratee-flatmap)
- (only-in io/iteratee/enumerators
- enumerator/select-from-n-lists)
- (only-in io/iteratee/iteratees
- lister)
+ Iteratee Enumerator Enumeratee
+ enumeratee-groupby
+ enumeratee-flatmap
+ enumerator/select-from-n-lists
+ list-sink)
(only-in httpclient/uri
parse-uri)
(only-in "configuration.rkt"
@@ -103,7 +100,7 @@
(: in-mem-parse-sort-partition-text (All (D) (Text (TextParse D) (Sort D) -> (Listof D))))
(define (in-mem-parse-sort-partition-text text parser sorter)
(define: enum-text : (Enumerator D (Listof D)) (enum/text text parser))
- (define: iter-list : (Iteratee D (Listof D)) (lister))
+ (define: iter-list : (Iteratee D (Listof D)) (list-sink))
(sort (icomplete (enum-text iter-list)) sorter))
(: fetch-sorted-text-blocks (All (D) S3Partition (TextParse D) (Sort D) -> (Listof (Listof D))))
@@ -33,7 +33,7 @@
(only-in "logging.rkt"
log-mr-error
log-mr-info)
- (only-in io/iteratee/iteratee
+ (only-in iteratee
Iteratee Stream Continue Done)
(only-in prelude/std/opt
opt-map)
@@ -46,9 +46,8 @@
(only-in aws/s3/types
Key Key-key Keys-truncated?
Keys Keys-objects)
- (only-in aws/s3/uri
- s3-path->prefix
- s3-uri-bucket-and-path)
+ (only-in aws/s3/s3-uri
+ s3-uri-path->prefix)
(only-in aws/s3/invoke
S3Response)
(only-in aws/s3/objects
@@ -31,12 +31,10 @@
Uri)
(only-in httpclient/uri/filescheme
local-file-uri->path)
- (only-in io/iteratee/iteratee
- Iteratee Enumerator icomplete)
- (only-in io/iteratee/enumerators
- enumerator/select-from-n-lists)
- (only-in io/iteratee/iteratees
- lister)
+ (only-in iteratee
+ Iteratee Enumerator icomplete
+ enumerator/select-from-n-lists
+ list-sink)
(only-in "../types.rkt"
TextParse
BlockSet BlockSet-uri BlockSet-blocks
@@ -8,12 +8,10 @@
"../frame/frame.rkt"
racket/match
racket/pretty
- (only-in "../../io/iteratee/enumerators.rkt"
- enumerator/text-input-port)
- (only-in "../../io/iteratee/iteratee.rkt"
+ (only-in iteratee
+ enumerator/text-input-port
Iteratee Stream Continue Done
- icomplete)
- (only-in "../../io/iteratee/iteratees.rkt"
+ icomplete
head-n)
(only-in "../frame/categorical-series-builder.rkt"
CSeriesBuilder

0 comments on commit a951c46

Please sign in to comment.