allow control of read size

RevolutionAnalytics · Mar 27, 2013 · 7a03b29 · 7a03b29
1 parent 769f1b5
commit 7a03b29
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/pkg/R/IO.R b/pkg/R/IO.R
@@ -95,7 +95,7 @@ make.typedbytes.input.format = function(recycle = TRUE) {
   obj.buffer = list()
   obj.buffer.rmr.length = 0
   raw.buffer = raw()
-  read.size = 10^7
+  read.size = rmr.options(read.size)
   function(con, keyval.length) {
     while(length(obj.buffer) < 2 || 
       obj.buffer.rmr.length < keyval.length) {

diff --git a/pkg/R/mapreduce.R b/pkg/R/mapreduce.R
@@ -17,7 +17,8 @@
 rmr.options.env = new.env(parent=emptyenv())
 
 rmr.options.env$backend = "hadoop"
-rmr.options.env$keyval.length = 10000
+rmr.options.env$keyval.length = 10^4
+rmr.options.env$read.size = 10^7
 rmr.options.env$profile.nodes = "off"
 rmr.options.env$dfs.tempdir = tempdir()
 rmr.options.env$depend.check = FALSE
@@ -27,7 +28,8 @@ rmr.options.env$rscript.cmd = 'Rscript'
 rmr.options = 
   function(backend = c("hadoop", "local"), 
            profile.nodes = c("off", "calls", "memory", "both"),
-           keyval.length = 10000,
+           keyval.length = 10^4,
+           read.size = 10^7,
            dfs.tempdir = tempdir(),
            rscript.cmd = 'Rscript'#,
            #depend.check = FALSE,