increase target auto-sample size to 2.5M

datarobot · Oct 25, 2016 · c0312d1 · c0312d1
1 parent 05937de
commit c0312d1
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/datarobot_batch_scoring/utils.py b/datarobot_batch_scoring/utils.py
@@ -479,7 +479,7 @@ def auto_sampler(dataset, encoding, ui):
 
     buf.close()
     avg_line = int(size_bytes / csv_lines)
-    chunk_size_goal = int(1.5 * 1024 ** 2)  # size we want per batch
+    chunk_size_goal = int(2.5 * 1024 ** 2)  # size we want per batch
     lines_per_sample = int(chunk_size_goal / avg_line) + 1
     ui.debug('auto_sampler: lines counted: {},  avgerage line size: {}, '
              'recommended lines per sample: {}'.format(csv_lines, avg_line,

diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -235,7 +235,7 @@ def test_auto_sample():
     with UI(None, logging.DEBUG, stdout=False) as ui:
         data = 'tests/fixtures/criteo_top30_1m.csv.gz'
         encoding = investigate_encoding_and_dialect(data, None, ui)
-        assert auto_sampler(data, encoding, ui) == 8988
+        assert auto_sampler(data, encoding, ui) == 14980
         ui.close()