Merge pull request #202 from keitaroinc/parametrize-job-config

Read chunk size, chunk inserts and timeouts from config

datapusher/jobs.py

@@ -30,8 +30,9 @@
     locale.setlocale(locale.LC_ALL, '')
 
 MAX_CONTENT_LENGTH = web.app.config.get('MAX_CONTENT_LENGTH') or 10485760
-CHUNK_SIZE = 16 * 1024  # 16kb
-DOWNLOAD_TIMEOUT = 30
+CHUNK_SIZE = web.app.config.get('CHUNK_SIZE') or 16384
+CHUNK_INSERT_ROWS = web.app.config.get('CHUNK_INSERT_ROWS') or 250
+DOWNLOAD_TIMEOUT = web.app.config.get('DOWNLOAD_TIMEOUT') or 30
 
 if web.app.config.get('SSL_VERIFY') in ['False', 'FALSE', '0', False, 0]:
     SSL_VERIFY = False
@@ -503,7 +504,7 @@ def row_iterator():
         return headers_dicts, result
 
     count = 0
-    for i, chunk in enumerate(chunky(result, 250)):
+    for i, chunk in enumerate(chunky(result, CHUNK_INSERT_ROWS)):
         records, is_it_the_last_chunk = chunk
         count += len(records)
         logger.info('Saving chunk {number} {is_last}'.format(