a little realer

controllers/datasets.py

@@ -1,10 +1,9 @@
-import json
-
 from lib.exceptions import JSONError
 from lib.mongo import mongo_to_json
 from lib.io import create_dataset_from_url, create_dataset_from_csv, \
          open_data_file
 from lib.tasks.summarize import summarize
+from lib.utils import dump_or_error
 from models.dataset import Dataset
 from models.observation import Observation
 
@@ -25,7 +24,7 @@ def DELETE(self, dataset_id):
             Dataset.delete(dataset_id)
             Observation.delete(dataset)
             result = {'success': 'deleted dataset: %s' % dataset_id}
-        return json.dumps(result or {'error': 'id not found'})
+        return dump_or_error(result, 'id not found')
 
     def GET(self, dataset_id, summary=False, query='{}', select=None,
             group=None):
@@ -49,13 +48,20 @@ def GET(self, dataset_id, summary=False, query='{}', select=None,
         except JSONError, e:
             result = {'error': e.__str__()}
 
-        return json.dumps(result or {'error': 'id not found'})
+        return dump_or_error(result, 'id not found')
 
     def POST(self, url=None, csv_file=None):
         """
-        Read data from URL *url*.
-        If URL is not provided and data is provided, read posted data *data*.
+        If *url* is provided read data from URL *url*.
+        If *csv_file* is provided read data from *csv_file*.
+        Otherwise return an error message.
         """
-        if not csv_file:
-            return json.dumps(create_dataset_from_url(url))
-        return json.dumps(create_dataset_from_csv(csv_file))
+        result = None
+
+        if url:
+            result = create_dataset_from_url(url)
+
+        if csv_file:
+            result = create_dataset_from_csv(csv_file)
+
+        return dump_or_error(result, 'url or csv_file required')

docs/basic_commands.rst

@@ -4,17 +4,19 @@ Basic Commands
 Storing data in Bamboo
 ----------------------
 
-store a URL in Bamboo:
+upload data from a URL to Bamboo:
 ^^^^^^^^^^^^^^^^^^^^^^
 
 ``curl -X POST -d "url=http://formhub.org/mberg/forms/good_eats/data.csv" http://localhost:8080/datasets``
 
-store a local file in Bamboo:
+upload data from a file to Bamboo:
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-given ``data.csv`` is in ``.``
+given the file ``/home/modilabs/good_eats.csv`` exists locally on your
+filesystem
+
+``curl -X POST -F csv_file=@/home/modilabs/good_eats.csv http://localhost:8080/datasets``
 
-``curl -X POST -d "url=file://data.csv" http://localhost:8080/datasets``
 
 Retrieve data 
 -------------

lib/io.py

@@ -30,11 +30,8 @@ def open_data_file(url):
 
 def read_uploaded_file(_file, chunk_size=8192):
     data = ''
-    while True:
-        chunk = _file.file.read(chunk_size)
-        if not chunk:
-            break
-        data += chunk
+    for line in _file.file.xreadlines():
+        data += line
     return data
 
 

lib/utils.py

@@ -1,3 +1,4 @@
+import json
 from math import isnan
 
 import numpy as np
@@ -26,3 +27,7 @@ def series_to_jsondict(series):
 
 def df_to_jsondict(dframe):
     return [series_to_jsondict(series) for idx, series in dframe.iterrows()]
+
+
+def dump_or_error(data, error_message):
+    return json.dumps(data or {'error': error_message})

tests/controllers/test_datasets.py

@@ -11,12 +11,13 @@ class TestDatasets(TestBase):
 
     def setUp(self):
         TestBase.setUp(self)
-        self._file = 'file://tests/fixtures/good_eats.csv'
+        self._file_path = 'tests/fixtures/good_eats.csv'
+        self._file_uri = 'file://%s' % self._file_path
         self.url = 'http://formhub.org/mberg/forms/good_eats/data.csv'
         self.controller = Datasets()
 
     def _post_file(self):
-        self.dataset_id = json.loads(self.controller.POST(self._file))['id']
+        self.dataset_id = json.loads(self.controller.POST(self._file_uri))['id']
 
     def _test_results(self, results):
         results = json.loads(results)
@@ -36,7 +37,13 @@ def _test_get_with_query_or_select(self, query='{}', select=None):
             self.assertEqual(len(results), 11)
 
     def test_POST_file(self):
-        result = json.loads(self.controller.POST(self._file))
+        _file = open(self._file_path, 'r')
+        result = json.loads(self.controller.POST(csv_file=_file.read()))
+        self.assertTrue(isinstance(result, dict))
+        self.assertTrue('id' in result)
+
+    def test_POST_file_as_url(self):
+        result = json.loads(self.controller.POST(self._file_uri))
         self.assertTrue(isinstance(result, dict))
         self.assertTrue('id' in result)