fix and re-include file upload tests

It is important that we test this functionality

features/steps/splinter.py

@@ -61,14 +61,32 @@ def bucket_contains(context, bucket_name, sequence_matcher):
 
     bucket = context.client.storage()[bucket_name]
     records = [datetimes_to_strings(record) for record in bucket.find()]
+    records = [ints_to_floats(record) for record in records]
+    records = [nones_to_zeroes(record) for record in records]
 
     assert_that(records, sequence_matcher(*matchers))
 
 
 def datetimes_to_strings(record):
     for key, value in record.items():
         if isinstance(value, datetime.datetime):
-            record[key] = utc(value).isoformat()
+            record[key] = unicode(utc(value).isoformat())
+
+    return record
+
+
+def ints_to_floats(record):
+    for key, value in record.items():
+        if isinstance(value, int):
+            record[key] = float(value)
+
+    return record
+
+
+def nones_to_zeroes(record):
+    for key, value in record.items():
+        if value is None:
+            record[key] = 0.0
 
     return record
 

features/write_api/excel_upload.feature

@@ -4,7 +4,10 @@ Feature: excel upload
 
     Scenario: Upload XLSX data
        Given a file named "data.xlsx" with fixture "data.xlsx"
+         and I have a bucket named "my_xlsx_bucket"
+         and bucket setting upload_format is "excel"
          and I am logged in
+         and I can upload to "my_xlsx_bucket"
         when I go to "/my_xlsx_bucket/upload"
          and I enter "data.xlsx" into the file upload field
          and I click "Upload"
@@ -15,7 +18,10 @@ Feature: excel upload
              """
     Scenario: using _timestamp for an auto id
        Given a file named "LPA_MI_EXAMPLE.xls" with fixture "LPA_MI_EXAMPLE.xls"
+         and I have a bucket named "bucket_with_timestamp_auto_id"
+         and bucket setting upload_format is "excel"
          and I am logged in
+         and I can upload to "bucket_with_timestamp_auto_id"
         when I go to "/bucket_with_timestamp_auto_id/upload"
          and I enter "LPA_MI_EXAMPLE.xls" into the file upload field
          and I click "Upload"

features/write_api/upload_validation.feature

@@ -10,7 +10,9 @@ Feature: csv upload validation
              Max,35,Italian,male
              """
          and I have a bucket named "foo"
+         and bucket setting upload_format is "csv"
          and I am logged in
+         and I can upload to "foo"
         when I go to "/foo/upload"
          and I enter "data.csv" into the file upload field
          and I click "Upload"
@@ -25,7 +27,9 @@ Feature: csv upload validation
              Max,35,Italian
              """
          and I have a bucket named "foo"
+         and bucket setting upload_format is "csv"
          and I am logged in
+         and I can upload to "foo"
         when I go to "/foo/upload"
          and I enter "data.csv" into the file upload field
          and I click "Upload"
@@ -35,7 +39,9 @@ Feature: csv upload validation
     Scenario: file too large
        Given a file named "data.csv" of size "1000000" bytes
          and I have a bucket named "foo"
+         and bucket setting upload_format is "csv"
          and I am logged in
+         and I can upload to "foo"
         when I go to "/foo/upload"
          and I enter "data.csv" into the file upload field
          and I click "Upload"
@@ -45,7 +51,9 @@ Feature: csv upload validation
     Scenario: non UTF8 characters
        Given a file named "data.csv" with fixture "bad-characters.csv"
          and I have a bucket named "foo"
+         and bucket setting upload_format is "csv"
          and I am logged in
+         and I can upload to "foo"
         when I go to "/foo/upload"
          and I enter "data.csv" into the file upload field
          and I click "Upload"
@@ -55,6 +63,7 @@ Feature: csv upload validation
     Scenario: no file is provided
        Given I am logged in
          and I have a bucket named "foo"
+         and I can upload to "foo"
         when I go to "/foo/upload"
          and I click "Upload"
         then I should see the text "There was an error with your upload"

run_tests.sh

@@ -41,7 +41,7 @@ display_result $? 1 "Unit tests"
 python -m coverage.__main__ xml --include=backdrop*
 
 # run feature tests
-behave --stop --tags=~@wip --tags=~@file_upload_test
+behave --stop --tags=~@wip
 display_result $? 2 "Feature tests"
 
 # run style checks

tests/support/file_upload_test_case.py

@@ -1,13 +1,49 @@
+from cStringIO import StringIO
 import io
+import os
 import unittest
 import werkzeug.datastructures
+from backdrop.write.uploaded_file import UploadedFile
 
 
 class FileUploadTestCase(unittest.TestCase):
-    def _file_storage_wrapper(self, contents, filename="test", content_type=None, content_length=None):
-        stream = io.StringIO(initial_value=unicode(contents))
+    def _file_storage_wrapper(self, contents, server_filename='/tmp/uploaded_file',
+                              browser_filename='browser_filename.txt', content_type=None, content_length=None):
+        with open(server_filename, 'w') as f:
+            f.write(contents)
+        g = open(server_filename, 'r')
+
         storage = werkzeug.datastructures \
-            .FileStorage(stream=stream, filename = filename,
+            .FileStorage(stream=g,
+                         filename=browser_filename,
                          content_type=content_type,
                          content_length=content_length)
         return storage
+
+    def _uploaded_file_wrapper(self, contents=None, fixture_name=None):
+        if len([i for i in [contents, fixture_name] if i is not None]) != 1:
+            raise TypeError("Takes one of contents or fixture_name argument")
+
+        if contents is not None:
+            server_filename = '/tmp/file.txt'
+            file_storage = self._file_storage_wrapper(
+                contents=contents,
+                server_filename=server_filename)
+
+            upload = UploadedFile(
+                file_storage,
+                server_filename
+            )
+            return upload
+
+        elif fixture_name is not None:
+            full_filename = fixture_path = os.path.join('features', 'fixtures', fixture_name)
+            with open(full_filename, 'rb') as f:
+                file_storage = self._file_storage_wrapper(
+                    contents=f.read(),
+                    server_filename=full_filename)
+            upload = UploadedFile(
+                file_storage,
+                full_filename  # server filename
+            )
+            return upload

tests/write/test_scanned_file.py

@@ -10,7 +10,7 @@
 class TestScannedFile(FileUploadTestCase):
 
     def setUp(self):
-        self.file_object = self._file_storage_wrapper("This is a test", "abc.txt")
+        self.file_object = self._file_storage_wrapper("This is a test", browser_filename="abc.txt")
         self.scanner = ScannedFile(self.file_object)
 
     def tearDown(self):

tests/write/test_uploaded_file.py

@@ -7,95 +7,68 @@
 
 class TestUploadedFile(FileUploadTestCase):
     def test_getting_a_file_stream(self):
-        upload = UploadedFile(self._file_storage_wrapper("This is a test", ""))
-        assert_that(upload.file_stream().read(), is_(u'This is a test'))
+        contents = 'This is a test'
+
+        upload = self._uploaded_file_wrapper(contents)
+        assert_that(upload.file_stream().read(), is_(contents))
 
     def test_files_under_1000000_octets_are_valid(self):
-        upload = UploadedFile(self._file_storage_wrapper(
-            "foo",
-            content_type="text/csv",
-            content_length=999999))
+        csv_length_999999 = '\n'.join(['aa,bb,ccc' for i in range(100000)])
+        upload = self._uploaded_file_wrapper(contents=csv_length_999999)
 
         assert_that(upload.valid, is_(True))
 
-    def test_files_over_1000000_octets_are_valid(self):
-        upload = UploadedFile(self._file_storage_wrapper(
-            "foo",
-            content_type="text/csv",
-            content_length=1000001))
+    def test_files_over_1000000_octets_are_not_valid(self):
+        csv_length_1000009 = '\n'.join(['aa,bb,ccc' for i in range(100001)])
+        upload = self._uploaded_file_wrapper(contents=csv_length_1000009)
 
         assert_that(upload.valid, is_(False))
 
     def test_saving_file(self):
-        upload = UploadedFile(self._file_storage_wrapper(
-            contents="some data",
-            content_type="text/csv"
-        ))
+        csv_contents = '\n'.join(['aa,bb,ccc' for i in range(100000)])
+        upload = self._uploaded_file_wrapper(contents=csv_contents)
         bucket = Mock()
         parser = Mock()
         upload.perform_virus_scan = Mock()
-        parser.return_value = "some data"
+        parser.return_value = csv_contents
         upload.save(bucket, parser)
         assert_that(parser.called, is_(True))
-        bucket.parse_and_store.assert_called_with("some data")
+        bucket.parse_and_store.assert_called_with(csv_contents)
 
     def test_saving_invalid_file_should_throw_an_exception(self):
-        upload = UploadedFile(self._file_storage_wrapper(
-            contents="some content"
-        ))
+        upload = self._uploaded_file_wrapper(contents='')
         bucket = Mock()
         parser = Mock()
-        self.assertRaises(FileUploadException, upload.save, bucket, parser )
+        self.assertRaises(FileUploadException, upload.save, bucket, parser)
         assert_that(bucket.called, is_(False))
 
-    def test_uploaded_file_must_have_a_file(self):
-        storage = self._file_storage_wrapper("boo", filename=None)
-        self.assertRaises(FileUploadException, UploadedFile, storage)
-
 
 class TestUploadedFileContentTypeValidation(FileUploadTestCase):
     def test_csv_uploads_are_valid(self):
-        upload = UploadedFile(self._file_storage_wrapper("This is a test",
-                                                         content_type="text/csv"))
+        upload = self._uploaded_file_wrapper("This is a test")
 
         assert_that(upload.valid, is_(True))
 
     def test_json_uploads_are_valid(self):
-        upload = UploadedFile(self._file_storage_wrapper("This is a test",
-                                                         content_type="application/json"))
+        upload = self._uploaded_file_wrapper('{"This": "is a test"}')
 
         assert_that(upload.valid, is_(True))
 
-    def test_excel_uploads_are_valid(self):
-        upload = UploadedFile(self._file_storage_wrapper(
-            "This is a test", filename="test",
-            content_type="application/vnd.ms-excel"))
-
+    def test_xls_uploads_are_valid(self):
+        upload = self._uploaded_file_wrapper(fixture_name='xlsfile.xls')
         assert_that(upload.valid, is_(True))
 
     def test_xlsx_spreadsheets_are_valid(self):
-        upload = UploadedFile(self._file_storage_wrapper(
-            "This is a test",
-            content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
-        ))
 
+        upload = self._uploaded_file_wrapper(fixture_name='data.xlsx')
         assert_that(upload.valid, is_(True))
 
     def test_exe_files_are_not_valid(self):
-        upload = UploadedFile(
-            self._file_storage_wrapper('foo', content_type="application/exe"))
-
-        assert_that(upload.valid, is_(False))
-
-    def test_files_with_no_content_type_are_invalid(self):
-        upload = UploadedFile(
-            self._file_storage_wrapper('foo', content_type=None))
-
+        upload = self._uploaded_file_wrapper(fixture_name='donothing.exe')
         assert_that(upload.valid, is_(False))
 
     @patch('backdrop.write.scanned_file.ScannedFile.has_virus_signature')
     def test_perform_virus_scan(self, has_virus_signature):
-        file_storage_wrapper = self._file_storage_wrapper('foo', content_type=None)
-        upload = UploadedFile(file_storage_wrapper)
+        upload = self._uploaded_file_wrapper('[fake empty content]')
         has_virus_signature.return_value = True
         self.assertRaises(VirusSignatureError, upload.perform_virus_scan)