Merge branch 'master' of github.com:modilabs/formhub into map-filter-…

…select_one-in-filter

.gitignore

@@ -1,40 +1,42 @@
+/data
+error.log
+load_script.logs
+load_script.pid
+local_settings.py
 pyxform
+search_index
 site_media/attachments
-
-tmp/*
-*.pyc
-*.sql
-*.tar.gz
-*~
-.DS_Store
-*.sqlite3
-error.log
 site_media/css/sass/.sass-cache/*
-.sass-cache
-.odk
-*orig
-.idea
-search_index
-
-#for a local virtualenv (as done in mangrove repo)
-ve/
-
-# file created by tests
-registration.xml
+tmp/*
+xform_manager_dataset.json
 
 # folder to hold csv files
 csvs
 
-xform_manager_dataset.json
-
-/data
-local_settings.py
-.~lock.*
-
 # media folder used by tests
 # todo: figure out a way to clean this up rather than ignore it.
 /test_media
 /media
 
-load_script.logs
-load_script.pid
+# file created by tests
+registration.xml
+
+# for a local virtualenv (as done in mangrove repo)
+ve/
+
+.~lock.*
+.DS_Store
+.idea
+.odk
+.project
+.pydevproject
+.sass-cache
+.settings
+
+*.bak
+*.pyc
+*.sql
+*.sqlite3
+*.tar.gz
+*~
+*orig

local_settings.py.example

@@ -0,0 +1,29 @@
+# mysql
+#DATABASES = {
+#    'default': {
+#        'ENGINE': 'django.db.backends.mysql',
+#        'NAME': 'formhub_dev',
+#        'USER': 'formhub_dev',
+#        'PASSWORD': '',
+#    }
+#}
+
+# postgres
+#DATABASES = {
+#    'default': {
+#        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+#        'NAME': 'formhub_dev',
+#        'USER': 'formhub_dev',
+#        'PASSWORD': '',
+#    }
+#}
+
+# sqlite
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': 'db.sqlite3',
+    }
+}
+
+TOUCHFORMS_URL = 'http://localhost:9000/'

main/tests/test_base.py

@@ -23,6 +23,11 @@ def _login(self, username, password):
         assert client.login(username=username, password=password)
         return client
 
+    def _logout(self, client=None):
+        if not client:
+            client = self.client
+        client.logout()
+
     def _create_user_and_login(self, username="bob", password="bob"):
         self.user = self._create_user(username, password)
         self.client = self._login(username, password)
@@ -37,6 +42,11 @@ def _publish_xls_file(self, path):
             post_data = {'xls_file': xls_file}
             return self.client.post('/%s/' % self.user.username, post_data)
 
+    def _share_form_data(self, id_string='transportation_2011_07_25'):
+        xform = XForm.objects.get(id_string=id_string)
+        xform.shared_data = True
+        xform.save()
+
     def _publish_transportation_form(self):
         xls_path = os.path.join(self.this_directory, "fixtures",
                 "transportation", "transportation.xls")

main/tests/test_google_docs_export.py

@@ -25,6 +25,14 @@ def test_google_docs_export(self):
         }))
         self.assertEqual(response.status_code, 302)
         self.assertEqual(response['Location'], 'https://docs.google.com')
+        # share the data, log out, and check the export
+        self._share_form_data()
+        self._logout()
+        response = self.client.get(reverse(google_xls_export, kwargs={
+            'username': self.user.username,
+            'id_string': self.xform.id_string
+        }))
+        self.assertEqual(response.status_code, 302)
 
     def _refresh_token(self):
         self.assertEqual(TokenStorageModel.objects.all().count(), 0)

odk_logger/import_tools.py

@@ -66,26 +66,29 @@ def import_instance(path_to_instance_folder, status, user):
 
 
 def iterate_through_odk_instances(dirpath, callback):
-    count = 0
+    total_file_count = 0
+    success_count = 0
     errors = []
     for directory, subdirs, subfiles in os.walk(dirpath):
         for filename in subfiles:
             filepath = os.path.join(directory, filename)
             if XFormInstanceFS.is_valid_odk_instance(filepath):
                 xfxs = XFormInstanceFS(filepath)
                 try:
-                    count += callback(xfxs)
+                    success_count += callback(xfxs)
                 except Exception, e:
-                    errors.append(str(e))
+                    errors.append("%s => %s" % (xfxs.filename, str(e)))
                 del(xfxs)
-    return (count, errors)
+                total_file_count += 1
+    return (total_file_count, success_count, errors)
 
 
 def import_instances_from_zip(zipfile_path, user, status="zip"):
     count = 0
     try:
         temp_directory = tempfile.mkdtemp()
         zf = zipfile.ZipFile(zipfile_path)
+
         zf.extractall(temp_directory)
         def callback(xform_fs):
             """
@@ -108,7 +111,7 @@ def callback(xform_fs):
                 return 1
             else:
                 return 0
-        count, errors = iterate_through_odk_instances(temp_directory, callback)
+        total_count, success_count, errors = iterate_through_odk_instances(temp_directory, callback)
     finally:
         shutil.rmtree(temp_directory)
-    return (count, errors)
+    return (total_count, success_count, errors)

odk_logger/views.py

@@ -61,12 +61,12 @@ def bulksubmission(request, username):
         our_tempfile.write(postfile.read())
         our_tempfile.close()
         our_tf = open(our_tfpath, 'rb')
-        count, errors = import_instances_from_zip(our_tf, user=posting_user)
+        total_count, success_count, errors = import_instances_from_zip(our_tf, user=posting_user)
         os.remove(our_tfpath)
         json_msg = {
-            'message': "Your ODK submission was successful. %d surveys imported. Your user now has %d instances." % \
-                    (count, posting_user.surveys.count()),
-            'errors': errors
+            'message': "Submission successful. Out of %d survey instances, %d were imported (%d were rejected--duplicates, missing forms, etc.)" % \
+                    (total_count, success_count, total_count - success_count),
+            'errors': "%d %s" % (len(errors), errors)
         }
         response = HttpResponse(json.dumps(json_msg))
         response.status_code = 200
@@ -170,7 +170,8 @@ def submission(request, username=None):
     return response
 
 def download_xform(request, username, id_string):
-    xform = XForm.objects.get(user__username=username, id_string=id_string)
+    xform = get_object_or_404(XForm,
+            user__username=username, id_string=id_string)
     # TODO: protect for users who have settings to use auth
     response = response_with_mimetype_and_name('xml', id_string,
         show_date=False)

odk_viewer/models/parsed_instance.py

@@ -1,13 +1,15 @@
 import base64
 import datetime
 import re
+import json
 
 from bson import json_util
 from django.conf import settings
 from django.db import models
 from django.db.models.signals import post_save, pre_delete
 import json
 
+
 from utils.model_tools import queryset_iterator
 from odk_logger.models import Instance
 from common_tags import START_TIME, START, END_TIME, END, ID, UUID, ATTACHMENTS
@@ -34,7 +36,7 @@ def datetime_from_str(text):
 def dict_for_mongo(d):
     for key, value in d.items():
         if type(value) == list:
-            value = map(dict_for_mongo, [e for e in value if type(e) == dict])
+            value = [dict_for_mongo(e) if type(e) == dict else e for e in value]
         if type(value) == dict:
             value = dict_for_mongo(value)
         if key == '_id':
@@ -111,20 +113,37 @@ def dicts(cls, xform):
         qs = cls.objects.filter(instance__xform=xform)
         for parsed_instance in queryset_iterator(qs):
             yield parsed_instance.to_dict()
+
+    def _get_name_for_type(self, type_value):
+        """
+        We cannot assume that start time and end times always use the same XPath
+        This is causing problems for other peoples' forms.
+        
+        This is a quick fix to determine from the original XLSForm's JSON representation
+        what the 'name' was for a given type_value ('start' or 'end')
+        """
+        datadict = json.loads(self.instance.xform.json)
+        for item in datadict['children']:
+            if type(item)==dict and item.get(u'type')==type_value:
+                return item['name']
 
     def _set_start_time(self):
         doc = self.to_dict()
-        if START_TIME in doc:
-            date_time_str = doc[START_TIME]
-            self.start_time = datetime_from_str(date_time_str)
-        elif START in doc:
-            date_time_str = doc[START]
+        start_time_key1 = self._get_name_for_type(START)
+        start_time_key2 = self._get_name_for_type(START_TIME)
+        start_time_key = start_time_key1 or start_time_key2 # if both, can take either
+        if start_time_key is not None and start_time_key in doc:
+            date_time_str = doc[start_time_key]
             self.start_time = datetime_from_str(date_time_str)
         else:
             self.start_time = None
 
     def _set_end_time(self):
         doc = self.to_dict()
+        end_time_key1 = self._get_name_for_type(START)
+        end_time_key2 = self._get_name_for_type(START_TIME)
+        end_time_key = end_time_key1 or end_time_key2
+
         if END_TIME in doc:
             date_time_str = doc[END_TIME]
             self.end_time = datetime_from_str(date_time_str)

odk_viewer/tests/__init__.py

@@ -2,3 +2,4 @@
 from form_submission import TestFormSubmission
 from mongo_data_output import TestMongoData
 from test_map_view import TestMapView
+from test_exports import TestExports

odk_viewer/tests/test_exports.py

@@ -0,0 +1,13 @@
+from main.tests.test_base import MainTestCase
+from django.core.urlresolvers import reverse
+from odk_logger.views import download_xlsform
+from odk_viewer.xls_writer import XlsWriter
+
+class TestExports(MainTestCase):
+    def test_unique_xls_sheet_name(self):
+        xls_writer = XlsWriter()
+        xls_writer.add_sheet('section9_pit_latrine_with_slab_group')
+        xls_writer.add_sheet('section9_pit_latrine_without_slab_group')
+        # create a set of sheet names keys
+        sheet_names_set = set(xls_writer._sheets.keys())
+        self.assertEqual(len(sheet_names_set), 2)

odk_viewer/views.py

@@ -243,7 +243,7 @@ def cached_get_labels(xpath):
             table_rows.append('<tr><td>%s</td><td>%s</td></tr>' % (key, value))
         img_urls = image_urls(pi.instance)
         img_url = img_urls[0] if img_urls else ""
-        data_for_template.append({"name":id_string, "id": pi.id, "lat": pi.lat, "lng": pi.lng,'image_urls': img_urls, "table": '<table border="1"><a href="#"><img width="210" class="thumbnail" src="%s" alt=""></a><%s</table>' % (img_url,''.join(table_rows))})
+        data_for_template.append({"name":id_string, "id": pi.id, "lat": pi.lat, "lng": pi.lng,'image_urls': img_urls, "table": '<table border="1"><a href="#"><img width="210" class="thumbnail" src="%s" alt=""></a>%s</table>' % (img_url,''.join(table_rows))})
     context.data = data_for_template
     response = render_to_response("survey.kml",
         context_instance=context,
@@ -252,6 +252,7 @@ def cached_get_labels(xpath):
     return response
 
 
+@login_required
 def google_xls_export(request, username, id_string):
     owner = User.objects.get(username=username)
     xform = XForm.objects.get(id_string=id_string, user=owner)

odk_viewer/xls_writer.py

@@ -7,6 +7,8 @@ class XlsWriter(object):
     def __init__(self):
         self.set_file()
         self.reset_workbook()
+        self.sheet_name_limit = 30
+        self._generated_sheet_name_dict = {}
 
     def set_file(self, file_object=None):
         """
@@ -25,10 +27,12 @@ def reset_workbook(self):
         self._columns = defaultdict(list)
         def one(): return 1
         self._current_index = defaultdict(one)
+        self._generated_sheet_name_dict = {}
 
     def add_sheet(self, name):
-        sheet = self._workbook.add_sheet(name[0:20])
-        self._sheets[name] = sheet
+        unique_sheet_name = self._unique_name_for_xls(name)
+        sheet = self._workbook.add_sheet(unique_sheet_name)
+        self._sheets[unique_sheet_name] = sheet
 
     def add_column(self, sheet_name, column_name):
         index = len(self._columns[sheet_name])
@@ -51,7 +55,9 @@ def add_obs(self, obs):
         self._fix_indices(obs)
         for sheet_name, rows in obs.items():
             for row in rows:
-                self.add_row(sheet_name, row)
+                actual_sheet_name = self._generated_sheet_name_dict.get(
+                        sheet_name, sheet_name)
+                self.add_row(actual_sheet_name, row)
 
     def _fix_indices(self, obs):
         for sheet_name, rows in obs.items():
@@ -66,7 +72,7 @@ def write_tables_to_workbook(self, tables):
         tables should be a list of pairs, the first element in the
         pair is the name of the table, the second is the actual data.
 
-        todo: figure out how to write to the xls file rather than keep
+        TODO: figure out how to write to the xls file rather than keep
         the whole workbook in memory.
         """
         self.reset_workbook()
@@ -98,3 +104,28 @@ def _add_sheets(self):
                     if isinstance(f, Question) and\
                             not question_types_to_exclude(f.type):
                         self.add_column(sheet_name, f.name)
+
+    def _unique_name_for_xls(self, sheet_name):
+        # excel worksheet name limit seems to be 31 characters (30 to be safe)
+        unique_sheet_name = sheet_name[0:self.sheet_name_limit]
+        unique_sheet_name = self._generate_unique_sheet_name(unique_sheet_name)
+        self._generated_sheet_name_dict[sheet_name] = unique_sheet_name
+        return unique_sheet_name
+
+    def _generate_unique_sheet_name(self, sheet_name):
+        # check if sheet name exists
+        if(not self._sheets.has_key(sheet_name)):
+            return sheet_name
+        else:
+            i = 1
+            unique_name = sheet_name
+            while(self._sheets.has_key(unique_name)):
+                number_len = len(str(i))
+                allowed_name_len = self.sheet_name_limit - number_len
+                # make name required len
+                if(len(unique_name) > allowed_name_len):
+                    unique_name = unique_name[0:allowed_name_len]
+                unique_name = "{0}{1}".format(unique_name, i)
+                i = i + 1
+            return unique_name
+

requirements.pip

@@ -1,5 +1,5 @@
 Django==1.3
--e git://github.com/modilabs/pyxform.git@v0.9.4.1#egg=pyxform
+-e git://github.com/modilabs/pyxform.git@v0.9.4.3#egg=pyxform
 South==0.7.3
 xlrd==0.7.1
 xlwt==0.7.2