diff --git a/.gitignore b/.gitignore index d33ddee6a..faac2074e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,40 +1,42 @@ +/data +error.log +load_script.logs +load_script.pid +local_settings.py pyxform +search_index site_media/attachments - -tmp/* -*.pyc -*.sql -*.tar.gz -*~ -.DS_Store -*.sqlite3 -error.log site_media/css/sass/.sass-cache/* -.sass-cache -.odk -*orig -.idea -search_index - -#for a local virtualenv (as done in mangrove repo) -ve/ - -# file created by tests -registration.xml +tmp/* +xform_manager_dataset.json # folder to hold csv files csvs -xform_manager_dataset.json - -/data -local_settings.py -.~lock.* - # media folder used by tests # todo: figure out a way to clean this up rather than ignore it. /test_media /media -load_script.logs -load_script.pid +# file created by tests +registration.xml + +# for a local virtualenv (as done in mangrove repo) +ve/ + +.~lock.* +.DS_Store +.idea +.odk +.project +.pydevproject +.sass-cache +.settings + +*.bak +*.pyc +*.sql +*.sqlite3 +*.tar.gz +*~ +*orig diff --git a/local_settings.py.example b/local_settings.py.example new file mode 100644 index 000000000..f9a74287f --- /dev/null +++ b/local_settings.py.example @@ -0,0 +1,29 @@ +# mysql +#DATABASES = { +# 'default': { +# 'ENGINE': 'django.db.backends.mysql', +# 'NAME': 'formhub_dev', +# 'USER': 'formhub_dev', +# 'PASSWORD': '', +# } +#} + +# postgres +#DATABASES = { +# 'default': { +# 'ENGINE': 'django.db.backends.postgresql_psycopg2', +# 'NAME': 'formhub_dev', +# 'USER': 'formhub_dev', +# 'PASSWORD': '', +# } +#} + +# sqlite +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'db.sqlite3', + } +} + +TOUCHFORMS_URL = 'http://localhost:9000/' diff --git a/main/tests/test_base.py b/main/tests/test_base.py index fb54a92ed..1512e25e3 100644 --- a/main/tests/test_base.py +++ b/main/tests/test_base.py @@ -23,6 +23,11 @@ def _login(self, username, password): assert client.login(username=username, password=password) return client + def _logout(self, client=None): + if not client: + client = self.client + client.logout() + def _create_user_and_login(self, username="bob", password="bob"): self.user = self._create_user(username, password) self.client = self._login(username, password) @@ -37,6 +42,11 @@ def _publish_xls_file(self, path): post_data = {'xls_file': xls_file} return self.client.post('/%s/' % self.user.username, post_data) + def _share_form_data(self, id_string='transportation_2011_07_25'): + xform = XForm.objects.get(id_string=id_string) + xform.shared_data = True + xform.save() + def _publish_transportation_form(self): xls_path = os.path.join(self.this_directory, "fixtures", "transportation", "transportation.xls") diff --git a/main/tests/test_google_docs_export.py b/main/tests/test_google_docs_export.py index 3b2ac5bd4..e50e9fc1e 100644 --- a/main/tests/test_google_docs_export.py +++ b/main/tests/test_google_docs_export.py @@ -25,6 +25,14 @@ def test_google_docs_export(self): })) self.assertEqual(response.status_code, 302) self.assertEqual(response['Location'], 'https://docs.google.com') + # share the data, log out, and check the export + self._share_form_data() + self._logout() + response = self.client.get(reverse(google_xls_export, kwargs={ + 'username': self.user.username, + 'id_string': self.xform.id_string + })) + self.assertEqual(response.status_code, 302) def _refresh_token(self): self.assertEqual(TokenStorageModel.objects.all().count(), 0) diff --git a/odk_logger/import_tools.py b/odk_logger/import_tools.py index 4c9101cb6..a36047844 100644 --- a/odk_logger/import_tools.py +++ b/odk_logger/import_tools.py @@ -66,7 +66,8 @@ def import_instance(path_to_instance_folder, status, user): def iterate_through_odk_instances(dirpath, callback): - count = 0 + total_file_count = 0 + success_count = 0 errors = [] for directory, subdirs, subfiles in os.walk(dirpath): for filename in subfiles: @@ -74,11 +75,12 @@ def iterate_through_odk_instances(dirpath, callback): if XFormInstanceFS.is_valid_odk_instance(filepath): xfxs = XFormInstanceFS(filepath) try: - count += callback(xfxs) + success_count += callback(xfxs) except Exception, e: - errors.append(str(e)) + errors.append("%s => %s" % (xfxs.filename, str(e))) del(xfxs) - return (count, errors) + total_file_count += 1 + return (total_file_count, success_count, errors) def import_instances_from_zip(zipfile_path, user, status="zip"): @@ -86,6 +88,7 @@ def import_instances_from_zip(zipfile_path, user, status="zip"): try: temp_directory = tempfile.mkdtemp() zf = zipfile.ZipFile(zipfile_path) + zf.extractall(temp_directory) def callback(xform_fs): """ @@ -108,7 +111,7 @@ def callback(xform_fs): return 1 else: return 0 - count, errors = iterate_through_odk_instances(temp_directory, callback) + total_count, success_count, errors = iterate_through_odk_instances(temp_directory, callback) finally: shutil.rmtree(temp_directory) - return (count, errors) + return (total_count, success_count, errors) diff --git a/odk_logger/views.py b/odk_logger/views.py index b43401ce9..18bfaa1cd 100644 --- a/odk_logger/views.py +++ b/odk_logger/views.py @@ -61,12 +61,12 @@ def bulksubmission(request, username): our_tempfile.write(postfile.read()) our_tempfile.close() our_tf = open(our_tfpath, 'rb') - count, errors = import_instances_from_zip(our_tf, user=posting_user) + total_count, success_count, errors = import_instances_from_zip(our_tf, user=posting_user) os.remove(our_tfpath) json_msg = { - 'message': "Your ODK submission was successful. %d surveys imported. Your user now has %d instances." % \ - (count, posting_user.surveys.count()), - 'errors': errors + 'message': "Submission successful. Out of %d survey instances, %d were imported (%d were rejected--duplicates, missing forms, etc.)" % \ + (total_count, success_count, total_count - success_count), + 'errors': "%d %s" % (len(errors), errors) } response = HttpResponse(json.dumps(json_msg)) response.status_code = 200 @@ -170,7 +170,8 @@ def submission(request, username=None): return response def download_xform(request, username, id_string): - xform = XForm.objects.get(user__username=username, id_string=id_string) + xform = get_object_or_404(XForm, + user__username=username, id_string=id_string) # TODO: protect for users who have settings to use auth response = response_with_mimetype_and_name('xml', id_string, show_date=False) diff --git a/odk_viewer/models/parsed_instance.py b/odk_viewer/models/parsed_instance.py index 2e54dbef6..1b4d51557 100644 --- a/odk_viewer/models/parsed_instance.py +++ b/odk_viewer/models/parsed_instance.py @@ -1,6 +1,7 @@ import base64 import datetime import re +import json from bson import json_util from django.conf import settings @@ -8,6 +9,7 @@ from django.db.models.signals import post_save, pre_delete import json + from utils.model_tools import queryset_iterator from odk_logger.models import Instance from common_tags import START_TIME, START, END_TIME, END, ID, UUID, ATTACHMENTS @@ -34,7 +36,7 @@ def datetime_from_str(text): def dict_for_mongo(d): for key, value in d.items(): if type(value) == list: - value = map(dict_for_mongo, [e for e in value if type(e) == dict]) + value = [dict_for_mongo(e) if type(e) == dict else e for e in value] if type(value) == dict: value = dict_for_mongo(value) if key == '_id': @@ -111,20 +113,37 @@ def dicts(cls, xform): qs = cls.objects.filter(instance__xform=xform) for parsed_instance in queryset_iterator(qs): yield parsed_instance.to_dict() + + def _get_name_for_type(self, type_value): + """ + We cannot assume that start time and end times always use the same XPath + This is causing problems for other peoples' forms. + + This is a quick fix to determine from the original XLSForm's JSON representation + what the 'name' was for a given type_value ('start' or 'end') + """ + datadict = json.loads(self.instance.xform.json) + for item in datadict['children']: + if type(item)==dict and item.get(u'type')==type_value: + return item['name'] def _set_start_time(self): doc = self.to_dict() - if START_TIME in doc: - date_time_str = doc[START_TIME] - self.start_time = datetime_from_str(date_time_str) - elif START in doc: - date_time_str = doc[START] + start_time_key1 = self._get_name_for_type(START) + start_time_key2 = self._get_name_for_type(START_TIME) + start_time_key = start_time_key1 or start_time_key2 # if both, can take either + if start_time_key is not None and start_time_key in doc: + date_time_str = doc[start_time_key] self.start_time = datetime_from_str(date_time_str) else: self.start_time = None def _set_end_time(self): doc = self.to_dict() + end_time_key1 = self._get_name_for_type(START) + end_time_key2 = self._get_name_for_type(START_TIME) + end_time_key = end_time_key1 or end_time_key2 + if END_TIME in doc: date_time_str = doc[END_TIME] self.end_time = datetime_from_str(date_time_str) diff --git a/odk_viewer/tests/__init__.py b/odk_viewer/tests/__init__.py index 1acf6cf18..991ea3fa2 100644 --- a/odk_viewer/tests/__init__.py +++ b/odk_viewer/tests/__init__.py @@ -2,3 +2,4 @@ from form_submission import TestFormSubmission from mongo_data_output import TestMongoData from test_map_view import TestMapView +from test_exports import TestExports diff --git a/odk_viewer/tests/test_exports.py b/odk_viewer/tests/test_exports.py new file mode 100644 index 000000000..2d95f634e --- /dev/null +++ b/odk_viewer/tests/test_exports.py @@ -0,0 +1,13 @@ +from main.tests.test_base import MainTestCase +from django.core.urlresolvers import reverse +from odk_logger.views import download_xlsform +from odk_viewer.xls_writer import XlsWriter + +class TestExports(MainTestCase): + def test_unique_xls_sheet_name(self): + xls_writer = XlsWriter() + xls_writer.add_sheet('section9_pit_latrine_with_slab_group') + xls_writer.add_sheet('section9_pit_latrine_without_slab_group') + # create a set of sheet names keys + sheet_names_set = set(xls_writer._sheets.keys()) + self.assertEqual(len(sheet_names_set), 2) diff --git a/odk_viewer/views.py b/odk_viewer/views.py index ec71a6410..34096d79f 100644 --- a/odk_viewer/views.py +++ b/odk_viewer/views.py @@ -243,7 +243,7 @@ def cached_get_labels(xpath): table_rows.append('%s%s' % (key, value)) img_urls = image_urls(pi.instance) img_url = img_urls[0] if img_urls else "" - data_for_template.append({"name":id_string, "id": pi.id, "lat": pi.lat, "lng": pi.lng,'image_urls': img_urls, "table": '<%s
' % (img_url,''.join(table_rows))}) + data_for_template.append({"name":id_string, "id": pi.id, "lat": pi.lat, "lng": pi.lng,'image_urls': img_urls, "table": '%s
' % (img_url,''.join(table_rows))}) context.data = data_for_template response = render_to_response("survey.kml", context_instance=context, @@ -252,6 +252,7 @@ def cached_get_labels(xpath): return response +@login_required def google_xls_export(request, username, id_string): owner = User.objects.get(username=username) xform = XForm.objects.get(id_string=id_string, user=owner) diff --git a/odk_viewer/xls_writer.py b/odk_viewer/xls_writer.py index 0a908a00b..91f7495ff 100644 --- a/odk_viewer/xls_writer.py +++ b/odk_viewer/xls_writer.py @@ -7,6 +7,8 @@ class XlsWriter(object): def __init__(self): self.set_file() self.reset_workbook() + self.sheet_name_limit = 30 + self._generated_sheet_name_dict = {} def set_file(self, file_object=None): """ @@ -25,10 +27,12 @@ def reset_workbook(self): self._columns = defaultdict(list) def one(): return 1 self._current_index = defaultdict(one) + self._generated_sheet_name_dict = {} def add_sheet(self, name): - sheet = self._workbook.add_sheet(name[0:20]) - self._sheets[name] = sheet + unique_sheet_name = self._unique_name_for_xls(name) + sheet = self._workbook.add_sheet(unique_sheet_name) + self._sheets[unique_sheet_name] = sheet def add_column(self, sheet_name, column_name): index = len(self._columns[sheet_name]) @@ -51,7 +55,9 @@ def add_obs(self, obs): self._fix_indices(obs) for sheet_name, rows in obs.items(): for row in rows: - self.add_row(sheet_name, row) + actual_sheet_name = self._generated_sheet_name_dict.get( + sheet_name, sheet_name) + self.add_row(actual_sheet_name, row) def _fix_indices(self, obs): for sheet_name, rows in obs.items(): @@ -66,7 +72,7 @@ def write_tables_to_workbook(self, tables): tables should be a list of pairs, the first element in the pair is the name of the table, the second is the actual data. - todo: figure out how to write to the xls file rather than keep + TODO: figure out how to write to the xls file rather than keep the whole workbook in memory. """ self.reset_workbook() @@ -98,3 +104,28 @@ def _add_sheets(self): if isinstance(f, Question) and\ not question_types_to_exclude(f.type): self.add_column(sheet_name, f.name) + + def _unique_name_for_xls(self, sheet_name): + # excel worksheet name limit seems to be 31 characters (30 to be safe) + unique_sheet_name = sheet_name[0:self.sheet_name_limit] + unique_sheet_name = self._generate_unique_sheet_name(unique_sheet_name) + self._generated_sheet_name_dict[sheet_name] = unique_sheet_name + return unique_sheet_name + + def _generate_unique_sheet_name(self, sheet_name): + # check if sheet name exists + if(not self._sheets.has_key(sheet_name)): + return sheet_name + else: + i = 1 + unique_name = sheet_name + while(self._sheets.has_key(unique_name)): + number_len = len(str(i)) + allowed_name_len = self.sheet_name_limit - number_len + # make name required len + if(len(unique_name) > allowed_name_len): + unique_name = unique_name[0:allowed_name_len] + unique_name = "{0}{1}".format(unique_name, i) + i = i + 1 + return unique_name + diff --git a/requirements.pip b/requirements.pip index e3410a508..8fab3f1dd 100644 --- a/requirements.pip +++ b/requirements.pip @@ -1,5 +1,5 @@ Django==1.3 --e git://github.com/modilabs/pyxform.git@v0.9.4.1#egg=pyxform +-e git://github.com/modilabs/pyxform.git@v0.9.4.3#egg=pyxform South==0.7.3 xlrd==0.7.1 xlwt==0.7.2 diff --git a/templates/base.html b/templates/base.html index 60b644379..611f6e856 100644 --- a/templates/base.html +++ b/templates/base.html @@ -5,6 +5,7 @@ formhub +