Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge branch 'master' of github.com:modilabs/formhub into map-filter-…

…select_one-in-filter
  • Loading branch information...
commit 8ed3d737091de51aa307c6cd78589ca5fdb711db 2 parents 7398f8d + f0498d3
Peter Lubell-Doughtie pld authored
58 .gitignore
View
@@ -1,40 +1,42 @@
+/data
+error.log
+load_script.logs
+load_script.pid
+local_settings.py
pyxform
+search_index
site_media/attachments
-
-tmp/*
-*.pyc
-*.sql
-*.tar.gz
-*~
-.DS_Store
-*.sqlite3
-error.log
site_media/css/sass/.sass-cache/*
-.sass-cache
-.odk
-*orig
-.idea
-search_index
-
-#for a local virtualenv (as done in mangrove repo)
-ve/
-
-# file created by tests
-registration.xml
+tmp/*
+xform_manager_dataset.json
# folder to hold csv files
csvs
-xform_manager_dataset.json
-
-/data
-local_settings.py
-.~lock.*
-
# media folder used by tests
# todo: figure out a way to clean this up rather than ignore it.
/test_media
/media
-load_script.logs
-load_script.pid
+# file created by tests
+registration.xml
+
+# for a local virtualenv (as done in mangrove repo)
+ve/
+
+.~lock.*
+.DS_Store
+.idea
+.odk
+.project
+.pydevproject
+.sass-cache
+.settings
+
+*.bak
+*.pyc
+*.sql
+*.sqlite3
+*.tar.gz
+*~
+*orig
29 local_settings.py.example
View
@@ -0,0 +1,29 @@
+# mysql
+#DATABASES = {
+# 'default': {
+# 'ENGINE': 'django.db.backends.mysql',
+# 'NAME': 'formhub_dev',
+# 'USER': 'formhub_dev',
+# 'PASSWORD': '',
+# }
+#}
+
+# postgres
+#DATABASES = {
+# 'default': {
+# 'ENGINE': 'django.db.backends.postgresql_psycopg2',
+# 'NAME': 'formhub_dev',
+# 'USER': 'formhub_dev',
+# 'PASSWORD': '',
+# }
+#}
+
+# sqlite
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.sqlite3',
+ 'NAME': 'db.sqlite3',
+ }
+}
+
+TOUCHFORMS_URL = 'http://localhost:9000/'
10 main/tests/test_base.py
View
@@ -23,6 +23,11 @@ def _login(self, username, password):
assert client.login(username=username, password=password)
return client
+ def _logout(self, client=None):
+ if not client:
+ client = self.client
+ client.logout()
+
def _create_user_and_login(self, username="bob", password="bob"):
self.user = self._create_user(username, password)
self.client = self._login(username, password)
@@ -37,6 +42,11 @@ def _publish_xls_file(self, path):
post_data = {'xls_file': xls_file}
return self.client.post('/%s/' % self.user.username, post_data)
+ def _share_form_data(self, id_string='transportation_2011_07_25'):
+ xform = XForm.objects.get(id_string=id_string)
+ xform.shared_data = True
+ xform.save()
+
def _publish_transportation_form(self):
xls_path = os.path.join(self.this_directory, "fixtures",
"transportation", "transportation.xls")
8 main/tests/test_google_docs_export.py
View
@@ -25,6 +25,14 @@ def test_google_docs_export(self):
}))
self.assertEqual(response.status_code, 302)
self.assertEqual(response['Location'], 'https://docs.google.com')
+ # share the data, log out, and check the export
+ self._share_form_data()
+ self._logout()
+ response = self.client.get(reverse(google_xls_export, kwargs={
+ 'username': self.user.username,
+ 'id_string': self.xform.id_string
+ }))
+ self.assertEqual(response.status_code, 302)
def _refresh_token(self):
self.assertEqual(TokenStorageModel.objects.all().count(), 0)
15 odk_logger/import_tools.py
View
@@ -66,7 +66,8 @@ def import_instance(path_to_instance_folder, status, user):
def iterate_through_odk_instances(dirpath, callback):
- count = 0
+ total_file_count = 0
+ success_count = 0
errors = []
for directory, subdirs, subfiles in os.walk(dirpath):
for filename in subfiles:
@@ -74,11 +75,12 @@ def iterate_through_odk_instances(dirpath, callback):
if XFormInstanceFS.is_valid_odk_instance(filepath):
xfxs = XFormInstanceFS(filepath)
try:
- count += callback(xfxs)
+ success_count += callback(xfxs)
except Exception, e:
- errors.append(str(e))
+ errors.append("%s => %s" % (xfxs.filename, str(e)))
del(xfxs)
- return (count, errors)
+ total_file_count += 1
+ return (total_file_count, success_count, errors)
def import_instances_from_zip(zipfile_path, user, status="zip"):
@@ -86,6 +88,7 @@ def import_instances_from_zip(zipfile_path, user, status="zip"):
try:
temp_directory = tempfile.mkdtemp()
zf = zipfile.ZipFile(zipfile_path)
+
zf.extractall(temp_directory)
def callback(xform_fs):
"""
@@ -108,7 +111,7 @@ def callback(xform_fs):
return 1
else:
return 0
- count, errors = iterate_through_odk_instances(temp_directory, callback)
+ total_count, success_count, errors = iterate_through_odk_instances(temp_directory, callback)
finally:
shutil.rmtree(temp_directory)
- return (count, errors)
+ return (total_count, success_count, errors)
11 odk_logger/views.py
View
@@ -61,12 +61,12 @@ def bulksubmission(request, username):
our_tempfile.write(postfile.read())
our_tempfile.close()
our_tf = open(our_tfpath, 'rb')
- count, errors = import_instances_from_zip(our_tf, user=posting_user)
+ total_count, success_count, errors = import_instances_from_zip(our_tf, user=posting_user)
os.remove(our_tfpath)
json_msg = {
- 'message': "Your ODK submission was successful. %d surveys imported. Your user now has %d instances." % \
- (count, posting_user.surveys.count()),
- 'errors': errors
+ 'message': "Submission successful. Out of %d survey instances, %d were imported (%d were rejected--duplicates, missing forms, etc.)" % \
+ (total_count, success_count, total_count - success_count),
+ 'errors': "%d %s" % (len(errors), errors)
}
response = HttpResponse(json.dumps(json_msg))
response.status_code = 200
@@ -170,7 +170,8 @@ def submission(request, username=None):
return response
def download_xform(request, username, id_string):
- xform = XForm.objects.get(user__username=username, id_string=id_string)
+ xform = get_object_or_404(XForm,
+ user__username=username, id_string=id_string)
# TODO: protect for users who have settings to use auth
response = response_with_mimetype_and_name('xml', id_string,
show_date=False)
31 odk_viewer/models/parsed_instance.py
View
@@ -1,6 +1,7 @@
import base64
import datetime
import re
+import json
from bson import json_util
from django.conf import settings
@@ -8,6 +9,7 @@
from django.db.models.signals import post_save, pre_delete
import json
+
from utils.model_tools import queryset_iterator
from odk_logger.models import Instance
from common_tags import START_TIME, START, END_TIME, END, ID, UUID, ATTACHMENTS
@@ -34,7 +36,7 @@ def datetime_from_str(text):
def dict_for_mongo(d):
for key, value in d.items():
if type(value) == list:
- value = map(dict_for_mongo, [e for e in value if type(e) == dict])
+ value = [dict_for_mongo(e) if type(e) == dict else e for e in value]
if type(value) == dict:
value = dict_for_mongo(value)
if key == '_id':
@@ -111,20 +113,37 @@ def dicts(cls, xform):
qs = cls.objects.filter(instance__xform=xform)
for parsed_instance in queryset_iterator(qs):
yield parsed_instance.to_dict()
+
+ def _get_name_for_type(self, type_value):
+ """
+ We cannot assume that start time and end times always use the same XPath
+ This is causing problems for other peoples' forms.
+
+ This is a quick fix to determine from the original XLSForm's JSON representation
+ what the 'name' was for a given type_value ('start' or 'end')
+ """
+ datadict = json.loads(self.instance.xform.json)
+ for item in datadict['children']:
+ if type(item)==dict and item.get(u'type')==type_value:
+ return item['name']
def _set_start_time(self):
doc = self.to_dict()
- if START_TIME in doc:
- date_time_str = doc[START_TIME]
- self.start_time = datetime_from_str(date_time_str)
- elif START in doc:
- date_time_str = doc[START]
+ start_time_key1 = self._get_name_for_type(START)
+ start_time_key2 = self._get_name_for_type(START_TIME)
+ start_time_key = start_time_key1 or start_time_key2 # if both, can take either
+ if start_time_key is not None and start_time_key in doc:
+ date_time_str = doc[start_time_key]
self.start_time = datetime_from_str(date_time_str)
else:
self.start_time = None
def _set_end_time(self):
doc = self.to_dict()
+ end_time_key1 = self._get_name_for_type(START)
+ end_time_key2 = self._get_name_for_type(START_TIME)
+ end_time_key = end_time_key1 or end_time_key2
+
if END_TIME in doc:
date_time_str = doc[END_TIME]
self.end_time = datetime_from_str(date_time_str)
1  odk_viewer/tests/__init__.py
View
@@ -2,3 +2,4 @@
from form_submission import TestFormSubmission
from mongo_data_output import TestMongoData
from test_map_view import TestMapView
+from test_exports import TestExports
13 odk_viewer/tests/test_exports.py
View
@@ -0,0 +1,13 @@
+from main.tests.test_base import MainTestCase
+from django.core.urlresolvers import reverse
+from odk_logger.views import download_xlsform
+from odk_viewer.xls_writer import XlsWriter
+
+class TestExports(MainTestCase):
+ def test_unique_xls_sheet_name(self):
+ xls_writer = XlsWriter()
+ xls_writer.add_sheet('section9_pit_latrine_with_slab_group')
+ xls_writer.add_sheet('section9_pit_latrine_without_slab_group')
+ # create a set of sheet names keys
+ sheet_names_set = set(xls_writer._sheets.keys())
+ self.assertEqual(len(sheet_names_set), 2)
3  odk_viewer/views.py
View
@@ -243,7 +243,7 @@ def cached_get_labels(xpath):
table_rows.append('<tr><td>%s</td><td>%s</td></tr>' % (key, value))
img_urls = image_urls(pi.instance)
img_url = img_urls[0] if img_urls else ""
- data_for_template.append({"name":id_string, "id": pi.id, "lat": pi.lat, "lng": pi.lng,'image_urls': img_urls, "table": '<table border="1"><a href="#"><img width="210" class="thumbnail" src="%s" alt=""></a><%s</table>' % (img_url,''.join(table_rows))})
+ data_for_template.append({"name":id_string, "id": pi.id, "lat": pi.lat, "lng": pi.lng,'image_urls': img_urls, "table": '<table border="1"><a href="#"><img width="210" class="thumbnail" src="%s" alt=""></a>%s</table>' % (img_url,''.join(table_rows))})
context.data = data_for_template
response = render_to_response("survey.kml",
context_instance=context,
@@ -252,6 +252,7 @@ def cached_get_labels(xpath):
return response
+@login_required
def google_xls_export(request, username, id_string):
owner = User.objects.get(username=username)
xform = XForm.objects.get(id_string=id_string, user=owner)
39 odk_viewer/xls_writer.py
View
@@ -7,6 +7,8 @@ class XlsWriter(object):
def __init__(self):
self.set_file()
self.reset_workbook()
+ self.sheet_name_limit = 30
+ self._generated_sheet_name_dict = {}
def set_file(self, file_object=None):
"""
@@ -25,10 +27,12 @@ def reset_workbook(self):
self._columns = defaultdict(list)
def one(): return 1
self._current_index = defaultdict(one)
+ self._generated_sheet_name_dict = {}
def add_sheet(self, name):
- sheet = self._workbook.add_sheet(name[0:20])
- self._sheets[name] = sheet
+ unique_sheet_name = self._unique_name_for_xls(name)
+ sheet = self._workbook.add_sheet(unique_sheet_name)
+ self._sheets[unique_sheet_name] = sheet
def add_column(self, sheet_name, column_name):
index = len(self._columns[sheet_name])
@@ -51,7 +55,9 @@ def add_obs(self, obs):
self._fix_indices(obs)
for sheet_name, rows in obs.items():
for row in rows:
- self.add_row(sheet_name, row)
+ actual_sheet_name = self._generated_sheet_name_dict.get(
+ sheet_name, sheet_name)
+ self.add_row(actual_sheet_name, row)
def _fix_indices(self, obs):
for sheet_name, rows in obs.items():
@@ -66,7 +72,7 @@ def write_tables_to_workbook(self, tables):
tables should be a list of pairs, the first element in the
pair is the name of the table, the second is the actual data.
- todo: figure out how to write to the xls file rather than keep
+ TODO: figure out how to write to the xls file rather than keep
the whole workbook in memory.
"""
self.reset_workbook()
@@ -98,3 +104,28 @@ def _add_sheets(self):
if isinstance(f, Question) and\
not question_types_to_exclude(f.type):
self.add_column(sheet_name, f.name)
+
+ def _unique_name_for_xls(self, sheet_name):
+ # excel worksheet name limit seems to be 31 characters (30 to be safe)
+ unique_sheet_name = sheet_name[0:self.sheet_name_limit]
+ unique_sheet_name = self._generate_unique_sheet_name(unique_sheet_name)
+ self._generated_sheet_name_dict[sheet_name] = unique_sheet_name
+ return unique_sheet_name
+
+ def _generate_unique_sheet_name(self, sheet_name):
+ # check if sheet name exists
+ if(not self._sheets.has_key(sheet_name)):
+ return sheet_name
+ else:
+ i = 1
+ unique_name = sheet_name
+ while(self._sheets.has_key(unique_name)):
+ number_len = len(str(i))
+ allowed_name_len = self.sheet_name_limit - number_len
+ # make name required len
+ if(len(unique_name) > allowed_name_len):
+ unique_name = unique_name[0:allowed_name_len]
+ unique_name = "{0}{1}".format(unique_name, i)
+ i = i + 1
+ return unique_name
+
2  requirements.pip
View
@@ -1,5 +1,5 @@
Django==1.3
--e git://github.com/modilabs/pyxform.git@v0.9.4.1#egg=pyxform
+-e git://github.com/modilabs/pyxform.git@v0.9.4.3#egg=pyxform
South==0.7.3
xlrd==0.7.1
xlwt==0.7.2
1  templates/base.html
View
@@ -5,6 +5,7 @@
<title>formhub</title>
<meta name="description" content="">
<meta name="author" content="">
+ <meta name="google-site-verification" content="8pAiKPreksD8GiYnSCtWm9QG3i6wyuKg7E2Y95RTzRM" />
<!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
<!--[if lt IE 9]>
Please sign in to comment.
Something went wrong with that request. Please try again.