fix bug running cumulative csv briefcase rows, more tests

smartvadhis2/core/briefcase.py

@@ -26,7 +26,6 @@ def __init__(self):
 
         self._log_version()
         self.timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-        self.filename = "briefcases.csv"
 
     def _get_arguments(self, all_briefcases):
         """Create the argument list to provide to the Briefcase JAR
@@ -41,13 +40,13 @@ def _get_arguments(self, all_briefcases):
             '--aggregate_url', ODKConfig.baseurl,
             '--odk_username', ODKConfig.username,
             '--odk_password', ODKConfig.password,
-            '--export_filename', self.filename,
             '--exclude_media_export'
         ]
         logger.info("Connecting to ODK Briefcase on {} ...".format(ODKConfig.baseurl))
 
         if not all_briefcases:
             start, end = get_timewindow()
+            export_filename = "briefcases_from_{}_at_{}.csv".format(start.replace('/', ''), self.timestamp)
             time_window = [
                 '--export_start_date', start,
                 '--export_end_date', end
@@ -56,7 +55,10 @@ def _get_arguments(self, all_briefcases):
             logger.info("Fetching briefcases from {} to {} ...".format(start, end))
         else:
             logger.info("Fetching ALL briefcases...")
-        return arguments
+            export_filename = "briefcases_all_{}.csv".format(self.timestamp)
+
+        arguments.extend(['--export_filename', export_filename])
+        return arguments, export_filename
 
     def _log_version(self):
         with subprocess.Popen(['java', '-jar', self.jar_path, '-v'],
@@ -79,18 +81,18 @@ def _log_subprocess_output(process):
                 raise BriefcaseException(line)
 
             else:
-                # remove timestamp for better readabilityf
+                # remove timestamp for better readability
                 line = re.sub(r'^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2},\d{3}\s', '', line)
                 logger.info(str(line).replace('\n', ''))
 
     def download_briefcases(self, all_briefcases):
         """Do the actual call to the JAR file and log output messages"""
-        args = self._get_arguments(all_briefcases)
+        args, filename = self._get_arguments(all_briefcases)
         with subprocess.Popen(args,
                               bufsize=1,
                               universal_newlines=True,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT) as process:
             self._log_subprocess_output(process)
-        return os.path.join(ODKConfig.briefcases_dir, self.filename)
+        return os.path.join(ODKConfig.briefcases_dir, filename)
 

smartvadhis2/core/dhis.py

@@ -33,7 +33,7 @@ def __init__(self, response):
             self.updated = int(response['response']['updated'])
             self.ignored = int(response['response']['ignored'])
             self.deleted = int(response['response']['deleted'])
-        except (ValueError, KeyError):
+        except (ValueError, KeyError, TypeError):
             logger.debug(response)
             raise GenericImportError("Error parsing response: {}".format(response))
 
@@ -144,11 +144,15 @@ def root_orgunit(self):
             'filter': 'level:eq:1'
         }
         req = self.get(endpoint='organisationUnits', params=params).json()
-        if len(req['organisationUnits']) > 1:
+        return self._get_root_id(req)
+
+    @staticmethod
+    def _get_root_id(response):
+        if len(response['organisationUnits']) > 1:
             raise DhisApiException("More than one Organisation Units found. Can not proceed.")
-        if len(req['organisationUnits']) == 0:
+        if len(response['organisationUnits']) == 0:
             raise DhisApiException("No Organisation Unit found. Can not proceed.")
-        return req['organisationUnits'][0]['id']
+        return response['organisationUnits'][0]['id']
 
     def assign_orgunit_to_program(self, data):
         """Assign OrgUnit to program"""

smartvadhis2/core/exceptions/errors.py

@@ -33,17 +33,8 @@ def __init__(self):
         super(ValidationError, self).__init__(self.message, self.code)
 
 
-class DeathDateMissingError(ValidationError):
-    code = 602
-    err_type = 'VALIDATION'
-    message = "[death_date] missing"
-
-    def __init__(self):
-        super(ValidationError, self).__init__(self.message, self.code)
-
-
 class AgeParseError(ValidationError):
-    code = 603
+    code = 602
     err_type = 'VALIDATION'
     message = "Could not parse [age] as Integer"
 
@@ -52,7 +43,7 @@ def __init__(self):
 
 
 class AgeOutOfBoundsError(ValidationError):
-    code = 604
+    code = 603
     err_type = 'VALIDATION'
     message = "[age] is not between 0 and 120 years"
 
@@ -61,25 +52,16 @@ def __init__(self):
 
 
 class AgeMissingError(ValidationError):
-    code = 605
+    code = 604
     err_type = 'VALIDATION'
     message = "[age] is missing"
 
     def __init__(self):
         super(ValidationError, self).__init__(self.message, self.code)
 
 
-class CauseOfDeathMissingError(ValidationError):
-    code = 606
-    err_type = 'VALIDATION'
-    message = "[cause34] (cause of death) is missing"
-
-    def __init__(self):
-        super(ValidationError, self).__init__(self.message, self.code)
-
-
 class Icd10ParseError(ValidationError):
-    code = 607
+    code = 605
     err_type = 'VALIDATION'
     message = "[icd10] does not match mapping"
 
@@ -88,7 +70,7 @@ def __init__(self):
 
 
 class Icd10MissingError(ValidationError):
-    code = 608
+    code = 606
     err_type = 'VALIDATION'
     message = "[icd10] missing"
 
@@ -97,7 +79,7 @@ def __init__(self):
 
 
 class SexParseError(ValidationError):
-    code = 609
+    code = 607
     err_type = 'VALIDATION'
     message = "[sex] is not an Integer in (1, 2, 3, 8, 9)"
 
@@ -106,7 +88,7 @@ def __init__(self):
 
 
 class SexMissingError(ValidationError):
-    code = 610
+    code = 608
     err_type = 'VALIDATION'
     message = "[sex] is missing"
 
@@ -115,7 +97,7 @@ def __init__(self):
 
 
 class SidParseError(ValidationError):
-    code = 611
+    code = 609
     err_type = 'VALIDATION'
     message = "[sid] does not match regex expression"
 
@@ -124,7 +106,7 @@ def __init__(self):
 
 
 class SidMissingError(ValidationError):
-    code = 612
+    code = 610
     err_type = 'VALIDATION'
     message = "[sid] is missing"
 
@@ -133,7 +115,7 @@ def __init__(self):
 
 
 class OrgunitMissingError(ValidationError):
-    code = 613
+    code = 611
     err_type = 'VALIDATION'
     message = "orgunit is missing"
 
@@ -142,7 +124,7 @@ def __init__(self):
 
 
 class OrgunitNotValidError(ValidationError):
-    code = 614
+    code = 612
     err_type = 'VALIDATION'
     message = "orgunit UID is not a valid UID"
 
@@ -212,11 +194,9 @@ def __init__(self, response):
     'ValidationError',
     'BirthDateParseError',
     'DeathDateParseError',
-    'DeathDateMissingError',
     'AgeParseError',
     'AgeOutOfBoundsError',
     'AgeMissingError',
-    'CauseOfDeathMissingError',
     'Icd10ParseError',
     'Icd10MissingError',
     'SexParseError',

smartvadhis2/core/helpers.py

@@ -1,6 +1,5 @@
 import csv
 import datetime
-import hashlib
 import os
 import re
 
@@ -38,19 +37,8 @@ def read_csv(path):
             yield row
 
 
-def sha256_checksum(filename, block_size=65536):
-    """Calculate checksum on file"""
-    sha256 = hashlib.sha256()
-    with open(filename, 'rb') as f:
-        for block in iter(lambda: f.read(block_size), b''):
-            sha256.update(block)
-    return sha256.hexdigest()
-
-
 def csv_with_content(fpath):
-    """Return true if file is existing AND file has content, false otherwise
-    If it's a csv file, open it and count the rows
-    """
+    """Return true if file exists AND file has more than 1 row, false otherwise"""
     if fpath and os.path.exists(fpath):
         row_count = sum(1 for line in open(fpath))
         return row_count > 1

smartvadhis2/core/verbalautopsy.py

@@ -77,17 +77,10 @@ def verbal_autopsy_factory(data):
             exceptions.append(e)
         except ValidationWarning as e:
             warnings.append(e)
-        except Exception as e:
-            logger.exception((e, k, v))
 
     # set attributes not in CSV
-    try:
-        va.algorithm_version = SmartVAConfig.algorithm_version
-        va.questionnaire_version = ODKConfig.form_id
-    except ValidationWarning as e:
-        warnings.append(e)
-    except ValidationError as e:
-        exceptions.append(e)
+    va.algorithm_version = SmartVAConfig.algorithm_version
+    va.questionnaire_version = ODKConfig.form_id
 
     return va, exceptions, warnings
 
@@ -110,10 +103,6 @@ def __getattr__(self, _):
         """Return None if instance attribute does not exist"""
         return None
 
-    def keys(self):
-        """Return keys for attributes that are not 'private' or methods"""
-        return [k for k in dir(self) if not k.startswith('_') and not callable(self[k])]
-
     def __str__(self):
         """Print VerbalAutopsy instance as JSON"""
         return json.dumps(dict(self))
@@ -382,24 +371,17 @@ def __init__(self, va):
         self.program = DhisConfig.program_uid
         self.orgunit = va.orgunit
         self.datavalues = va
+        self.event_date = va.death_date
 
         self.payload = {
             "program": self.program,
-            "orgUnit": va.orgunit,
-            "eventDate": va.death_date,
+            "orgUnit": self.orgunit,
+            "eventDate": self.event_date,
             "status": "COMPLETED",
             "storedBy": "smartvadhis2_v{}".format(__version__),
             "dataValues": self.datavalues
         }
 
-    @property
-    def program(self):
-        return self._program
-
-    @program.setter
-    def program(self, value):
-        self._program = value
-
     @property
     def datavalues(self):
         return self._datavalues

smartvadhis2/run.py

@@ -73,6 +73,7 @@ def _run(manual, download_all):
     else:
         briefcase_file = briefcase.download_briefcases(download_all)
         smartva_file = None
+        print(briefcase_file)
         if csv_with_content(briefcase_file):
             smartva_file = smartva.run(briefcase_file)
         else:
@@ -123,4 +124,3 @@ def launch():
         logger.warning("Aborted!")
     except Exception as e:
         logger.exception(e)
-        raise SmartVADHIS2Exception(e)

tests/test_briefcase.py

@@ -12,16 +12,16 @@ def test_briefcase_jar_exists():
 
 def test_briefcase_args_timewindows():
     briefcase = ODKBriefcase()
-    actual = briefcase._get_arguments(all_briefcases=False)
-    assert actual[18] == '--export_start_date'
-    assert actual[20] == '--export_end_date'
+    actual, filename = briefcase._get_arguments(all_briefcases=False)
+    assert filename
+    assert actual[16] == '--export_start_date'
+    assert actual[18] == '--export_end_date'
     assert len(actual) == 22
 
 
 def test_briefcase_args_all():
-    now = datetime.datetime.now()
     briefcase = ODKBriefcase()
-    actual = briefcase._get_arguments(all_briefcases=True)
+    actual, filename = briefcase._get_arguments(all_briefcases=True)
     expected = [
         'java', '-jar', briefcase.jar_path,
         '--storage_directory', ODKConfig.briefcases_dir,
@@ -30,11 +30,11 @@ def test_briefcase_args_all():
         '--aggregate_url', ODKConfig.baseurl,
         '--odk_username', ODKConfig.username,
         '--odk_password', ODKConfig.password,
-        #'--export_filename', 'briefcase_{}.csv'.format(now.strftime('%Y%m%d_%H%M%S')),
         '--exclude_media_export'
     ]
     for o in expected:
         assert o in actual
 
+    assert filename
     assert '--export_start_date' not in actual
     assert '--export_end_date' not in actual

tests/test_cli.py

@@ -614,19 +614,17 @@ def test_print_error_categories(capsys):
     expected = "Validation Errors (600-699)\n" \
                "- ID:600 - Could not parse [birth_date]\n" \
                "- ID:601 - Could not parse [death_date]\n" \
-               "- ID:602 - [death_date] missing\n" \
-               "- ID:603 - Could not parse [age] as Integer\n" \
-               "- ID:604 - [age] is not between 0 and 120 years\n" \
-               "- ID:605 - [age] is missing\n" \
-               "- ID:606 - [cause34] (cause of death) is missing\n" \
-               "- ID:607 - [icd10] does not match mapping\n" \
-               "- ID:608 - [icd10] missing\n" \
-               "- ID:609 - [sex] is not an Integer in (1, 2, 3, 8, 9)\n" \
-               "- ID:610 - [sex] is missing\n" \
-               "- ID:611 - [sid] does not match regex expression\n" \
-               "- ID:612 - [sid] is missing\n" \
-               "- ID:613 - orgunit is missing\n" \
-               "- ID:614 - orgunit UID is not a valid UID\n" \
+               "- ID:602 - Could not parse [age] as Integer\n" \
+               "- ID:603 - [age] is not between 0 and 120 years\n" \
+               "- ID:604 - [age] is missing\n" \
+               "- ID:605 - [icd10] does not match mapping\n" \
+               "- ID:606 - [icd10] missing\n" \
+               "- ID:607 - [sex] is not an Integer in (1, 2, 3, 8, 9)\n" \
+               "- ID:608 - [sex] is missing\n" \
+               "- ID:609 - [sid] does not match regex expression\n" \
+               "- ID:610 - [sid] is missing\n" \
+               "- ID:611 - orgunit is missing\n" \
+               "- ID:612 - orgunit UID is not a valid UID\n" \
                "Import Errors (700-799)\n" \
                "- ID:700 - OrgUnit is not a valid UID\n" \
                "- ID:701 - Program is not a valid program\n" \