added scheduling and more intelligent handling of fresh data

Pipfile

@@ -1,27 +1,21 @@
 [[source]]
-
 url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 
-
 [dev-packages]
-
 pytest = "*"
 pytest-coverage = "*"
 sphinx = "*"
 sphinx-rtd-theme = "*"
 
-
 [packages]
-
 requests = "*"
 records = "*"
 logzero = "*"
 alembic = "*"
 colorama = {markers = "sys_platform == 'win32'"}
-
+apscheduler = "*"
 
 [requires]
-
 python_version = "3.5"

setup.py

@@ -74,8 +74,7 @@ def run(self):
     version=__version__,
     description='Integration of Verbal Autopsy data into DHIS2.',
     long_description=readme,
-    author='David Huser',
-    author_email='dhuser@baosystems.com',
+    author='Data For Health Initiative - Verbal Autopsy',
     url='https://github.com/D4H-VA/smartva-dhis2',
     keywords='smartva verbal autopsy dhis2 tariff odk',
     license='MIT',
@@ -85,6 +84,12 @@ def run(self):
         'logzero',
         'alembic'
     ],
+    entry_points={
+        'console_scripts': [
+            'smartvadhis2 = smartvadhis2.run.smartvadhis2_run',
+            'smartvadhis2-cli = smartvadhis2.cli.main'
+        ]
+    },
     classifiers=[
         # Trove classifiers
         # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers

smartvadhis2/__main__.py

@@ -1,66 +1,5 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-import sys
-
-from logzero import logger
-
-from .core.config import setup as setup_with_config
-from .core.helpers import read_csv, parse_args
-from .core.verbalautopsy import Event, verbal_autopsy_factory
-from .core.exceptions.base import SmartVADHIS2Exception
-from .core.exceptions.errors import ImportException, DuplicateEventImportError
-
-
-def smartva_to_dhis2(db, dhis, smartva_file):
-    if smartva_file:
-        for i, record in enumerate(read_csv(smartva_file), 1):
-            logger.info("{0} ROW NUMBER: {1} {0}".format('----------', i))
-            va, exceptions, warnings = verbal_autopsy_factory(record)
-            if warnings:
-                [logger.warn("{} for record {}".format(warning, record)) for warning in warnings]
-            if not exceptions:
-                event = Event(va)
-                try:
-                    dhis.is_duplicate(va.sid)
-                except DuplicateEventImportError as e:
-                    logger.exception(e)
-                    db.write_errors(va, e)
-                else:
-                    try:
-                        dhis.post_event(event.payload)
-                    except ImportException as e:
-                        logger.exception("{}\nfor payload {}".format(e, event.payload))
-                        db.write_errors(va, [e])
-            else:
-                [logger.exception("{} for record {}".format(exception, record)) for exception in exceptions]
-                db.write_errors(va, exceptions)
-
-
-def run(arguments):
-    dhis, briefcase, smartva, db = setup_with_config()
-
-    smartva_file = None
-    if arguments.briefcase_file:
-        smartva_file = smartva.run(arguments.briefcase_file, manual=True)
-    else:
-        briefcase_file = briefcase.download_briefcases(arguments.all)
-        if briefcase_file:
-            smartva_file = smartva.run(briefcase_file)
-        else:
-            logger.warning("No new briefcases downloaded")
-
-    smartva_to_dhis2(db, dhis, smartva_file)
-
-
-def main():
-    try:
-        args = parse_args(sys.argv[1:])
-        run(args)
-    except KeyboardInterrupt:
-        logger.warning("Aborted!")
-    except Exception as e:
-        raise SmartVADHIS2Exception(e)
+from .run import launch
 
 
 if __name__ == '__main__':
-    main()
+    launch()

smartvadhis2/core/briefcase.py

@@ -23,7 +23,7 @@ def __init__(self):
 
         self._log_version()
         self.timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-        self.filename = "briefcase_{}.csv".format(self.timestamp)
+        self.filename = "briefcases.csv"
 
     def _get_arguments(self, all_briefcases):
         """Create the argument list to provide to the Briefcase JAR
@@ -75,6 +75,5 @@ def download_briefcases(self, all_briefcases):
                                   stderr=subprocess.STDOUT) as process:
                 log_subprocess_output(process)
             return os.path.join(ODKConfig.briefcases_dir, self.filename)
-
         except subprocess.CalledProcessError as e:
             logger.exception(e)

smartvadhis2/core/database.py

@@ -136,7 +136,7 @@ def _to_sql_rows(data):
         """Convert data rows to a dict ready for insertion"""
         try:
             d = {
-                mapping.code_name: data[mapping.code_name]
+                mapping.code_name: data[mapping.csv_name]
                 for mapping in Mapping.properties()
                 if mapping.csv_name is not None
             }

smartvadhis2/core/dhis.py

@@ -89,6 +89,7 @@ def __init__(self):
         elif not url.startswith('https://'):
             url = 'https://{}'.format(url)
 
+        logger.info("Connecting to DHIS2 on {} ...".format(url))
         self.api_url = '{}/api/{}'.format(url, api_version)
         self.api = requests.Session()
         self.auth = (DhisConfig.username, DhisConfig.password)
@@ -99,7 +100,7 @@ def __init__(self):
     def get(self, endpoint, params=None):
         """DHIS2 HTTP GET, returns requests.Response object"""
         url = '{}/{}.json'.format(self.api_url, endpoint)
-        logger.debug('GET: {} - Params: {}'.format(url, params))
+        # logger.debug('GET: {} - Params: {}'.format(url, params))
         return self.api.get(url, params=params, auth=self.auth, headers=self.headers)
 
     def post(self, endpoint, data, params=None):

smartvadhis2/core/exceptions/base.py

@@ -16,3 +16,9 @@ class FileException(SmartVADHIS2Exception):
 class DhisApiException(SmartVADHIS2Exception):
     """Exceptions involving DHIS2"""
     pass
+
+
+class NoODKDataException(SmartVADHIS2Exception):
+    """Exception when there is no new data downloaded"""
+    pass
+

smartvadhis2/core/helpers.py

@@ -8,7 +8,7 @@
 from logzero import logger
 
 from .config import SmartVAConfig
-from .exceptions import FileException
+from .exceptions import FileException, NoODKDataException
 from .mapping import Mapping
 
 """
@@ -26,40 +26,17 @@ class Color:
     END = '\033[0m'
 
 
-def parse_args(args):
-    """Parse arguments"""
-    parser = argparse.ArgumentParser(usage='%(prog)s', description="Run SmartVA and import to DHIS2")
-
-    group = parser.add_mutually_exclusive_group()
-
-    group.add_argument('--briefcase',
-                       dest='briefcase_file',
-                       action='store',
-                       required=False,
-                       help="Skip download of briefcase file, provide local file path instead"
-                       )
-
-    group.add_argument('--all',
-                       dest='all',
-                       action='store_true',
-                       default=False,
-                       required=False,
-                       help="Pull all briefcases instead of relative time window"
-                       )
-
-    arguments = parser.parse_args(args)
-    if arguments.briefcase_file and not os.path.exists(arguments.briefcase_file):
-        raise FileNotFoundError("Briefcase file does not exist: {}".format(arguments.briefcase_file))
-    return arguments
-
-
 def log_subprocess_output(process):
-    """Log output from subprocess (e.g. smartva, Briefcase)"""
-    [
-        logger.info(str(line).replace('\n', ''))
-        for line in process.stdout
-        if not any(stop in line for stop in {'ETA: ', 'Time: '})  # don't log progress bars (smartva messages)
-    ]
+    """
+    Log output from subprocess (e.g. smartva, Briefcase)
+    - Raises Exception if no new data was detected
+    - Does not log progress bars of smartva
+    """
+    for line in process.stdout:
+        if re.compile('^Source file \".*\" does not contain data').match(line):
+            raise NoODKDataException
+        if not any(stop in line for stop in {'ETA: ', 'Time: '}):
+            logger.info(str(line).replace('\n', ''))
 
 
 def read_csv(path):
@@ -88,12 +65,12 @@ def sha256_checksum(filename, block_size=65536):
 
 def is_non_zero_file(fpath):
     """Return true if file is existing AND file has content, false otherwise"""
-    logger.debug(fpath)
-    if fpath:
-        exists = os.path.exists(fpath)
-        logger.debug("File exists: {}".format(fpath))
-        if exists:
-            return os.stat(fpath).st_size > 0
+    if fpath and os.path.exists(fpath):
+        has_data = os.stat(fpath).st_size > 0
+        if not has_data:
+            logger.debug("File exists but does not contain data: {}".format(fpath))
+        else:
+            return True
     return False
 
 

smartvadhis2/core/smartva.py

@@ -6,6 +6,7 @@
 
 from .config import SmartVAConfig, ODKConfig
 from .helpers import log_subprocess_output, is_non_zero_file
+from .exceptions import NoODKDataException
 
 """
 Module for running SmartVA / lib/smartva binary
@@ -21,7 +22,6 @@ def run(self, input_file, manual=False):
         """Entry method to run smartva"""
         if is_non_zero_file(input_file):
             input_path = input_file if manual else os.path.join(self.briefcase_dir, input_file)
-            logger.debug(input_path)
 
             logger.info("Running SmartVA ...")
             self._execute([SmartVAConfig.smartva_executable, '--version'])
@@ -41,6 +41,8 @@ def _execute(arguments):
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.STDOUT) as print_info:
                 log_subprocess_output(print_info)
+        except NoODKDataException:
+            raise
         except subprocess.CalledProcessError as e:
             logger.exception(e)
 

smartvadhis2/core/verbalautopsy.py

@@ -190,7 +190,6 @@ def age_category(self, value):
 
     @property
     def cause_of_death(self):
-        print(self._age_category, self._icd10)
         return cause_of_death_option_code(self._age_category, self._icd10)
 
     @property
@@ -246,7 +245,8 @@ def interview_date(self, interview_date):
                     d = datetime.strptime(interview_date, DATE_FMT_2)
                     self._interview_date = d.strftime(DATE_FMT_1)
                 except ValueError:
-                    raise InterviewDateParseError()
+                    #raise InterviewDateParseError()
+                    pass
         else:
             raise InterviewDateMissingWarning()