From 5ef3d85ca89d128fd2395bc08e853e245938d6eb Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 7 May 2018 15:33:08 -0500 Subject: [PATCH] type hints work again; organize imports Organize imports by stdlib, 3rd party, 1st party per PEP8. --- csv_load.py | 14 +++++++++----- eventlog.py | 3 ++- i2p_tasks.py | 50 +++++++++++++++++++++++++++----------------------- 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/csv_load.py b/csv_load.py index a753354..05220dc 100644 --- a/csv_load.py +++ b/csv_load.py @@ -1,10 +1,13 @@ from collections import defaultdict from csv import DictReader from datetime import datetime +from typing import Dict + +from sqlalchemy import func, MetaData, Table, Column # type: ignore +from sqlalchemy.types import String # type: ignore + from etl_tasks import DBAccessTask from param_val import StrParam, IntParam -from sqlalchemy import func, MetaData, Table, Column -from sqlalchemy.types import String import logging import sqlalchemy as sqla @@ -34,17 +37,18 @@ def run(self) -> None: self.setStatus() def load(self) -> None: - def sz(l, chunk=16): + def sz(l: int, chunk: int=16) -> int: return max(chunk, chunk * ((l + chunk - 1) // chunk)) db = self._dbtarget().engine schema = MetaData() l = list() - with open(self.csvname) as fin: + with open(self.csvname) as fin: # ISSUE: ambient dr = DictReader(fin) - mcl = defaultdict(int) + Dict # for tools that don't see type: comments. + mcl = defaultdict(int) # type: Dict[str, int] for row in dr: l.append(row) for col in dr.fieldnames: diff --git a/eventlog.py b/eventlog.py index 1de5799..30a6c12 100644 --- a/eventlog.py +++ b/eventlog.py @@ -74,7 +74,8 @@ def __init__(self, logger: logging.Logger, event: JSONObject, List # let flake8 know we're using it def __repr__(self) -> str: - return '%s(%s, %s)' % (self.__class__.__name__, self.name, self.event) + # typeshed doesn't know that LoggerAdapter has .name? + return '%s(%s, %s)' % (self.__class__.__name__, self.name, self.event) # type: ignore def process(self, msg: str, kwargs: KWArgs) -> Tuple[str, KWArgs]: extra = dict(kwargs.get('extra', {}), diff --git a/i2p_tasks.py b/i2p_tasks.py index 54a88e4..dae3c27 100644 --- a/i2p_tasks.py +++ b/i2p_tasks.py @@ -1,14 +1,17 @@ """i2p_tasks -- Luigi CDM task support. """ +from typing import List + +import luigi +from sqlalchemy.engine import RowProxy +from sqlalchemy.exc import DatabaseError + from csv_load import LoadCSV from etl_tasks import SqlScriptTask from param_val import IntParam from script_lib import Script -from sql_syntax import Environment -from sqlalchemy.engine import RowProxy -from sqlalchemy.exc import DatabaseError -from typing import List +from sql_syntax import Environment, Params class CDMScriptTask(SqlScriptTask): @@ -24,63 +27,63 @@ def variables(self) -> Environment: class condition(CDMScriptTask): script = Script.condition - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] class death(CDMScriptTask): script = Script.death - def requires(self): + def requires(self) -> List[luigi.Task]: return [demographic()] class death_cause(CDMScriptTask): script = Script.death_cause - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class demographic(CDMScriptTask): script = Script.demographic - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class diagnosis(CDMScriptTask): script = Script.diagnosis - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] class dispensing(CDMScriptTask): script = Script.dispensing - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] class encounter(CDMScriptTask): script = Script.encounter - def requires(self): + def requires(self) -> List[luigi.Task]: return [demographic()] class enrollment(CDMScriptTask): script = Script.enrollment - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class harvest(CDMScriptTask): script = Script.harvest - def requires(self): + def requires(self) -> List[luigi.Task]: return [condition(), death(), death_cause(), diagnosis(), dispensing(), enrollment(), lab_result_cm(), med_admin(), obs_clin(), obs_gen(), pcornet_trial(), prescribing(), pro_cm(), procedures(), provider(), vital()] @@ -89,14 +92,14 @@ def requires(self): class lab_result_cm(CDMScriptTask): script = Script.lab_result_cm - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] class med_admin(CDMScriptTask): script = Script.med_admin - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] @@ -126,7 +129,8 @@ def results(self) -> List[RowProxy]: chunk_num <= :chunk_max) order by chunk_num ''' - params = dict(chunk_max=self.patient_chunk_max, chunk_qty=self.patient_chunks) + Params + params = dict(chunk_max=self.patient_chunk_max, chunk_qty=self.patient_chunks) # type: Params try: return lc.execute(q, params=params).fetchall() @@ -137,42 +141,42 @@ def results(self) -> List[RowProxy]: class pcornet_init(CDMScriptTask): script = Script.pcornet_init - def requires(self): + def requires(self) -> List[luigi.Task]: return [loadLabNormal(), loadHarvestLocal()] class pcornet_loader(CDMScriptTask): script = Script.pcornet_loader - def requires(self): + def requires(self) -> List[luigi.Task]: return [harvest()] class pcornet_trial(CDMScriptTask): script = Script.pcornet_trial - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class prescribing(CDMScriptTask): script = Script.prescribing - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] class pro_cm(CDMScriptTask): script = Script.pro_cm - def requires(self): + def requires(self) -> List[luigi.Task]: return [pcornet_init()] class procedures(CDMScriptTask): script = Script.procedures - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()] @@ -183,7 +187,7 @@ class provider(CDMScriptTask): class vital(CDMScriptTask): script = Script.vital - def requires(self): + def requires(self) -> List[luigi.Task]: return [encounter()]