Increase test coverage (#77)

Add test dependency on responses to test requests calls

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+omit =
+    tests/*
+
+[report]
+exclude_also = 
+    if __name__ == .__main__.:

adtl/__init__.py

@@ -48,8 +48,6 @@ def get_value(row: StrDict, rule: Rule, ctx: Context = None) -> Any:
             return float(value)
         except ValueError:
             return value
-    except TypeError:
-        return value
 
 
 def get_value_unhashed(row: StrDict, rule: Rule, ctx: Context = None) -> Any:
@@ -321,6 +319,7 @@ def replace_val(
         for k, v in item.items():
             if not isinstance(k, str):
                 block[k] = v
+                continue
             rk = k.format(**replace)
             if isinstance(v, dict):
                 block[rk] = replace_val(v, replace)
@@ -497,16 +496,21 @@ def __init__(
         for table in (t for t in self.tables if self.tables[t]["kind"] == "oneToMany"):
             self.spec[table] = expand_for(self.spec[table])
         for table in self.tables:
+            if self.tables[table].get("groupBy"):
+                self.data[table] = defaultdict(dict)
+            else:
+                self.data[table] = []
             if schema := self.tables[table].get("schema"):
                 optional_fields = self.tables[table].get("optional-fields")
                 if schema.startswith("http"):
                     try:
-                        if (res := requests.get(schema)).status_code != 200:
+                        res = requests.get(schema)
+                        if res.status_code != 200:
                             logging.warning(
                                 f"Could not fetch schema for table {table!r}, will not validate"
                             )
                             continue
-                    except ConnectionError:
+                    except ConnectionError:  # pragma: no cover
                         logging.warning(
                             f"Could not fetch schema for table {table!r}, will not validate"
                         )
@@ -522,11 +526,6 @@ def __init__(
                 self.date_fields.extend(get_date_fields(self.schemas[table]))
                 self.validators[table] = fastjsonschema.compile(self.schemas[table])
 
-            if self.tables[table].get("groupBy"):
-                self.data[table] = defaultdict(dict)
-            else:
-                self.data[table] = []
-
         self._set_field_names()
 
     @lru_cache
@@ -655,8 +654,6 @@ def update_table(self, table: str, row: StrDict):
         group_field = self.tables[table].get("groupBy")
         kind = self.tables[table].get("kind")
         if group_field:
-            if table not in self.data:
-                self.data[table] = defaultdict(dict)
             group_key = get_value(row, self.spec[table][group_field])
             for attr in self.spec[table]:
                 value = get_value(row, self.spec[table][attr], self.ctx(attr))
@@ -679,9 +676,6 @@ def update_table(self, table: str, row: StrDict):
         elif kind == "constant":  # only one row
             self.data[table] = [self.spec[table]]
         else:
-            # no grouping, one-to-one mapping
-            if table not in self.data:
-                self.data[table] = []
             self.data[table].append(
                 remove_null_keys(
                     {
@@ -711,7 +705,7 @@ def parse_rows(self, rows: Iterable[StrDict], skip_validation=False):
             for table in self.tables:
                 try:
                     self.update_table(table, row)
-                except ValueError:
+                except ValueError:  # pragma: no cover
                     print(
                         "\n".join(
                             [
@@ -804,7 +798,7 @@ def save(self, output: Optional[str] = None):
             self.write_csv(table, f"{output}-{table}.csv")
 
 
-def main():
+def main(argv=None):
     cmd = argparse.ArgumentParser(
         prog="adtl",
         description="Transforms data into CSV given a specification (+validation)",
@@ -832,7 +826,7 @@ def main():
         action="append",
         help="Include external definition (TOML or JSON)",
     )
-    args = cmd.parse_args()
+    args = cmd.parse_args(argv)
     include_defs = args.include_def or []
     spec = Parser(args.spec, include_defs=include_defs, quiet=args.quiet)
 

adtl/transformations.py

@@ -5,7 +5,7 @@
 
 try:
     import zoneinfo
-except ImportError:
+except ImportError:  # pragma: no cover
     from backports import zoneinfo  # noqa
 
 import pint

pyproject.toml

@@ -28,6 +28,7 @@ test = [
   "pytest",
   "pytest-cov",
   "syrupy==4.*",
+  "responses",
   "pytest-unordered"
 ]
 

tests/__snapshots__/test_parser.ambr

@@ -7,6 +7,30 @@
 
   '''
 # ---
+# name: test_main
+  '''
+  adtl_valid,adtl_error,epoch,id,some_date,text
+  True,,1999-01-11,1,1999-01-24,Lorem ipsum
+  True,,2022-12-19,2,2023-01-27,example
+
+  '''
+# ---
+# name: test_main_web_schema
+  '''
+  adtl_valid,adtl_error,epoch,id,some_date,text
+  True,,1999-01-11,1,1999-01-24,Lorem ipsum
+  True,,2022-12-19,2,2023-01-27,example
+
+  '''
+# ---
+# name: test_main_web_schema_missing
+  '''
+  epoch,id,some_date,text
+  11/01/1999,1,24/01/1999,Lorem ipsum
+  19/12/2022,2,27/01/2023,example
+
+  '''
+# ---
 # name: test_multi_id_groupby
   '''
   admission_date,country_iso3,dataset_id,enrolment_date,sex_at_birth,subject_id
@@ -21,6 +45,21 @@
   2020-06-08,GBR,dataset-2020-03-23,2020-05-06,male,S007
   2020-06-08,GBR,dataset-2020-03-23,2022-01-11,female,S001
 
+  '''
+# ---
+# name: test_show_report
+  '''
+
+  |table       	|valid	|total	|percentage_valid|
+  |---------------|-------|-------|----------------|
+  |table         	|8	|10	|80.000000% |
+
+  ## table
+
+  * 1: data must be valid exactly by one definition (0 matches found)
+  * 1: data must contain ['epoch'] properties
+
+
   '''
 # ---
 # name: test_skip_field_pattern_absent

tests/parsers/epoch-web-schema.json

@@ -0,0 +1,27 @@
+{
+  "adtl": {
+    "name": "default-date-format",
+    "description": "Tests default date format",
+    "defaultDateFormat": "%d/%m/%Y",
+    "tables": {
+      "table": {
+        "kind": "oneToOne",
+        "schema": "http://example.com/schemas/epoch-data.schema.json"
+      }
+    }
+  },
+  "table": {
+    "id": {
+      "field": "Entry_ID"
+    },
+    "epoch": {
+      "field": "Epoch"
+    },
+    "some_date": {
+      "field": "SomeDate"
+    },
+    "text": {
+      "field": "Text"
+    }
+  }
+}

tests/parsers/epoch.yml

@@ -0,0 +1,11 @@
+adtl:
+  name: default-date-format
+  description: Tests default date format
+  defaultDateFormat: %d/%m/%Y
+  tables:
+    - table: { kind: oneToOne, schema: ../schemas/epoch-data.schema.json }
+table:
+  id: { field: Entry_ID }
+  epoch: { field: Epoch }
+  some_date: { field: SomeDate }
+  text: { field: Text }

tests/parsers/oneToMany-commonMappings.json

@@ -0,0 +1,46 @@
+{
+    "adtl": {
+        "name": "oneToManyWithCommonMappings",
+        "description": "One to Many example, with common mappings",
+        "tables": {
+            "observation": {
+                "kind": "oneToMany",
+                "common": {
+                    "dataset_id": "ONE_TO_MANY"
+                }
+            }
+        }
+    },
+    "observation": [
+        {
+            "date": {
+                "field": "dt"
+            },
+            "name": "headache",
+            "is_present": true,
+            "if": {
+                "headache_cmyn": 1
+            }
+        },
+        {
+            "date": {
+                "field": "dt"
+            },
+            "name": "cough",
+            "is_present": true,
+            "if": {
+                "cough_cmyn": 1
+            }
+        },
+        {
+            "date": {
+                "field": "dt"
+            },
+            "name": "dyspnea",
+            "is_present": true,
+            "if": {
+                "dyspnea_cmyn": 1
+            }
+        }
+    ]
+}