Add a "nopool" argument to validate (#527)

* Added nopool option to validate * Added nopool tests
frictionlessdata · Nov 12, 2020 · 32c9c29 · 32c9c29
1 parent 3800260
commit 32c9c29
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 9 deletions.
diff --git a/frictionless/validate/inquiry.py b/frictionless/validate/inquiry.py
@@ -9,7 +9,7 @@
 
 
 @Report.from_validate
-def validate_inquiry(source):
+def validate_inquiry(source, *, nopool=False):
     """Validate inquiry
 
     API      | Usage
@@ -18,6 +18,7 @@ def validate_inquiry(source):
 
     Parameters:
         source (dict|str): an inquiry descriptor
+        nopool? (bool): disable multiprocessing
 
     Returns:
         Report: validation report
@@ -44,13 +45,14 @@ def validate_inquiry(source):
             continue
         tasks.append(task)
 
-    # Validate task
-    if len(tasks) == 1:
-        report = validate(**helpers.create_options(tasks[0]))
-        reports.append(report)
+    # Validate sequentially
+    if len(tasks) == 1 or nopool:
+        for task in tasks:
+            report = validate(**helpers.create_options(task))
+            reports.append(report)
 
-    # Validate tasks
-    if len(tasks) > 1:
+    # Validate in-parallel
+    else:
         with Pool() as pool:
             reports.extend(pool.map(partial(helpers.apply_function, validate), tasks))
 

diff --git a/frictionless/validate/package.py b/frictionless/validate/package.py
@@ -8,7 +8,13 @@
 
 @Report.from_validate
 def validate_package(
-    source, basepath=None, trusted=False, noinfer=False, nolookup=False, **options
+    source,
+    basepath=None,
+    trusted=False,
+    noinfer=False,
+    nolookup=False,
+    nopool=False,
+    **options
 ):
     """Validate package
 
@@ -22,6 +28,7 @@ def validate_package(
         trusted? (bool): don't raise an exception on unsafe paths
         noinfer? (bool): don't call `package.infer`
         nolookup? (bool): don't read lookup tables skipping integrity checks
+        nopool? (bool): disable multiprocessing
         **options (dict): options for every extracted table
 
     Returns:
@@ -62,7 +69,7 @@ def validate_package(
 
     # Validate inquiry
     inquiry = Inquiry(descriptor)
-    report = validate_inquiry(inquiry)
+    report = validate_inquiry(inquiry, nopool=nopool)
 
     # Return report
     return Report(time=timer.time, errors=report["errors"], tables=report["tables"])
diff --git a/tests/validate/test_inquiry.py b/tests/validate/test_inquiry.py
@@ -101,3 +101,20 @@ def test_validate_with_multiple_packages():
         [3, 3, None, "primary-key-error"],
         [4, 4, None, "blank-row"],
     ]
+
+
+def test_validate_with_multiple_packages_with_nopool():
+    report = validate(
+        {
+            "tasks": [
+                {"source": "data/package/datapackage.json"},
+                {"source": "data/invalid/datapackage.json"},
+            ]
+        },
+        nopool=True,
+    )
+    assert report.flatten(["tablePosition", "rowPosition", "fieldPosition", "code"]) == [
+        [3, 3, None, "blank-row"],
+        [3, 3, None, "primary-key-error"],
+        [4, 4, None, "blank-row"],
+    ]
diff --git a/tests/validate/test_package.py b/tests/validate/test_package.py
@@ -146,6 +146,15 @@ def test_validate_package_dialect_header_false():
     assert report.valid
 
 
+def test_validate_with_nopool():
+    report = validate("data/invalid/datapackage.json", nopool=True)
+    assert report.flatten(["tablePosition", "rowPosition", "fieldPosition", "code"]) == [
+        [1, 3, None, "blank-row"],
+        [1, 3, None, "primary-key-error"],
+        [2, 4, None, "blank-row"],
+    ]
+
+
 # Checksum
 
 DESCRIPTOR_SH = {