PLAT-860: Use loky that is vendored with joblib.

This is to ensure compatibility.

GitOrigin-RevId: 77fed364af774f920136b74afe277cec13ec4bba

examples/synthetic_records.ipynb

@@ -23,7 +23,7 @@
    "source": [
     "# Google Colab support\n",
     "# Note: Click \"Runtime->Change Runtime Type\" set Hardware Accelerator to \"GPU\"\n",
-    "# Note: Use \"pip install -U gretel-synthetics tensorflow==2.11\" to install Gretel Synthetics and Tensorflow\n",
+    "# Note: Use pip install gretel-synthetics[tf] to install tensorflow if necessary\n",
     "# \n",
     "#!pip install gretel-synthetics --upgrade"
    ]
@@ -105,7 +105,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -119,7 +119,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.10"
+   "version": "3.7.5"
   }
  },
  "nbformat": 4,

requirements.txt

@@ -1,5 +1,5 @@
 category-encoders==2.2.2
-loky==2.9.0
+joblib==1.2.0
 numpy>=1.18.0,<1.24
 packaging==21.3
 pandas>=1.1.0,<2

src/gretel_synthetics/generate_parallel.py

@@ -10,7 +10,7 @@
 from concurrent import futures
 from typing import List, Optional, Set, Tuple, TYPE_CHECKING, Union
 
-import loky
+import joblib.externals.loky as loky
 
 if TYPE_CHECKING:
     from gretel_synthetics.generate import BaseGenerator, GenText, Settings

tests/test_generate_parallel.py

@@ -24,7 +24,7 @@
         (8, 100, 5, -0.5, 4),
     ],
 )
-@patch("loky.cpu_count")
+@patch("joblib.externals.loky.cpu_count")
 def test_split_work(
     cpu_count, num_cpus, total_lines, chunk_size, parallelism, expected_workers
 ):
@@ -66,7 +66,7 @@ def mock_submit(unused_fn, chunk_size: int, hard_limit: Optional[int] = None):
         for failure_rate in (0.0, 0.25, 0.5)
     ],
 )
-@patch("loky.ProcessPoolExecutor")
+@patch("joblib.externals.loky.ProcessPoolExecutor")
 def test_generate_parallel(pool_mock, num_lines, num_workers, chunk_size, failure_rate):
     pool_instance = pool_mock.return_value
     pool_instance.submit.side_effect = _mock_submitter(failure_rate)
@@ -82,7 +82,7 @@ def test_generate_parallel(pool_mock, num_lines, num_workers, chunk_size, failur
     assert invalid_lines <= ceil(failure_rate * num_lines)
 
 
-@patch("loky.ProcessPoolExecutor")
+@patch("joblib.externals.loky.ProcessPoolExecutor")
 def test_generate_parallel_too_many_invalid(pool_mock):
     pool_instance = pool_mock.return_value
     pool_instance.submit.side_effect = _mock_submitter(0.5)  # 0.5 failure rate