Switch multiprocessing backend to loky (#49)

examples/generate_as_module.py

@@ -1,15 +1,14 @@
 
 """
 Example module on how to run data generation from a standlone python invocation. Tensorflow
-requires that processes are launch with "spawn" mode, which requires the use of ``freeze_support()``
-that has to be called in the `__main__` scope of the module.
+requires that processes are launch with "spawn" mode, for which it is a good practice to ensure
+that any code is only executed after checking that we are in the main module
+(`if __name__ == '__main__'`).
 
 In the event that you choose to export a Notebook to a pure module, please note the changes below. These
 changes will have a ``NOTE:`` comment.
 """
 
-# NOTE: Required import for launching from standlone module
-from multiprocessing import freeze_support
 from pathlib import Path
 
 from gretel_synthetics.config import LocalConfig
@@ -38,7 +37,7 @@
 
 # Let's generate some text!
 #
-# The ``generate_text`` funtion is a generator that will return
+# The ``generate_text`` function is a generator that will return
 # a line of predicted text based on the ``gen_lines`` setting in your
 # config.
 #
@@ -72,7 +71,7 @@ def start():
         print(line)
 
 
-# NOTE: Invoke your generation this way
+# NOTE: It is preferred to always invoke your generation this way. Simply invoking start() from
+# the top-level of the main module *should* work, but YMMV.
 if __name__ == "__main__":
-    freeze_support()
     start()

requirements.txt

@@ -5,4 +5,4 @@ smart_open==2.0.0
 pandas>=1.0.0
 numpy>=1.18.0
 tqdm<5.0
-cloudpickle==1.5.0
+loky==2.8.0

setup.py

@@ -28,7 +28,7 @@
         'pandas>=1.0.0',
         'numpy>=1.18.0',
         'dataclasses==0.7;python_version<"3.7"',
-        'cloudpickle==1.5.0',
+        'loky==2.8.0',
     ],
     extras_require={
         'tf': ['tensorflow==2.1.0']

src/gretel_synthetics/batch.py

@@ -25,6 +25,7 @@
 
 from gretel_synthetics.config import LocalConfig
 from gretel_synthetics.generate import gen_text, generate_text
+from gretel_synthetics.generator import TooManyInvalidError
 from gretel_synthetics.train import train_rnn
 
 
@@ -433,7 +434,7 @@ def generate_batch_lines(
                 else:
                     t2.update(1)
                     batch.gen_data_invalid.append(line)
-        except RuntimeError:
+        except TooManyInvalidError:
             if raise_on_exceed_invalid:
                 raise
             else:

src/gretel_synthetics/generate.py

@@ -13,7 +13,7 @@
 
 from gretel_synthetics.generator import Generator, Settings
 from gretel_synthetics.generator import gen_text, PredString  # noqa # pylint: disable=unused-import
-from gretel_synthetics.generate_parallel import split_work, generate_parallel
+from gretel_synthetics.generate_parallel import get_num_workers, generate_parallel
 
 if TYPE_CHECKING:  # pragma: no cover
     from gretel_synthetics.config import LocalConfig
@@ -104,10 +104,10 @@ def my_validator(raw_line: str):
     else:
         _line_count = config.gen_lines
 
-    num_workers, chunks = split_work(parallelism, _line_count)
+    num_workers = get_num_workers(parallelism, _line_count, chunk_size=5)
 
     if num_workers == 1:  # Sequential operation
         gen = Generator(settings)
         yield from gen.generate_next(_line_count)
     else:
-        yield from generate_parallel(settings, num_workers, chunks, num_lines)
+        yield from generate_parallel(settings, _line_count, num_workers, chunk_size=5)