Py2 compatibility

Many small changes to restore Py2 compatibility.

src/python/pimlico/__init__.py

@@ -32,9 +32,6 @@
             pass
     reload(site)
 
-from pimlico.core.dependencies.base import check_and_install
-from pimlico.core.dependencies.core import CORE_PIMLICO_DEPENDENCIES, coloredlogs_dependency
-
 PIMLICO_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", ".."))
 
 # Fetch current version number from PIMLICO_ROOT/admin/release.txt
@@ -62,6 +59,8 @@
 
 
 def install_core_dependencies():
+    from pimlico.core.dependencies.base import check_and_install
+    from pimlico.core.dependencies.core import CORE_PIMLICO_DEPENDENCIES, coloredlogs_dependency
     # Always check that core dependencies are satisfied before running anything
     # Core dependencies are not allowed to depend on the local config, as we can't get to it at this point
     # We just pass in an empty dictionary

src/python/pimlico/datatypes/base.py

@@ -19,7 +19,7 @@
 from builtins import next
 from builtins import zip
 from builtins import object
-from future.utils import with_metaclass
+from future.utils import with_metaclass, PY3
 
 import json
 import os
@@ -101,13 +101,17 @@ def _get_reader_cls(cls):
                 if "__weakref__" in my_dict:
                     del my_dict["__weakref__"]
                 # Set the reader's __qualname__ so it's properly treated as a nested class of the datatype
-                my_dict["__qualname__"] = "{}.Reader".format(cls.__qualname__)
+                if PY3:
+                    my_dict["__qualname__"] = "{}.Reader".format(cls.__qualname__)
                 my_dict["__module__"] = cls.__module__
 
                 # No new documentation is provided, then we don't want to inherit the
                 # superclass' docstring, but instead let the reader follow the link to see that
                 if my_dict["__doc__"] is None:
-                    my_dict["__doc__"] = "Reader class for {}".format(cls.__qualname__)
+                    if PY3:
+                        my_dict["__doc__"] = "Reader class for {}".format(cls.__qualname__)
+                    else:
+                        my_dict["__doc__"] = "Reader class for {}".format(cls.__name__)
 
                 reader_cls = PimlicoDatatypeReaderMeta("Reader", (parent_reader,), my_dict)
             setattr(cls, _cache_name, reader_cls)
@@ -166,13 +170,17 @@ def _get_some_writer_cls(cls):
                     if "__weakref__" in new_cls_dict:
                         del new_cls_dict["__weakref__"]
                     # Set the writer's __qualname__ so it's properly treated as a nested class of the datatype
-                    new_cls_dict["__qualname__"] = "{}.Writer".format(cls.__qualname__)
+                    if PY3:
+                        new_cls_dict["__qualname__"] = "{}.Writer".format(cls.__qualname__)
                     new_cls_dict["__module__"] = cls.__module__
 
                     # No new documentation is provided, then we don't want to inherit the
                     # superclass' docstring, but instead let the reader follow the link to see that
                     if new_cls_dict["__doc__"] is None:
-                        new_cls_dict["__doc__"] = "Writer class for {}".format(cls.__qualname__)
+                        if PY3:
+                            new_cls_dict["__doc__"] = "Writer class for {}".format(cls.__qualname__)
+                        else:
+                            new_cls_dict["__doc__"] = "Writer class for {}".format(cls.__name__)
 
                 # Perform subclassing so that a new Writer is created that is a subclass of the parent's writer
                 writer_cls = type("Writer", (parent_writer,), new_cls_dict)
@@ -233,14 +241,18 @@ def _get_setup_cls(cls):
             if "__weakref__" in my_dict:
                 del my_dict["__weakref__"]
             # Set the reader setup's __qualname__ so it's properly treated as a nested class of the datatype's reader
-            my_dict["__qualname__"] = "{}.Setup".format(cls.__qualname__)
+            if PY3:
+                my_dict["__qualname__"] = "{}.Setup".format(cls.__qualname__)
             my_dict["__module__"] = cls.__module__
 
             if my_setup is parent_setup or my_dict["__doc__"] is None:
                 # If setup was not overridden: don't use the base class' doc
                 # If no new documentation is provided, then we don't want to inherit the
                 #  superclass' docstring, but instead let the reader follow the link to see that
-                my_dict["__doc__"] = "Setup class for {}".format(cls.__qualname__)
+                if PY3:
+                    my_dict["__doc__"] = "Setup class for {}".format(cls.__qualname__)
+                else:
+                    my_dict["__doc__"] = "Setup class for {}".format(cls.__name__)
 
             setup_cls = type("Setup", (parent_setup,), my_dict)
             setattr(cls, _cache_name, setup_cls)

src/python/pimlico/datatypes/corpora/data_points.py

@@ -11,7 +11,7 @@
 from traceback import format_exc
 
 from builtins import object
-from future.utils import with_metaclass
+from future.utils import with_metaclass, PY3
 
 __all__ = ["DataPointType", "RawDocumentType", "TextDocumentType", "RawTextDocumentType", "DataPointError",
            "InvalidDocument"]
@@ -57,7 +57,8 @@ def _get_document_cls(cls):
                 if "__weakref__" in new_dict:
                     del new_dict["__weakref__"]
                 # Set the reader setup's __qualname__ so it's properly treated as a nested class of the datatype's reader
-                new_dict["__qualname__"] = "{}.Document".format(cls.__qualname__)
+                if PY3:
+                    new_dict["__qualname__"] = "{}.Document".format(cls.__qualname__)
                 new_dict["__module__"] = cls.__module__
 
                 # If no new documentation is provided, then we don't want to inherit the

src/python/pimlico/datatypes/embeddings.py

@@ -19,12 +19,10 @@
 import io
 import os
 
-from sklearn.metrics.pairwise import cosine_distances
-
 from backports import csv
 
 from pimlico.core.dependencies.python import numpy_dependency, PythonPackageOnPip
-from pimlico.datatypes import PimlicoDatatype, NamedFileCollection
+from pimlico.datatypes import PimlicoDatatype
 from pimlico.datatypes.files import NamedFileCollection
 from pimlico.utils.core import cached_property
 
@@ -240,6 +238,7 @@ def run_browser(self, reader, opts):
         """
         import numpy as np
         from sklearn.metrics import euclidean_distances
+        from sklearn.metrics.pairwise import cosine_distances
 
         print("Reading embeddings...")
         num_vectors = len(reader)

src/python/pimlico/datatypes/gensim.py

@@ -49,7 +49,7 @@ def write_model(self, model):
 
 class GensimLdaSeqModel(PimlicoDatatype):
     """
-    A trained LDA-seq model – i.e. Dynamic Topic Model (DTM).
+    A trained LDA-seq model - i.e. Dynamic Topic Model (DTM).
 
     As well as the Gensim model, it also stores the list of slice labels, so that
     we can easily look up the appropriate time slice for a document paired with

src/python/pimlico/test/pipeline.py

@@ -15,17 +15,17 @@
 several) of how to use them -- how pipeline config should look and what sort of input data to use.
 
 """
+if __name__ == "__main__":
+    from pimlico import install_core_dependencies
+    install_core_dependencies()
+
 from builtins import object
 import traceback
 
 from past.utils import PY2
 
 from pimlico.utils.core import remove_duplicates
 
-if __name__ == "__main__":
-    from pimlico import install_core_dependencies
-    install_core_dependencies()
-
 import argparse
 import os
 import shutil

src/python/pimlico/test/suite.py

@@ -1,13 +1,14 @@
+
+if __name__ == "__main__":
+    from pimlico import install_core_dependencies
+    install_core_dependencies()
+
 import argparse
 import sys
 
 from pimlico.test.pipeline import run_test_suite
 from pimlico.utils.logging import get_console_logger
 
-if __name__ == "__main__":
-    from pimlico import install_core_dependencies
-    install_core_dependencies()
-
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Pipeline test suite runner")

src/python/pimlico/utils/_core_py3.py

@@ -0,0 +1,12 @@
+
+def raise_from(exc, cause):
+    """
+    Like future's raise_from function. However, on Py3, just calls raise X from Y.
+    On Py2, defers to future's replacement.
+
+    This means that we get the full functionality of raise from on PY3, which
+    is our main priority. If you run on PY2, you get less debugging information
+    in some cases, but that's no reason to ruin the debugging information on PY3 too!
+
+    """
+    raise exc from cause

src/python/pimlico/utils/core.py

@@ -6,7 +6,7 @@
 from builtins import object
 from contextlib import contextmanager
 
-from future.utils import raise_from as future_raise_from, PY3
+from future.utils import PY3
 
 import sys
 import ast
@@ -224,17 +224,18 @@ def __get__(self, obj, cls):
         return value
 
 
-def raise_from(exc, cause):
-    """
-    Like future's raise_from function. However, on Py3, just calls raise X from Y.
-    On Py2, defers to future's replacement.
+"""
+Like future's raise_from function. However, on Py3, just calls raise X from Y.
+On Py2, defers to future's replacement.
 
-    This means that we get the full functionality of raise from on PY3, which
-    is our main priority. If you run on PY2, you get less debugging information
-    in some cases, but that's no reason to ruin the debugging information on PY3 too!
+This means that we get the full functionality of raise from on PY3, which
+is our main priority. If you run on PY2, you get less debugging information
+in some cases, but that's no reason to ruin the debugging information on PY3 too!
 
-    """
-    if PY3:
-        raise exc from cause
-    else:
-        future_raise_from(exc, cause)
+"""
+if PY3:
+    # noinspection PyUnresolvedReferences
+    from ._core_py3 import raise_from
+else:
+    # noinspection PyUnresolvedReferences
+    from future.utils import raise_from