Refactor the QueryCompiler module to separate backends (#510)

* Refactor the QueryCompiler module to separate backends * Preparing to factor out Metadata management into a separate class * Create `modin.backends.<base, pandas, pyarrow> * Fix path to pyarrow coverage
modin-project · Mar 24, 2019 · 8e69509 · 8e69509
1 parent 33a8631
commit 8e69509
Show file tree

Hide file tree

Showing 15 changed files with 10 additions and 21 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -15,7 +15,7 @@ omit =
     modin/engines/dask/*
     # Skip Gandiva because it is experimental
     modin/experimental/engines/pyarrow_on_ray/*
-    modin/data_management/query_compiler/pyarrow_query_compiler.py
+    modin/backends/pyarrow/*
 
 [report]
 exclude_lines =

diff --git a/modin/backends/__init__.py b/modin/backends/__init__.py
diff --git a/modin/backends/base/__init__.py b/modin/backends/base/__init__.py
diff --git a/...ent/query_compiler/base_query_compiler.py → modin/backends/base/query_compiler.py b/...ent/query_compiler/base_query_compiler.py → modin/backends/base/query_compiler.py
diff --git a/modin/backends/pandas/__init__.py b/modin/backends/pandas/__init__.py
diff --git a/...t/query_compiler/pandas_query_compiler.py → modin/backends/pandas/query_compiler.py b/...t/query_compiler/pandas_query_compiler.py → modin/backends/pandas/query_compiler.py
@@ -18,7 +18,7 @@
 
 from modin.engines.base.frame.partition_manager import BaseFrameManager
 from modin.error_message import ErrorMessage
-from .base_query_compiler import BaseQueryCompiler
+from modin.backends.base.query_compiler import BaseQueryCompiler
 
 
 class PandasQueryCompiler(BaseQueryCompiler):

diff --git a/modin/backends/pyarrow/__init__.py b/modin/backends/pyarrow/__init__.py
diff --git a/.../query_compiler/pyarrow_query_compiler.py → modin/backends/pyarrow/query_compiler.py b/.../query_compiler/pyarrow_query_compiler.py → modin/backends/pyarrow/query_compiler.py
@@ -1,4 +1,4 @@
-from .pandas_query_compiler import PandasQueryCompiler
+from modin.backends.pandas.query_compiler import PandasQueryCompiler
 import pyarrow as pa
 import pandas
 from pandas.core.computation.expr import Expr

diff --git a/modin/data_management/factories.py b/modin/data_management/factories.py
@@ -8,7 +8,7 @@
 
 from modin import __execution_engine__ as execution_engine
 from modin import __partition_format__ as partition_format
-from .query_compiler import PandasQueryCompiler
+from modin.backends.pandas.query_compiler import PandasQueryCompiler
 
 
 class BaseFactory(object):
@@ -298,7 +298,7 @@ class ExperimentalPyarrowOnRayFactory(BaseFactory):  # pragma: no cover
         PyarrowOnRayFrameManager,
     )
     from modin.experimental.engines.pyarrow_on_ray.io import PyarrowOnRayIO
-    from modin.data_management.query_compiler import PyarrowQueryCompiler
+    from modin.backends.pyarrow.query_compiler import PyarrowQueryCompiler
 
     query_compiler_cls = PyarrowQueryCompiler
     block_partitions_cls = PyarrowOnRayFrameManager

diff --git a/modin/data_management/query_compiler/__init__.py b/modin/data_management/query_compiler/__init__.py
diff --git a/modin/engines/base/io.py b/modin/engines/base/io.py
@@ -4,7 +4,7 @@
 
 import pandas
 from modin.error_message import ErrorMessage
-from modin.data_management.query_compiler import BaseQueryCompiler
+from modin.backends.base.query_compiler import BaseQueryCompiler
 
 
 class BaseIO(object):

diff --git a/modin/engines/dask/pandas_on_dask_delayed/io.py b/modin/engines/dask/pandas_on_dask_delayed/io.py
@@ -3,7 +3,7 @@
 from __future__ import print_function
 
 from modin.engines.base.io import BaseIO
-from modin.data_management.query_compiler import PandasQueryCompiler
+from modin.backends.pandas.query_compiler import PandasQueryCompiler
 from modin.engines.dask.pandas_on_dask_delayed.frame.partition_manager import (
     DaskFrameManager,
 )

diff --git a/modin/engines/python/pandas_on_python/io.py b/modin/engines/python/pandas_on_python/io.py
@@ -3,7 +3,7 @@
 from __future__ import print_function
 
 from modin.engines.base.io import BaseIO
-from modin.data_management.query_compiler import PandasQueryCompiler
+from modin.backends.pandas.query_compiler import PandasQueryCompiler
 from modin.engines.python.pandas_on_python.frame.partition_manager import (
     PythonFrameManager,
 )

diff --git a/modin/engines/ray/pandas_on_ray/io.py b/modin/engines/ray/pandas_on_ray/io.py
@@ -7,7 +7,7 @@
 from io import BytesIO
 import ray
 from modin.data_management.utils import split_result_of_axis_func_pandas
-from modin.data_management.query_compiler import PandasQueryCompiler
+from modin.backends.pandas.query_compiler import PandasQueryCompiler
 from modin.engines.ray.generic.io import RayIO
 from modin.engines.ray.pandas_on_ray.frame.partition_manager import (
     PandasOnRayFrameManager,

diff --git a/modin/experimental/engines/pyarrow_on_ray/io.py b/modin/experimental/engines/pyarrow_on_ray/io.py
@@ -1,4 +1,4 @@
-from modin.data_management.query_compiler import PyarrowQueryCompiler
+from modin.backends.pyarrow.query_compiler import PyarrowQueryCompiler
 from modin.engines.ray.generic.io import RayIO
 from modin.experimental.engines.pyarrow_on_ray.frame.partition_manager import (
     PyarrowOnRayFrameManager,