Skip to content

Commit

Permalink
fix ci
Browse files Browse the repository at this point in the history
  • Loading branch information
Retribution98 committed Feb 28, 2024
1 parent 5061311 commit 32db768
Show file tree
Hide file tree
Showing 6 changed files with 53 additions and 48 deletions.
36 changes: 36 additions & 0 deletions modin/core/execution/ray/generic/io/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,48 @@ class RayIO(BaseIO):

@classmethod
def from_ray(cls, ray_obj):

Check warning on line 23 in modin/core/execution/ray/generic/io/io.py

View check run for this annotation

Codecov / codecov/patch

modin/core/execution/ray/generic/io/io.py#L22-L23

Added lines #L22 - L23 were not covered by tests
"""
Create a Modin `query_compiler` from a Ray Dataset.
Parameters
----------
ray_obj : ray.data.Dataset
The Ray Dataset to convert from.
Returns
-------
BaseQueryCompiler
QueryCompiler containing data from the Ray Dataset.
Notes
-----
This function must be implemented in every subclass
otherwise NotImplementedError will be raised.
"""
raise NotImplementedError(
f"Modin dataset cann't be created from `ray.data.Dataset` using {cls}."
)

@classmethod
def to_ray(cls, modin_obj):

Check warning on line 47 in modin/core/execution/ray/generic/io/io.py

View check run for this annotation

Codecov / codecov/patch

modin/core/execution/ray/generic/io/io.py#L46-L47

Added lines #L46 - L47 were not covered by tests
"""
Convert a Modin DataFrame to a Ray Dataset.
Parameters
----------
modin_obj : modin.DataFrame
The Modin DataFrame to convert.
Returns
-------
ray.data.Dataset
Converted object with type depending on input.
Notes
-----
This function must be implemented in every subclass
otherwise NotImplementedError will be raised.
"""
raise NotImplementedError(
f"`ray.data.Dataset` cann't be created from Modin DataFrame using {cls}."
)
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@
PandasSQLParser,
)
from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler
from modin.distributed.dataframe.pandas.partitions import (

Check warning on line 42 in modin/core/execution/ray/implementations/pandas_on_ray/io/io.py

View check run for this annotation

Codecov / codecov/patch

modin/core/execution/ray/implementations/pandas_on_ray/io/io.py#L42

Added line #L42 was not covered by tests
from_partitions,
unwrap_partitions,
)
from modin.experimental.core.io import (
ExperimentalCSVGlobDispatcher,
ExperimentalCustomTextDispatcher,
Expand All @@ -53,10 +57,6 @@
ExperimentalPandasPickleParser,
ExperimentalPandasXmlParser,
)
from modin.distributed.dataframe.pandas.partitions import (
from_partitions,
unwrap_partitions,
)

from ..dataframe import PandasOnRayDataframe
from ..partitioning import PandasOnRayDataframePartition
Expand Down
2 changes: 1 addition & 1 deletion modin/core/io/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -720,7 +720,7 @@ def to_parquet(cls, obj, path, **kwargs): # noqa: PR01
return obj.to_parquet(path, **kwargs)

@classmethod
def to_ray(cls, modin_df):
def to_ray(cls, modin_obj):
"""
Convert a Modin DataFrame to a Ray Dataset.
Expand Down
28 changes: 1 addition & 27 deletions modin/pandas/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
from modin.error_message import ErrorMessage
from modin.logging import disable_logging
from modin.pandas import Categorical
from modin.pandas.io import from_non_pandas, from_pandas, to_pandas, from_ray, to_ray
from modin.pandas.io import from_non_pandas, from_pandas, to_pandas, to_ray

Check notice

Code scanning / CodeQL

Cyclic import Note

Import of module
modin.pandas.io
begins an import cycle.
from modin.utils import (
MODIN_UNNAMED_SERIES_LABEL,
_inherit_docstrings,
Expand Down Expand Up @@ -921,27 +921,6 @@ def from_dict(
)
)

@classmethod
def from_ray(cls, ray_obj):
"""
Convert a Ray Dataset into Modin DataFrame.
Parameters
----------
ray_obj : ray.data.Dataset
The Ray Dataset to convert from.
Returns
-------
DataFrame
A new Modin DataFrame object.
Notes
-----
Ray Dataset may only be converted to Modin Dataframe if Modin uses a Ray engine.
"""
return from_ray(ray_obj)

@classmethod
def from_records(
cls,
Expand Down Expand Up @@ -2254,11 +2233,6 @@ def to_ray(self):
"""
Convert a Modin DataFrame to a Ray Dataset.
Parameters
----------
modin_obj : modin.DataFrame
The Modin DataFrame to convert.
Returns
-------
ray.data.Dataset
Expand Down
4 changes: 2 additions & 2 deletions modin/pandas/test/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ def test_dataframe_api_equality():

ignore_in_pandas = ["timetuple"]
# modin - namespace for using experimental functionality
ignore_in_modin = ["modin", "to_pandas"]
ignore_in_modin = ["modin", "to_pandas", "to_ray"]
missing_from_modin = set(pandas_dir) - set(modin_dir)
assert not len(
missing_from_modin - set(ignore_in_pandas)
Expand All @@ -164,7 +164,7 @@ def test_dataframe_api_equality():
), "Differences found in API: {}".format(set(modin_dir) - set(pandas_dir))

# These have to be checked manually
allowed_different = ["to_hdf", "hist", "modin", "to_pandas"]
allowed_different = ["to_hdf", "hist", "modin", "to_pandas", "to_ray"]

assert_parameters_eq((pandas.DataFrame, pd.DataFrame), modin_dir, allowed_different)

Expand Down
23 changes: 9 additions & 14 deletions modin/pandas/test/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
TestReadFromSqlServer,
)
from modin.db_conn import ModinDatabaseConnection, UnsupportedDatabaseException
from modin.pandas.io import from_arrow, to_pandas, from_ray
from modin.pandas.io import from_arrow, from_ray, to_pandas
from modin.test.test_utils import warns_that_defaulting_to_pandas

from .utils import (
Expand Down Expand Up @@ -3246,26 +3246,21 @@ def test_to_ray():
RuntimeError,
match="Modin Dataframe may only be converted to a Ray Dataset if Modin uses a Ray engine.",
):
_= modin_df.to_ray()
_ = modin_df.to_ray()


@pytest.mark.skipif(
condition=Engine.get() != "Ray",
reason="Ray Dataset creation is only available for Ray engine",
)
@pytest.mark.filterwarnings(default_to_pandas_ignore_string)
def test_from_ray():
index = pandas.DatetimeIndex(
pandas.date_range("2000", freq="h", periods=len(TEST_DATA["col1"]))
)
modin_df, pandas_df = create_test_dfs(TEST_DATA, index=index)

from ray.data import from_pandas
ray_df = ray.data.from_pandas(pandas_df)

ray_df = from_pandas(pandas_df)

if Engine.get() == "Ray":
result_df = from_ray(ray_df)
df_equals(result_df, modin_df)
else:
with pytest.raises(
RuntimeError,
match="Modin Dataframe may only be converted to a Ray Dataset if Modin uses a Ray engine.",
):
_ = from_ray(ray_df)
result_df = from_ray(ray_df)
df_equals(result_df, modin_df)

0 comments on commit 32db768

Please sign in to comment.