Skip to content

Commit

Permalink
REFACTOR-#6889: Define __all__ in modin.config.__init__.py (#6886)
Browse files Browse the repository at this point in the history
Signed-off-by: Anatoly Myachev <anatoly.myachev@intel.com>
  • Loading branch information
anmyachev committed Jan 30, 2024
1 parent 46dc0a5 commit e726435
Show file tree
Hide file tree
Showing 9 changed files with 128 additions and 28 deletions.
94 changes: 92 additions & 2 deletions modin/config/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,95 @@

"""Module houses config entities which can be used for Modin behavior tuning."""

from .envvars import * # noqa: F403, F401
from .pubsub import Parameter # noqa: F401
from modin.config.envvars import (
AsvDataSizeConfig,
AsvImplementation,
AsyncReadMode,
BenchmarkMode,
CIAWSAccessKeyID,
CIAWSSecretAccessKey,
CpuCount,
DoUseCalcite,
Engine,
EnvironmentVariable,
ExperimentalGroupbyImpl,
ExperimentalNumPyAPI,
GithubCI,
GpuCount,
HdkFragmentSize,
HdkLaunchParameters,
IsDebug,
IsExperimental,
IsRayCluster,
LogFileSize,
LogMemoryInterval,
LogMode,
Memory,
MinPartitionSize,
ModinNumpy,
NPartitions,
PersistentPickle,
ProgressBar,
RangePartitioningGroupby,
RayRedisAddress,
RayRedisPassword,
ReadSqlEngine,
StorageFormat,
TestDatasetSize,
TestRayClient,
TestReadFromPostgres,
TestReadFromSqlServer,
TrackFileLeaks,
)
from modin.config.pubsub import Parameter, ValueSource

__all__ = [
"EnvironmentVariable",
"Parameter",
"ValueSource",
# General settings
"IsDebug",
"Engine",
"StorageFormat",
"CpuCount",
"GpuCount",
"Memory",
# Ray specific
"IsRayCluster",
"RayRedisAddress",
"RayRedisPassword",
"TestRayClient",
# Partitioning
"NPartitions",
"MinPartitionSize",
# HDK specific
"HdkFragmentSize",
"DoUseCalcite",
"HdkLaunchParameters",
# ASV specific
"TestDatasetSize",
"AsvImplementation",
"AsvDataSizeConfig",
# Specific features
"ProgressBar",
"BenchmarkMode",
"PersistentPickle",
"ModinNumpy",
"ExperimentalNumPyAPI",
"RangePartitioningGroupby",
"ExperimentalGroupbyImpl",
"AsyncReadMode",
"ReadSqlEngine",
"IsExperimental",
# For tests
"TrackFileLeaks",
"TestReadFromSqlServer",
"TestReadFromPostgres",
"GithubCI",
"CIAWSSecretAccessKey",
"CIAWSAccessKeyID",
# Logging
"LogMode",
"LogMemoryInterval",
"LogFileSize",
]
23 changes: 15 additions & 8 deletions modin/config/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,18 @@

import pandas

from . import * # noqa: F403, F401
from .pubsub import Parameter
import modin.config as cfg


def print_config_help() -> None:
"""Print configs help messages."""
for objname in sorted(globals()):
obj = globals()[objname]
if isinstance(obj, type) and issubclass(obj, Parameter) and not obj.is_abstract:
for objname in sorted(cfg.__all__):
obj = getattr(cfg, objname)
if (
isinstance(obj, type)
and issubclass(obj, cfg.Parameter)
and not obj.is_abstract
):
print(f"{obj.get_help()}\n\tCurrent value: {obj.get()}") # noqa: T201


Expand All @@ -51,9 +54,13 @@ def export_config_help(filename: str) -> None:
CpuCount="multiprocessing.cpu_count()",
NPartitions="equals to MODIN_CPUS env",
)
for objname in sorted(globals()):
obj = globals()[objname]
if isinstance(obj, type) and issubclass(obj, Parameter) and not obj.is_abstract:
for objname in sorted(cfg.__all__):
obj = getattr(cfg, objname)
if (
isinstance(obj, type)
and issubclass(obj, cfg.Parameter)
and not obj.is_abstract
):
data = {
"Config Name": obj.__name__,
"Env. Variable Name": getattr(
Expand Down
2 changes: 1 addition & 1 deletion modin/config/envvars.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from packaging import version
from pandas.util._decorators import doc # type: ignore[attr-defined]

from .pubsub import (
from modin.config.pubsub import (
_TYPE_PARAMS,
_UNSET,
DeprecationDescriptor,
Expand Down
2 changes: 1 addition & 1 deletion modin/config/pubsub.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
)

if TYPE_CHECKING:
from .envvars import EnvironmentVariable
from modin.config.envvars import EnvironmentVariable


class DeprecationDescriptor:
Expand Down
13 changes: 4 additions & 9 deletions modin/config/test/test_envvars.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,11 @@
from packaging import version

import modin.config as cfg
from modin.config.envvars import (
_UNSET,
EnvironmentVariable,
ExactStr,
Parameter,
_check_vars,
)
from modin.config.envvars import _check_vars
from modin.config.pubsub import _UNSET, ExactStr


def reset_vars(*vars: tuple[Parameter]):
def reset_vars(*vars: tuple[cfg.Parameter]):
"""
Reset value for the passed parameters.
Expand All @@ -51,7 +46,7 @@ def make_unknown_env():

@pytest.fixture(params=[str, ExactStr])
def make_custom_envvar(request):
class CustomVar(EnvironmentVariable, type=request.param):
class CustomVar(cfg.EnvironmentVariable, type=request.param):
"""custom var"""

default = 10
Expand Down
2 changes: 1 addition & 1 deletion modin/config/test/test_parameter.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

import pytest

from modin.config.pubsub import Parameter
from modin.config import Parameter


def make_prefilled(vartype, varinit):
Expand Down
10 changes: 7 additions & 3 deletions modin/pandas/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,13 @@


def _update_engine(publisher: Parameter):
from modin.config import CpuCount, Engine, StorageFormat
from modin.config.envvars import IsExperimental
from modin.config.pubsub import ValueSource
from modin.config import (
CpuCount,
Engine,
IsExperimental,
StorageFormat,
ValueSource,
)

# Set this so that Pandas doesn't try to multithread by itself
os.environ["OMP_NUM_THREADS"] = "1"
Expand Down
8 changes: 6 additions & 2 deletions modin/pandas/test/test_groupby.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,12 @@
import pytest

import modin.pandas as pd
from modin.config import NPartitions, StorageFormat
from modin.config.envvars import IsRayCluster, RangePartitioningGroupby
from modin.config import (
IsRayCluster,
NPartitions,
RangePartitioningGroupby,
StorageFormat,
)
from modin.core.dataframe.algebra.default2pandas.groupby import GroupBy
from modin.core.dataframe.pandas.partitioning.axis_partition import (
PandasDataframeAxisPartition,
Expand Down
2 changes: 1 addition & 1 deletion modin/pandas/test/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,13 @@
AsyncReadMode,
Engine,
IsExperimental,
MinPartitionSize,
ReadSqlEngine,
StorageFormat,
TestDatasetSize,
TestReadFromPostgres,
TestReadFromSqlServer,
)
from modin.config.envvars import MinPartitionSize
from modin.db_conn import ModinDatabaseConnection, UnsupportedDatabaseException
from modin.pandas.io import from_arrow, to_pandas
from modin.test.test_utils import warns_that_defaulting_to_pandas
Expand Down

0 comments on commit e726435

Please sign in to comment.