ray-project · matthewdeng · Oct 11, 2023 · Oct 6, 2023 · Oct 8, 2023 · Oct 9, 2023
diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
@@ -254,5 +254,4 @@
       - ./ci/env/env_info.sh
       - python ./ci/env/check_minimal_install.py
       - bazel test --config=ci $(./ci/run/bazel_export_options)  --build_tests_only --test_tag_filters=minimal
-        --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=1
         python/ray/train/...
diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml
@@ -332,7 +332,6 @@
     - ./ci/env/env_info.sh
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=client --test_env=RAY_CLIENT_MODE=1 python/ray/util/dask/...
     - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
-      --test_env=RAY_AIR_NEW_PERSISTENCE_MODE=1
       --test_tag_filters=client
       python/ray/tune/...
 

@@ -57,20 +57,6 @@
     "import os"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {
-    "tags": [
-     "remove-cell"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "# TODO(@justinvyu): Remove it\n",
-    "os.environ[\"RAY_AIR_NEW_PERSISTENCE_MODE\"] = \"1\""
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -127,7 +113,6 @@
     "            \"torch>=1.12.0\",\n",
     "            \"deepspeed==0.9.2\",\n",
     "        ],\n",
-    "        \"env_vars\": {\"RAY_AIR_NEW_PERSISTENCE_MODE\": \"1\"} # TODO(@justinvyu): Remove it\n",
     "    },\n",
     ")"
    ]

@@ -81,21 +81,6 @@
     "```"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "tags": [
-     "remove-cell"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "# TODO(@justinvyu): Remove it\n",
-    "import os\n",
-    "os.environ[\"RAY_AIR_NEW_PERSISTENCE_MODE\"] = \"1\""
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -118,7 +103,6 @@
     "            \"transformers==4.30.2\",\n",
     "            \"pytorch_lightning==2.0.3\",\n",
     "        ],\n",
-    "        \"env_vars\": {\"RAY_AIR_NEW_PERSISTENCE_MODE\": \"1\"} # TODO(@justinvyu): Remove it\n",
     "    }\n",
     ")"
    ]

@@ -44,13 +44,6 @@
 _VALIDATE_STORAGE_MARKER_FILENAME = ".validate_storage_marker"
 
 
-def _use_storage_context() -> bool:
-    # Whether to enable the new simple persistence mode.
-    from ray.train.constants import RAY_AIR_NEW_PERSISTENCE_MODE
-
-    return bool(int(os.environ.get(RAY_AIR_NEW_PERSISTENCE_MODE, "1")))
-
-
 class _ExcludingLocalFilesystem(LocalFileSystem):
     """LocalFileSystem wrapper to exclude files according to patterns.
 

@@ -78,8 +78,6 @@ def _get_defaults_results_dir() -> str:
 # PACK to SPREAD. 1 for True, 0 for False.
 TRAIN_ENABLE_WORKER_SPREAD_ENV = "TRAIN_ENABLE_WORKER_SPREAD"
 
-RAY_AIR_NEW_PERSISTENCE_MODE = "RAY_AIR_NEW_PERSISTENCE_MODE"
-
 # Set this to 0 to disable changing the working directory of each Tune Trainable
 # or Train worker to the trial directory. Defaults to 1.
 RAY_CHDIR_TO_TRIAL_DIR = "RAY_CHDIR_TO_TRIAL_DIR"
@@ -91,7 +89,6 @@ def _get_defaults_results_dir() -> str:
     ENABLE_SHARE_NEURON_CORES_ACCELERATOR_ENV,
     TRAIN_PLACEMENT_GROUP_TIMEOUT_S_ENV,
     TRAIN_ENABLE_WORKER_SPREAD_ENV,
-    RAY_AIR_NEW_PERSISTENCE_MODE,
     RAY_CHDIR_TO_TRIAL_DIR,
 }
 

@@ -24,7 +24,6 @@
     _download_from_fs_path,
 )
 from ray.train.base_trainer import TrainingFailedError
-from ray.train.constants import RAY_AIR_NEW_PERSISTENCE_MODE
 from ray.train.data_parallel_trainer import DataParallelTrainer
 from ray.tune.trainable.trainable import _DICT_CHECKPOINT_FILE_NAME
 
@@ -64,14 +63,6 @@ def dummy_context_manager(*args, **kwargs):
     yield "dummy value"
 
 
-@pytest.fixture(scope="module")
-def enable_new_persistence_mode():
-    with pytest.MonkeyPatch.context() as mp:
-        mp.setenv(RAY_AIR_NEW_PERSISTENCE_MODE, "1")
-        yield
-        mp.setenv(RAY_AIR_NEW_PERSISTENCE_MODE, "0")
-
-
 @pytest.fixture(autouse=True)
 def disable_driver_artifact_sync():
     # NOTE: Hack to make sure that the driver doesn't sync the artifacts.
@@ -84,7 +75,7 @@ def disable_driver_artifact_sync():
 
 
 @pytest.fixture(autouse=True, scope="module")
-def ray_start_4_cpus(enable_new_persistence_mode):
+def ray_start_4_cpus():
     # Make sure to set the env var before calling ray.init()
     ray.init(num_cpus=4)
     yield

@@ -2,15 +2,14 @@
 
 import ray
 from ray.air.constants import COPY_DIRECTORY_CHECKPOINTS_INSTEAD_OF_MOVING_ENV
-from ray.train.constants import RAY_AIR_NEW_PERSISTENCE_MODE, RAY_CHDIR_TO_TRIAL_DIR
+from ray.train.constants import RAY_CHDIR_TO_TRIAL_DIR
 
 DEFAULT_ENV_VARS = {
     # https://github.com/ray-project/ray/issues/28197
     "PL_DISABLE_FORK": "1"
 }
 ENV_VARS_TO_PROPAGATE = {
     COPY_DIRECTORY_CHECKPOINTS_INSTEAD_OF_MOVING_ENV,
-    RAY_AIR_NEW_PERSISTENCE_MODE,
     RAY_CHDIR_TO_TRIAL_DIR,
     "TUNE_CHECKPOINT_CLOUD_RETRY_NUM",
     "TUNE_CHECKPOINT_CLOUD_RETRY_WAIT_TIME_S",

diff --git a/release/BUILD b/release/BUILD
@@ -236,7 +236,6 @@ py_test(
     srcs = test_srcs,
     env = {
         "IS_SMOKE_TEST": "1",
-        "RAY_AIR_NEW_PERSISTENCE_MODE": "1",
     },
     main = "golden_notebook_tests/workloads/torch_tune_serve_test.py",
     tags = [

diff --git a/release/jobs_tests/workloads/jobs_basic.py b/release/jobs_tests/workloads/jobs_basic.py
@@ -15,7 +15,6 @@
 from ray.dashboard.modules.job.common import JobStatus
 
 from ray.job_submission import JobSubmissionClient
-from ray.train.constants import RAY_AIR_NEW_PERSISTENCE_MODE
 
 
 def wait_until_finish(
@@ -62,7 +61,6 @@ def wait_until_finish(
         runtime_env={
             "pip": ["ray[tune]"],
             "working_dir": args.working_dir,
-            "env_vars": {RAY_AIR_NEW_PERSISTENCE_MODE: "1"},
         },
     )
     timeout_s = 10 * 60

diff --git a/release/lightgbm_tests/app_config.yaml b/release/lightgbm_tests/app_config.yaml
@@ -1,5 +1,5 @@
 base_image: {{ env["RAY_IMAGE_NIGHTLY_CPU"] }}
-env_vars: { "RAY_AIR_NEW_PERSISTENCE_MODE": "1" }
+env_vars: {}
 debian_packages:
   - curl
 

diff --git a/release/lightgbm_tests/app_config_gpu.yaml b/release/lightgbm_tests/app_config_gpu.yaml
@@ -1,5 +1,5 @@
 base_image: {{ env["RAY_IMAGE_ML_NIGHTLY_GPU"] }}
-env_vars: { "RAY_AIR_NEW_PERSISTENCE_MODE": "1" }
+env_vars: {}
 debian_packages:
   - curl
 

diff --git a/release/long_running_distributed_tests/app_config.yaml b/release/long_running_distributed_tests/app_config.yaml
@@ -1,5 +1,5 @@
 base_image: {{ env["RAY_IMAGE_ML_NIGHTLY_GPU"] }}
-env_vars: { "RAY_AIR_NEW_PERSISTENCE_MODE": "1" }
+env_vars: {}
 debian_packages:
   - curl
 

diff --git a/release/long_running_tests/app_config.yaml b/release/long_running_tests/app_config.yaml
@@ -1,5 +1,5 @@
 base_image: {{ env["RAY_IMAGE_NIGHTLY_CPU"] }}
-env_vars: {"RLLIB_TEST_NO_JAX_IMPORT": "1", "RAY_AIR_NEW_PERSISTENCE_MODE": "1"}
+env_vars: { "RLLIB_TEST_NO_JAX_IMPORT": "1" }
 
 debian_packages:
 

diff --git a/release/long_running_tests/app_config_np.yaml b/release/long_running_tests/app_config_np.yaml
@@ -1,5 +1,5 @@
 base_image: {{ env["RAY_IMAGE_NIGHTLY_CPU"] }}
-env_vars: { "RAY_AIR_NEW_PERSISTENCE_MODE": "1" }
+env_vars: {}
 
 debian_packages:
   - curl