Don't include script directory in sys.path if it's started via python…

… -m (ray-project#28140)

Redo ray-project#28043

Signed-off-by: Jiajun Yao <jeromeyjj@gmail.com>

python/ray/_private/worker.py

@@ -2067,10 +2067,15 @@ def connect(
         # are the same.
         # When using an interactive shell, there is no script directory.
         if not interactive_mode:
-            script_directory = os.path.abspath(os.path.dirname(sys.argv[0]))
-            worker.run_function_on_all_workers(
-                lambda worker_info: sys.path.insert(1, script_directory)
-            )
+            script_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
+            # If driver's sys.path doesn't include the script directory
+            # (e.g driver is started via `python -m`,
+            # see https://peps.python.org/pep-0338/),
+            # then we shouldn't add it to the workers.
+            if script_directory in sys.path:
+                worker.run_function_on_all_workers(
+                    lambda worker_info: sys.path.insert(1, script_directory)
+                )
         # In client mode, if we use runtime envs with "working_dir", then
         # it'll be handled automatically.  Otherwise, add the current dir.
         if not job_config.client_job and not job_config.runtime_env_has_working_dir():

python/ray/tests/test_basic_5.py

@@ -4,6 +4,7 @@
 import os
 import sys
 import time
+import subprocess
 
 import pytest
 
@@ -180,6 +181,48 @@ def ready():
     run_string_as_driver(driver_script)
 
 
+def test_worker_sys_path_contains_driver_script_directory(tmp_path, monkeypatch):
+    package_folder = tmp_path / "package"
+    package_folder.mkdir()
+    init_file = tmp_path / "package" / "__init__.py"
+    init_file.write_text("")
+
+    module1_file = tmp_path / "package" / "module1.py"
+    module1_file.write_text(
+        f"""
+import sys
+import ray
+ray.init()
+
+@ray.remote
+def sys_path():
+    return sys.path
+
+assert r'{str(tmp_path / "package")}' in ray.get(sys_path.remote())
+"""
+    )
+    subprocess.check_call(["python", str(module1_file)])
+
+    # If the driver script is run via `python -m`,
+    # the script directory is not included in sys.path.
+    module2_file = tmp_path / "package" / "module2.py"
+    module2_file.write_text(
+        f"""
+import sys
+import ray
+ray.init()
+
+@ray.remote
+def sys_path():
+    return sys.path
+
+assert r'{str(tmp_path / "package")}' not in ray.get(sys_path.remote())
+"""
+    )
+    monkeypatch.chdir(str(tmp_path))
+    subprocess.check_call(["python", "-m", "package.module2"])
+
+
 if __name__ == "__main__":
     if os.environ.get("PARALLEL_CI"):
         sys.exit(pytest.main(["-n", "auto", "--boxed", "-vs", __file__]))