Make shell ops inherit env vars from outside environment (#6513)

python_modules/libraries/dagster-shell/dagster_shell/solids.py

@@ -64,8 +64,7 @@ def core_shell(dagster_decorator, decorator_name):
     )
     def shell_fn(context, shell_command):
         op_config = context.op_config.copy()
-        if not op_config.get("env"):
-            op_config["env"] = os.environ.copy()
+        op_config["env"] = {**os.environ, **op_config.get("env", {})}
         output, return_code = execute(shell_command=shell_command, log=context.log, **op_config)
 
         if return_code:
@@ -200,8 +199,7 @@ def core_create_shell_command(
     )
     def _shell_fn(context):
         op_config = context.op_config.copy()
-        if not op_config.get("env"):
-            op_config["env"] = os.environ.copy()
+        op_config["env"] = {**os.environ, **op_config.get("env", {})}
         output, return_code = execute(shell_command=shell_command, log=context.log, **op_config)
 
         if return_code:
@@ -328,8 +326,7 @@ def core_create_shell_script(
     )
     def _shell_script_fn(context):
         op_config = context.op_config.copy()
-        if not op_config.get("env"):
-            op_config["env"] = os.environ.copy()
+        op_config["env"] = {**os.environ, **op_config.get("env", {})}
         output, return_code = execute_script_file(
             shell_script_path=shell_script_path, log=context.log, **op_config
         )

python_modules/libraries/dagster-shell/dagster_shell_tests/test_solids.py

@@ -24,9 +24,28 @@ def test_shell_command(factory):
     assert result.output_values == {"result": "this is a test message: foobar\n"}
 
 
+@pytest.mark.parametrize("factory", [create_shell_command_solid, create_shell_command_op])
+def test_shell_command_inherits_environment(monkeypatch, factory):
+    # OUTSIDE_ENV_VAR represents an environment variable that should be available
+    # to jobs. eg. 12-factor app secrets, defined in your Docker container, etc.
+    monkeypatch.setenv("OUTSIDE_ENV_VAR", "foo")
+
+    solid = factory('echo "$OUTSIDE_ENV_VAR:$MY_ENV_VAR"', name="foobar")
+
+    # inherit outside environment variables if none specified for op
+    result = execute_solid(solid)
+    assert result.output_values == {"result": "foo:\n"}
+
+    # also inherit outside environment variables if env vars specified for op
+    result = execute_solid(
+        solid,
+        run_config={"solids": {"foobar": {"config": {"env": {"MY_ENV_VAR": "bar"}}}}},
+    )
+    assert result.output_values == {"result": "foo:bar\n"}
+
+
 @pytest.mark.parametrize("shell_defn,name", [(shell_op, "shell_op"), (shell_solid, "shell_solid")])
 def test_shell(shell_defn, name):
-
     result = execute_solid(
         shell_defn,
         input_values={"shell_command": 'echo "this is a test message: $MY_ENV_VAR"'},