From e6f8f1e2e337474c6d4ceb84caf77d0d2f563d86 Mon Sep 17 00:00:00 2001 From: jsemric Date: Thu, 22 Jul 2021 10:17:41 +0200 Subject: [PATCH 1/3] create virtualenv via python call --- airflow/utils/python_virtualenv.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airflow/utils/python_virtualenv.py b/airflow/utils/python_virtualenv.py index 2da609690da1c..3d58c25d17d2e 100644 --- a/airflow/utils/python_virtualenv.py +++ b/airflow/utils/python_virtualenv.py @@ -22,12 +22,13 @@ from typing import List, Optional import jinja2 +import virtualenv from airflow.utils.process_utils import execute_in_subprocess def _generate_virtualenv_cmd(tmp_dir: str, python_bin: str, system_site_packages: bool) -> List[str]: - cmd = ['virtualenv', tmp_dir] + cmd = [tmp_dir] if system_site_packages: cmd.append('--system-site-packages') if python_bin is not None: @@ -92,7 +93,7 @@ def prepare_virtualenv( :rtype: str """ virtualenv_cmd = _generate_virtualenv_cmd(venv_directory, python_bin, system_site_packages) - execute_in_subprocess(virtualenv_cmd) + virtualenv.cli_run(virtualenv_cmd) pip_cmd = _generate_pip_install_cmd(venv_directory, requirements) if pip_cmd: execute_in_subprocess(pip_cmd) From cc64c475016cbf211b6b9a506aa72c28dc9c3f88 Mon Sep 17 00:00:00 2001 From: jsemric Date: Thu, 22 Jul 2021 11:36:42 +0200 Subject: [PATCH 2/3] call virtualenv cmd via python -m --- airflow/utils/python_virtualenv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/utils/python_virtualenv.py b/airflow/utils/python_virtualenv.py index 3d58c25d17d2e..9344c6a3d647c 100644 --- a/airflow/utils/python_virtualenv.py +++ b/airflow/utils/python_virtualenv.py @@ -18,17 +18,17 @@ # """Utilities for creating a virtual environment""" import os +import sys from collections import deque from typing import List, Optional import jinja2 -import virtualenv from airflow.utils.process_utils import execute_in_subprocess def _generate_virtualenv_cmd(tmp_dir: str, python_bin: str, system_site_packages: bool) -> List[str]: - cmd = [tmp_dir] + cmd = [sys.executable, '-m', 'virtualenv', tmp_dir] if system_site_packages: cmd.append('--system-site-packages') if python_bin is not None: @@ -93,7 +93,7 @@ def prepare_virtualenv( :rtype: str """ virtualenv_cmd = _generate_virtualenv_cmd(venv_directory, python_bin, system_site_packages) - virtualenv.cli_run(virtualenv_cmd) + execute_in_subprocess(virtualenv_cmd) pip_cmd = _generate_pip_install_cmd(venv_directory, requirements) if pip_cmd: execute_in_subprocess(pip_cmd) From 299192ddad9180f7013f717c373e118eab754cc4 Mon Sep 17 00:00:00 2001 From: jsemric Date: Fri, 23 Jul 2021 11:11:00 +0200 Subject: [PATCH 3/3] prepend python call in prepare-virtualenv tests --- tests/utils/test_python_virtualenv.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/utils/test_python_virtualenv.py b/tests/utils/test_python_virtualenv.py index 01ff8803c4151..dc4ee7aa8153f 100644 --- a/tests/utils/test_python_virtualenv.py +++ b/tests/utils/test_python_virtualenv.py @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. # +import sys import unittest from unittest import mock @@ -29,7 +30,9 @@ def test_should_create_virtualenv(self, mock_execute_in_subprocess): venv_directory="/VENV", python_bin="pythonVER", system_site_packages=False, requirements=[] ) assert "/VENV/bin/python" == python_bin - mock_execute_in_subprocess.assert_called_once_with(['virtualenv', '/VENV', '--python=pythonVER']) + mock_execute_in_subprocess.assert_called_once_with( + [sys.executable, '-m', 'virtualenv', '/VENV', '--python=pythonVER'] + ) @mock.patch('airflow.utils.python_virtualenv.execute_in_subprocess') def test_should_create_virtualenv_with_system_packages(self, mock_execute_in_subprocess): @@ -38,7 +41,7 @@ def test_should_create_virtualenv_with_system_packages(self, mock_execute_in_sub ) assert "/VENV/bin/python" == python_bin mock_execute_in_subprocess.assert_called_once_with( - ['virtualenv', '/VENV', '--system-site-packages', '--python=pythonVER'] + [sys.executable, '-m', 'virtualenv', '/VENV', '--system-site-packages', '--python=pythonVER'] ) @mock.patch('airflow.utils.python_virtualenv.execute_in_subprocess') @@ -51,7 +54,9 @@ def test_should_create_virtualenv_with_extra_packages(self, mock_execute_in_subp ) assert "/VENV/bin/python" == python_bin - mock_execute_in_subprocess.assert_any_call(['virtualenv', '/VENV', '--python=pythonVER']) + mock_execute_in_subprocess.assert_any_call( + [sys.executable, '-m', 'virtualenv', '/VENV', '--python=pythonVER'] + ) mock_execute_in_subprocess.assert_called_with(['/VENV/bin/pip', 'install', 'apache-beam[gcp]'])