From 75fe56e738fdfe4d226c9366aab5801b03aa3440 Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 10 Sep 2025 20:15:06 +0000 Subject: [PATCH 1/5] Support --pat for mlc pull repo in dockerfile, support default_env in docker meta --- automation/script/docker.py | 8 ++++++++ script/build-dockerfile/customize.py | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index c97668ac3..5f5e06b45 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -55,6 +55,10 @@ def dockerfile(self_module, input_params): 'script_variation_tags': variation_tags } docker_settings = metadata.get('docker', {}) + docker_settings_default_env = docker_settings.get('default_env', {}) + for key in docker_settings_default_env: + env.setdefault(key, docker_settings_default_env[key]) + state_data['docker'] = docker_settings add_deps_recursive = input_params.get('add_deps_recursive', {}) @@ -301,6 +305,10 @@ def docker_run(self_module, i): folder_path_env_keys = meta.get('folder_path_env_keys', []) docker_settings = meta.get('docker', {}) + docker_settings_default_env = docker_settings.get('default_env', {}) + for key in docker_settings_default_env: + env.setdefault(key, docker_settings_default_env[key]) + state['docker'] = docker_settings run_state = { 'deps': [], 'fake_deps': [], 'parent': None, diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 987f7ba6f..31d526de8 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -328,6 +328,11 @@ def preprocess(i): EOL) f.write(EOL + '# Download MLC repo for scripts' + EOL) + pat = env.get('MLC_GH_TOKEN') + if pat: + token_string = f" --pat={pat}" + else: + token_string = "" if use_copy_repo: docker_repo_dest = "$HOME/MLC/repos/mlcommons@mlperf-automations" @@ -336,7 +341,7 @@ def preprocess(i): EOL) f.write(EOL + '# Register MLC repository' + EOL) - f.write('RUN mlc pull repo --url={} --quiet'.format(docker_repo_dest) + EOL) + f.write('RUN mlc pull repo --url={} {token_string} --quiet'.format(docker_repo_dest) + EOL) f.write(EOL) else: @@ -349,6 +354,7 @@ def preprocess(i): 'RUN mlc pull repo ' + mlc_mlops_repo + mlc_mlops_repo_branch_string + + token_string + x + EOL) From 2bb2cffe6609c122e8eb34fa28a147e4680d9dab Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 10 Sep 2025 20:15:50 +0000 Subject: [PATCH 2/5] [Automated Commit] Format Codebase [skip ci] --- automation/script/docker.py | 2 +- script/build-dockerfile/customize.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index 5f5e06b45..c44ef84e0 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -308,7 +308,7 @@ def docker_run(self_module, i): docker_settings_default_env = docker_settings.get('default_env', {}) for key in docker_settings_default_env: env.setdefault(key, docker_settings_default_env[key]) - + state['docker'] = docker_settings run_state = { 'deps': [], 'fake_deps': [], 'parent': None, diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 31d526de8..92a0b6937 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -341,7 +341,9 @@ def preprocess(i): EOL) f.write(EOL + '# Register MLC repository' + EOL) - f.write('RUN mlc pull repo --url={} {token_string} --quiet'.format(docker_repo_dest) + EOL) + f.write( + 'RUN mlc pull repo --url={} {token_string} --quiet'.format(docker_repo_dest) + + EOL) f.write(EOL) else: From 1a515d83f2a48052173423bb67e140c698b37e78 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 11 Sep 2025 04:14:38 +0530 Subject: [PATCH 3/5] Support dnf and yum installation of postfix --- script/get-generic-sys-util/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index cbb1bd9fc..d337ad438 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -698,6 +698,8 @@ variations: state: postfix: apt: postfix + dnf: postfix + yum: postfix psmisc: env: MLC_SYS_UTIL_NAME: psmisc From 5b7872c9178c33599e0daa604898fde5c8e73996 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 12 Sep 2025 18:09:46 +0530 Subject: [PATCH 4/5] Fix a bug in relative path for --outdirname --- automation/script/module.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 298a6e2bc..1f7bc6935 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -504,6 +504,9 @@ def _run(self, i): env[key] = os.environ[key] r = self._update_env_from_input(env, i) + if env.get('MLC_OUTDIRNAME', '') != '': + if not os.path.isabs(env['MLC_OUTDIRNAME']): + env['MLC_OUTDIRNAME'] = os.path.abspath(env['MLC_OUTDIRNAME']) ####################################################################### # Check if we want to skip cache (either by skip_cache or by fake_run) @@ -1748,12 +1751,7 @@ def _run(self, i): tmp_curdir = os.getcwd() if env.get('MLC_OUTDIRNAME', '') != '': - if os.path.isabs(env['MLC_OUTDIRNAME']) or recursion: - c_outdirname = env['MLC_OUTDIRNAME'] - else: - c_outdirname = os.path.join( - env['MLC_TMP_CURRENT_PATH'], env['MLC_OUTDIRNAME']) - env['MLC_OUTDIRNAME'] = c_outdirname + c_outdirname = env['MLC_OUTDIRNAME'] if not fake_run: # prevent permission error inside docker runs if not os.path.exists(c_outdirname): @@ -3445,7 +3443,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a env[kk] = tmp_env[kk] elif key in tmp_env: env[key] = tmp_env[key] - + if d.get("reuse_version", False): for k in tmp_env: if k.startswith('MLC_VERSION'): From 10db393e67205be52260161cd946a442da930e09 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 12 Sep 2025 12:40:13 +0000 Subject: [PATCH 5/5] [Automated Commit] Format Codebase [skip ci] --- automation/script/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index 1f7bc6935..0ee39ea44 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3443,7 +3443,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a env[kk] = tmp_env[kk] elif key in tmp_env: env[key] = tmp_env[key] - + if d.get("reuse_version", False): for k in tmp_env: if k.startswith('MLC_VERSION'):