Support default_env, gh_token for docker, relative path bug for --outdirname

automation/script/docker.py

@@ -55,6 +55,10 @@ def dockerfile(self_module, input_params):
             'script_variation_tags': variation_tags
         }
         docker_settings = metadata.get('docker', {})
+        docker_settings_default_env = docker_settings.get('default_env', {})
+        for key in docker_settings_default_env:
+            env.setdefault(key, docker_settings_default_env[key])
+
         state_data['docker'] = docker_settings
         add_deps_recursive = input_params.get('add_deps_recursive', {})
 
@@ -301,6 +305,10 @@ def docker_run(self_module, i):
         folder_path_env_keys = meta.get('folder_path_env_keys', [])
 
         docker_settings = meta.get('docker', {})
+        docker_settings_default_env = docker_settings.get('default_env', {})
+        for key in docker_settings_default_env:
+            env.setdefault(key, docker_settings_default_env[key])
+
         state['docker'] = docker_settings
         run_state = {
             'deps': [], 'fake_deps': [], 'parent': None,

automation/script/module.py

@@ -504,6 +504,9 @@ def _run(self, i):
                 env[key] = os.environ[key]
 
         r = self._update_env_from_input(env, i)
+        if env.get('MLC_OUTDIRNAME', '') != '':
+            if not os.path.isabs(env['MLC_OUTDIRNAME']):
+                env['MLC_OUTDIRNAME'] = os.path.abspath(env['MLC_OUTDIRNAME'])
 
         #######################################################################
         # Check if we want to skip cache (either by skip_cache or by fake_run)
@@ -1748,12 +1751,7 @@ def _run(self, i):
 
             tmp_curdir = os.getcwd()
             if env.get('MLC_OUTDIRNAME', '') != '':
-                if os.path.isabs(env['MLC_OUTDIRNAME']) or recursion:
-                    c_outdirname = env['MLC_OUTDIRNAME']
-                else:
-                    c_outdirname = os.path.join(
-                        env['MLC_TMP_CURRENT_PATH'], env['MLC_OUTDIRNAME'])
-                    env['MLC_OUTDIRNAME'] = c_outdirname
+                c_outdirname = env['MLC_OUTDIRNAME']
 
                 if not fake_run:  # prevent permission error inside docker runs
                     if not os.path.exists(c_outdirname):

script/build-dockerfile/customize.py

@@ -328,6 +328,11 @@ def preprocess(i):
         EOL)
 
     f.write(EOL + '# Download MLC repo for scripts' + EOL)
+    pat = env.get('MLC_GH_TOKEN')
+    if pat:
+        token_string = f" --pat={pat}"
+    else:
+        token_string = ""
 
     if use_copy_repo:
         docker_repo_dest = "$HOME/MLC/repos/mlcommons@mlperf-automations"
@@ -336,7 +341,9 @@ def preprocess(i):
             EOL)
 
         f.write(EOL + '# Register MLC repository' + EOL)
-        f.write('RUN mlc pull repo --url={} --quiet'.format(docker_repo_dest) + EOL)
+        f.write(
+            'RUN mlc pull repo --url={} {token_string} --quiet'.format(docker_repo_dest) +
+            EOL)
         f.write(EOL)
 
     else:
@@ -349,6 +356,7 @@ def preprocess(i):
             'RUN mlc pull repo ' +
             mlc_mlops_repo +
             mlc_mlops_repo_branch_string +
+            token_string +
             x +
             EOL)
 

script/get-generic-sys-util/meta.yaml

@@ -712,6 +712,8 @@ variations:
     state:
       postfix:
         apt: postfix
+        dnf: postfix
+        yum: postfix
   psmisc:
     env:
       MLC_SYS_UTIL_NAME: psmisc