diff --git a/script/get-dataset-mlperf-inference-llama3/meta.yaml b/script/get-dataset-mlperf-inference-llama3/meta.yaml index 9e8762f35..24aef9e20 100644 --- a/script/get-dataset-mlperf-inference-llama3/meta.yaml +++ b/script/get-dataset-mlperf-inference-llama3/meta.yaml @@ -36,12 +36,28 @@ variations: MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl MLC_DATASET_TYPE: validation MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + validation,rclone: + env: + MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + validation,r2-downloader: + env: + MLC_RCLONE_URL: "" + MLC_DATASET_FILE_NAME: "" calibration: group: dataset-type env: MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl MLC_DATASET_TYPE: calibration MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + calibration,rclone: + env: + MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + calibration,r2-downloader: + env: + MLC_RCLONE_URL: "" + MLC_DATASET_FILE_NAME: "" rclone: add_deps_recursive: dae: @@ -52,5 +68,13 @@ variations: MLC_DOWNLOAD_URL: <<>> MLC_RCLONE_CONFIG_NAME: mlc-inference group: download-tool + r2-downloader: + add_deps_recursive: + dae: + tags: _r2-downloader + env: + MLC_DOWNLOAD_FILENAME: checkpoint + MLC_RCLONE_CONFIG_NAME: mlc-inference + group: download-tool print_env_at_the_end: MLC_DATASET_LLAMA3_PATH: Path to the dataset diff --git a/script/get-dataset-waymo/meta.yaml b/script/get-dataset-waymo/meta.yaml index ef56e4a2e..e90315017 100644 --- a/script/get-dataset-waymo/meta.yaml +++ b/script/get-dataset-waymo/meta.yaml @@ -21,22 +21,12 @@ variations: group: download-src default: true prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_waymo - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_WAYMO_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_WAYMO_PATH - MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format extra_cache_tags: waymo,dataset force_cache: true names: @@ -54,6 +44,21 @@ variations: add_deps_recursive: dae: tags: _rclone + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_waymo + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader default: true dry-run: group: run-mode @@ -62,3 +67,12 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + rclone,mlc: + env: + MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/dataset.uri diff --git a/script/get-ml-model-llama2/customize.py b/script/get-ml-model-llama2/customize.py index 494b7c9cf..491e76248 100644 --- a/script/get-ml-model-llama2/customize.py +++ b/script/get-ml-model-llama2/customize.py @@ -29,12 +29,6 @@ def preprocess(i): else: if path == '' or not os.path.exists(path): env['MLC_TMP_REQUIRE_DOWNLOAD'] = 'yes' - if env['MLC_DOWNLOAD_SRC'] == "mlcommons": - i['run_script_input']['script_name'] = 'run-rclone' - if env.get('MLC_OUTDIRNAME', '') != '': - env['LLAMA2_CHECKPOINT_PATH'] = env['MLC_OUTDIRNAME'] - else: - env['LLAMA2_CHECKPOINT_PATH'] = os.getcwd() return {'return': 0} @@ -42,11 +36,12 @@ def preprocess(i): def postprocess(i): env = i['env'] - if env.get('LLAMA2_CHECKPOINT_PATH', '') == '': - env['LLAMA2_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] - else: - env['MLC_ML_MODEL_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] - env['MLC_ML_MODEL_LLAMA2_FILE_WITH_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_PATH'] + if env.get('MLC_DOWNLOAD_MODE', '') != "dry": + if env.get('LLAMA2_CHECKPOINT_PATH', '') == '': + env['LLAMA2_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] + else: + env['MLC_ML_MODEL_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] + env['MLC_ML_MODEL_LLAMA2_FILE_WITH_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_PATH'] return {'return': 0} diff --git a/script/get-ml-model-llama2/meta.yaml b/script/get-ml-model-llama2/meta.yaml index d0e14cf77..a2b0c3c1e 100644 --- a/script/get-ml-model-llama2/meta.yaml +++ b/script/get-ml-model-llama2/meta.yaml @@ -41,6 +41,12 @@ tags: - llama2-70b - text-summarization uid: 5db97be9f61244c6 +tests: + needs_pat: true + run_inputs: + - variations_list: + - r2-downloader,70b,mlc,dry-run + - r2-downloader,7b,mlc,dry-run variations: L40s: env: @@ -84,11 +90,11 @@ variations: MLC_ML_MODEL_PRECISION: int8 MLC_ML_MODEL_WEIGHT_DATA_TYPES: int8 group: precision - mlc: - group: download-source - default: true - env: - MLC_DOWNLOAD_SRC: mlcommons + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone prehook_deps: - tags: get,rclone enable_if_env: @@ -99,6 +105,55 @@ variations: enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - yes + r2-downloader: + group: download-tool + default: true + add_deps_recursive: + dae: + tags: _r2-downloader + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + mlc: + group: download-source + default: true + env: + MLC_DOWNLOAD_SRC: mlcommons + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: LLAMA2_CHECKPOINT_PATH + MLC_EXTRACT_FINAL_ENV_NAME: LLAMA2_CHECKPOINT_PATH + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + mlc,rclone,70b: + env: + MLC_DOWNLOAD_URL: mlc-llama2:Llama-2-70b-chat-hf + mlc,rclone,7b: + env: + MLC_DOWNLOAD_URL: mlc-llama2:Llama-2-7b-chat-hf + mlc,r2-downloader,70b: + env: + MLC_DOWNLOAD_URL: https://llama2.mlcommons-storage.org/metadata/llama-2-70b-chat-hf.uri + mlc,r2-downloader,7b: + env: + MLC_DOWNLOAD_URL: https://llama2.mlcommons-storage.org/metadata/llama-2-7b-chat-hf.uri hf: group: download-source env: diff --git a/script/get-ml-model-llama2/run-rclone.sh b/script/get-ml-model-llama2/run-rclone.sh deleted file mode 100644 index 1fc602a9f..000000000 --- a/script/get-ml-model-llama2/run-rclone.sh +++ /dev/null @@ -1,4 +0,0 @@ -cmd="rclone sync mlc-llama2:${MLC_GIT_CHECKOUT_FOLDER} ${LLAMA2_CHECKPOINT_PATH}/${MLC_GIT_CHECKOUT_FOLDER} -P" -echo $cmd -eval $cmd -test $? -eq 0 || exit $? diff --git a/script/get-ml-model-llama3/meta.yaml b/script/get-ml-model-llama3/meta.yaml index 8b26d7fff..5e2c04fc1 100644 --- a/script/get-ml-model-llama3/meta.yaml +++ b/script/get-ml-model-llama3/meta.yaml @@ -59,6 +59,27 @@ variations: env: MLC_ML_MODEL_NAME: Llama-3.1-8b-Instruct MLC_ML_MODEL_R2_HOSTED_NAME: llama3-1-8b-instruct + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_mlperf-llama3-1 + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + default: true + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader mlc: group: download-src default: true @@ -92,27 +113,6 @@ variations: adr: dae: extra_cache_tags: llama3,dataset,rclone - rclone: - group: download-tool - add_deps_recursive: - dae: - tags: _rclone - prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_mlperf-llama3-1 - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - default: true - r2-downloader: - group: download-tool - add_deps_recursive: - dae: - tags: _r2-downloader dry-run: group: run-mode env: diff --git a/script/get-ml-model-pointpainting/customize.py b/script/get-ml-model-pointpainting/customize.py index 07b18602d..ec30f0d2b 100644 --- a/script/get-ml-model-pointpainting/customize.py +++ b/script/get-ml-model-pointpainting/customize.py @@ -24,13 +24,6 @@ def preprocess(i): if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') == '' or env.get( 'MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" - if env['MLC_DOWNLOAD_SRC'] == "mlcommons": - i['run_script_input']['script_name'] = 'run-rclone' - if env.get('MLC_OUTDIRNAME', '') != '': - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'] = env['MLC_OUTDIRNAME'] - else: - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'] = os.path.join( - os.getcwd(), "model") return {'return': 0} @@ -39,21 +32,22 @@ def postprocess(i): env = i['env'] - if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') == '': - if env['MLC_ML_MODEL_PP_FORMAT'] == "onnx": - env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp.onnx") - else: - env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp_ep36.pth") - - if env.get('MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': - if env['MLC_ML_MODEL_DPLAB_RESNET50_FORMAT'] == "onnx": - env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "deeplabv3+.onnx") - else: - env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], - "best_deeplabv3plus_resnet50_waymo_os16.pth") + if env.get('MLC_DOWNLOAD_MODE', '') != "dry": + if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') == '': + if env['MLC_ML_MODEL_PP_FORMAT'] == "onnx": + env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp.onnx") + else: + env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp_ep36.pth") + + if env.get('MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': + if env['MLC_ML_MODEL_DPLAB_RESNET50_FORMAT'] == "onnx": + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "deeplabv3+.onnx") + else: + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], + "best_deeplabv3plus_resnet50_waymo_os16.pth") return {'return': 0} diff --git a/script/get-ml-model-pointpainting/meta.yaml b/script/get-ml-model-pointpainting/meta.yaml index b811d58d6..8047b6188 100644 --- a/script/get-ml-model-pointpainting/meta.yaml +++ b/script/get-ml-model-pointpainting/meta.yaml @@ -27,9 +27,11 @@ variations: env: MLC_ML_MODEL_PP_FORMAT: onnx MLC_ML_MODEL_DPLAB_RESNET50_FORMAT: onnx - mlc: - group: download-src - default: true + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone prehook_deps: - tags: get,rclone enable_if_env: @@ -40,6 +42,47 @@ variations: enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - yes + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader + default: true + mlc: + group: download-src + default: true + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_POINT_PAINTING_TMP_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_POINT_PAINTING_TMP_PATH + extra_cache_tags: waymo,dataset + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL env: MLC_DOWNLOAD_SRC: mlcommons - + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + rclone,mlc: + env: + MLC_DOWNLOAD_URL: mlc-waymo:waymo_preprocessed_dataset/model + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/model.uri diff --git a/script/get-ml-model-pointpainting/run-rclone.sh b/script/get-ml-model-pointpainting/run-rclone.sh deleted file mode 100644 index 9b76a1511..000000000 --- a/script/get-ml-model-pointpainting/run-rclone.sh +++ /dev/null @@ -1,4 +0,0 @@ -cmd="rclone sync mlc-waymo:waymo_preprocessed_dataset/model ${MLC_ML_MODEL_POINT_PAINTING_TMP_PATH} -P" -echo $cmd -eval $cmd -test $? -eq 0 || exit $? \ No newline at end of file