diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8db080676..3e70589a3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -3,3 +3,21 @@ - [ ] Target branch is `dev` 📌 Note: PRs must be raised against `dev`. Do not commit directly to `main`. + +### ✅ Testing & CI +- [ ] No existing GitHub Actions are failing because of this change + +### 📁 File Hygiene & Output Handling +- [ ] No unintended files (e.g., logs, cache, temp files, __pycache__, output folders) are committed + +### 📝 Comments & Communication +- [ ] Proper inline comments are added to explain important or non-obvious changes +- [ ] PR title and description clearly state what the PR does and why +- [ ] Related issues (if any) are properly referenced (`Fixes #`, `Related to #`, etc.) +- [ ] All reviewer feedback has been addressed + +### 🛡ī¸ Safety & Security +- [ ] No secrets or credentials are committed +- [ ] Paths, shell commands, and environment handling are safe and portable + + diff --git a/.github/workflows/build_wheel_off.yml b/.github/workflows/build_wheel_off.yml index 199c11d54..9a7eb65e9 100644 --- a/.github/workflows/build_wheel_off.yml +++ b/.github/workflows/build_wheel_off.yml @@ -1,5 +1,6 @@ name: Build wheel and release into PYPI (off now) + on: push: branches: @@ -7,7 +8,6 @@ on: paths: - VERSION - jobs: build_wheels: if: github.repository_owner == 'mlcommons' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..33b3ec5bb --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,89 @@ +name: "CodeQL Advanced" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '39 13 * * 1' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: actions + build-mode: none + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Add any setup steps before running the `github/codeql-action/init` action. + # This includes steps like installing compilers or runtimes (`actions/setup-node` + # or others). This is typically only required for manual builds. + # - name: Setup runtime (example) + # uses: actions/setup-example@v1 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/document-scripts.yml b/.github/workflows/document-scripts.yml index 6984ff719..53b621548 100644 --- a/.github/workflows/document-scripts.yml +++ b/.github/workflows/document-scripts.yml @@ -65,7 +65,8 @@ jobs: - name: Document meta.yaml file run: | echo "Documenting ${{ matrix.modified_metas.file }}" - + python3 -m venv mlcflow + . mlcflow/bin/activate pip install mlcflow mlc add repo automation-scripts mlc doc script ${{ matrix.modified_metas.uid}} --quiet @@ -75,8 +76,7 @@ jobs: git config --global user.name github-actions[bot] git config --global user.email "github-actions[bot]@users.noreply.github.com" # Commit changes - git commit -m '[Automated Commit] Document ${{ matrix.modified_metas.file}} [skip ci]' - git push + git diff-index --quiet HEAD || (git commit -am "[Automated Commit] Document ${{ matrix.modified_metas.file}} [skip ci]" ) && git push) diff --git a/.github/workflows/run_tests_on_modified_meta_with_secrets.yml b/.github/workflows/run_tests_on_modified_meta_with_secrets.yml index 3e2484ce3..8c8f84506 100644 --- a/.github/workflows/run_tests_on_modified_meta_with_secrets.yml +++ b/.github/workflows/run_tests_on_modified_meta_with_secrets.yml @@ -7,20 +7,6 @@ on: - 'script/**meta.yaml' jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - gdrive_secret: ${{ steps.op-load-secret.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }} - steps: - - name: Load secret from 1Password - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - GDRIVE_SERVICE_ACCOUNT_KEY: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - get_modified_files: runs-on: ubuntu-latest outputs: @@ -52,7 +38,6 @@ jobs: process_modified_files: needs: - get_modified_files - - fetch-secret runs-on: ubuntu-latest if: needs.get_modified_files.outputs.processed_files != '[]' && needs.get_modified_files.outputs.processed_files != '' strategy: @@ -60,12 +45,18 @@ jobs: matrix: file_info: ${{ fromJSON(needs.get_modified_files.outputs.processed_files).file_info }} steps: + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + GDRIVE_SERVICE_ACCOUNT_KEY: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential + - name: Set RCLONE Service account env var from secret shell: bash run: | - echo "::add-mask::${{ needs.fetch-secret.outputs.gdrive_secret }}" - echo "RCLONE_CONFIG_MLC_COGNATA_SERVICE_ACCOUNT_CREDENTIALS=${{ needs.fetch-secret.outputs.gdrive_secret }}" >> $GITHUB_ENV - echo "RCLONE_CONFIG_MLC_NUSCENES_SERVICE_ACCOUNT_CREDENTIALS=${{ needs.fetch-secret.outputs.gdrive_secret }}" >> $GITHUB_ENV + echo "RCLONE_CONFIG_MLC_COGNATA_SERVICE_ACCOUNT_CREDENTIALS=${{ steps.op-load-secrets.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }}" >> $GITHUB_ENV + echo "RCLONE_CONFIG_MLC_NUSCENES_SERVICE_ACCOUNT_CREDENTIALS=${{ steps.op-load-secrets.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }}" >> $GITHUB_ENV - name: Process meta.yaml file run: | diff --git a/.github/workflows/test-mlc-script-features.yml b/.github/workflows/test-mlc-script-features.yml index 4570d5bdf..cbc532bb2 100644 --- a/.github/workflows/test-mlc-script-features.yml +++ b/.github/workflows/test-mlc-script-features.yml @@ -173,18 +173,16 @@ jobs: Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append } - - name: Load secret + - name: Load secrets if: github.repository_owner == 'mlcommons' && env.run_step == 'true' - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - name: Push Results env: - GITHUB_TOKEN: ${{ steps.op-load-secret.outputs.PAT }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} if: github.repository_owner == 'mlcommons' && env.run_step == 'true' run: | git config --global user.name "mlcommons-bot" diff --git a/.github/workflows/test-mlperf-automotive.yml b/.github/workflows/test-mlperf-automotive.yml index a03560dc1..54c50a95e 100644 --- a/.github/workflows/test-mlperf-automotive.yml +++ b/.github/workflows/test-mlperf-automotive.yml @@ -7,35 +7,24 @@ on: - 'script/**meta.yaml' jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - gdrive_secret: ${{ steps.op-load-secret.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }} - steps: - - name: Load secret from 1Password - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - GDRIVE_SERVICE_ACCOUNT_KEY: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - run-mlperf: runs-on: ubuntu-latest - needs: - - fetch-secret steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 2 + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + GDRIVE_SERVICE_ACCOUNT_KEY: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - name: Set RCLONE Service account env var from secret shell: bash run: | - echo "::add-mask::${{ needs.fetch-secret.outputs.gdrive_secret }}" - echo "RCLONE_CONFIG_MLC_COGNATA_SERVICE_ACCOUNT_CREDENTIALS=${{ needs.fetch-secret.outputs.gdrive_secret }}" >> $GITHUB_ENV - echo "RCLONE_CONFIG_MLC_NUSCENES_SERVICE_ACCOUNT_CREDENTIALS=${{ needs.fetch-secret.outputs.gdrive_secret }}" >> $GITHUB_ENV + echo "RCLONE_CONFIG_MLC_COGNATA_SERVICE_ACCOUNT_CREDENTIALS=${{ steps.op-load-secrets.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }}" >> $GITHUB_ENV + echo "RCLONE_CONFIG_MLC_NUSCENES_SERVICE_ACCOUNT_CREDENTIALS=${{ steps.op-load-secrets.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }}" >> $GITHUB_ENV - name: Run MLPerf run: | pip install mlcflow diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml index f4ded9274..e5925e99a 100644 --- a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml +++ b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml @@ -9,35 +9,8 @@ on: - '!**.md' jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} - steps: - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Encrypt secret - id: encrypt-secret - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - run: | - # AES-256 encrypt - encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ - openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT - build: name: MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }} - needs: [fetch-secret] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -85,25 +58,15 @@ jobs: else echo "run_step=false" >> $GITHUB_ENV fi - - name: Decrypt secret - id: decrypt-secret - shell: bash + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - encrypted_secret: ${{ needs.fetch-secret.outputs.encrypted_secret }} - run: | - echo "Running on OS: ${{ matrix.os }}" - - # Decrypt - decrypted=$(echo "$encrypted_secret" | \ - openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "::add-mask::$decrypted" - echo "DECRYPTED_SECRET=$decrypted" >> $GITHUB_OUTPUT + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - name: Push Results env: - GITHUB_TOKEN: ${{ steps.decrypt-secret.outputs.decrypted_secret }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} if: github.repository_owner == 'mlcommons' && env.run_step == 'true' run: | git config --global user.name "mlcommons-bot" diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index 147fb0ff4..29acfbf1d 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -8,36 +8,9 @@ on: - '**' - '!**.md' -jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} - steps: - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Encrypt secret - id: encrypt-secret - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - run: | - # AES-256 encrypt - encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ - openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT - +jobs: build: name: MLPerf inference MLCommons C++ ResNet50 - needs: [fetch-secret] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -80,25 +53,15 @@ jobs: else echo "run_step=false" >> $GITHUB_ENV fi - - name: Decrypt secret - id: decrypt-secret - shell: bash + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - encrypted_secret: ${{ needs.fetch-secret.outputs.encrypted_secret }} - run: | - echo "Running on OS: ${{ matrix.os }}" - - # Decrypt - decrypted=$(echo "$encrypted_secret" | \ - openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "::add-mask::$decrypted" - echo "DECRYPTED_SECRET=$decrypted" >> $GITHUB_OUTPUT + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - name: Push Results env: - GITHUB_TOKEN: ${{ steps.decrypt-secret.outputs.decrypted_secret }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} USER: mlcommons-bot EMAIL: mlcommons-bot@users.noreply.github.com if: github.repository_owner == 'mlcommons' && env.run_step == 'true' diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 663dff628..868ae873d 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -5,35 +5,8 @@ on: - cron: '0 0 * * 0' # Runs once a week on Sunday at 00:00 UTC workflow_dispatch: {} # Allows manual triggering of the workflow jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} - steps: - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Encrypt secret - id: encrypt-secret - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - run: | - # AES-256 encrypt - encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ - openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT - build: name: MLPerf inference MLCommons ResNet50 Closed Division - needs: [fetch-secret] runs-on: ${{ matrix.os }} env: MLC_INDEX: "on" @@ -115,25 +88,16 @@ jobs: Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append } - - name: Decrypt secret - id: decrypt-secret - shell: bash + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - encrypted_secret: ${{ needs.fetch-secret.outputs.encrypted_secret }} - run: | - echo "Running on OS: ${{ matrix.os }}" + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - # Decrypt - decrypted=$(echo "$encrypted_secret" | \ - openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "::add-mask::$decrypted" - echo "DECRYPTED_SECRET=$decrypted" >> $GITHUB_OUTPUT - name: Push Results env: - GITHUB_TOKEN: ${{ steps.decrypt-secret.outputs.decrypted_secret }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} if: github.repository_owner == 'mlcommons' && env.run_step == 'true' run: | git config --global user.name "mlcommons-bot" diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index a99acc106..708d27b32 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -9,34 +9,7 @@ on: - '!**.md' jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} - steps: - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Encrypt secret - id: encrypt-secret - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - run: | - # AES-256 encrypt - encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ - openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT - mlc-run-with-results-upload: - needs: [fetch-secret] runs-on: ${{ matrix.os }} env: MLC_INDEX: "on" @@ -109,25 +82,16 @@ jobs: Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append } - - name: Decrypt secret - id: decrypt-secret - shell: bash + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - encrypted_secret: ${{ needs.fetch-secret.outputs.encrypted_secret }} - run: | - echo "Running on OS: ${{ matrix.os }}" + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - # Decrypt - decrypted=$(echo "$encrypted_secret" | \ - openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "::add-mask::$decrypted" - echo "DECRYPTED_SECRET=$decrypted" >> $GITHUB_OUTPUT - name: Push Results env: - GITHUB_TOKEN: ${{ steps.decrypt-secret.outputs.decrypted_secret }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} if: github.repository_owner == 'mlcommons' && env.run_step == 'true' run: | git config --global user.name "mlcommons-bot" diff --git a/.github/workflows/test-mlperf-inference-tvm-resnet50.yml b/.github/workflows/test-mlperf-inference-tvm-resnet50.yml index 393d1c5cd..05411562e 100644 --- a/.github/workflows/test-mlperf-inference-tvm-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-tvm-resnet50.yml @@ -46,18 +46,16 @@ jobs: else echo "run_step=false" >> $GITHUB_ENV fi - - name: Load secret + - name: Load secrets if: github.repository_owner == 'mlcommons' && env.run_step == 'true' - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - name: Push Results env: - GITHUB_TOKEN: ${{ steps.op-load-secret.outputs.PAT }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} USER: mlcommons-bot EMAIL: mlcommons-bot@users.noreply.github.com if: github.repository_owner == 'mlcommons' && env.run_step == 'true' diff --git a/automation/script/doc.py b/automation/script/doc.py index 7d69685a0..620442dc4 100644 --- a/automation/script/doc.py +++ b/automation/script/doc.py @@ -43,7 +43,10 @@ def generate_doc(self_module, input_params): tag_values = input_params.get('tags', '').split(",") variation_tags = [tag[1:] for tag in tag_values if tag.startswith("_")] - # Step 4: Iterate over scripts and generate Dockerfile + # capture path of generated README's for each script + generated_readme_paths = [] + + # Step 4: Iterate over scripts and generate readme for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): metadata = script.meta script_directory = script.path @@ -60,8 +63,11 @@ def generate_doc(self_module, input_params): generic_inputs) if r['return'] > 0: continue + else: + generated_readme_paths.append( + os.path.join(script_directory, "README.md")) - return {'return': 0} + return {'return': 0, 'generated_readme_paths': generated_readme_paths} def get_setup_readme(script_repo): @@ -70,10 +76,12 @@ def get_setup_readme(script_repo): if '@' in repo_name: repo_name = repo_name.split('@')[1] - setup_readme = f"""`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do + repo_location = script_repo.meta.get('url', repo_alias) + + setup_readme = f"""`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -94,7 +102,7 @@ def get_setup_readme(script_repo): Once `mlcflow` is installed: ```bash -mlc pull repo {repo_alias} --pat= +mlc pull repo {repo_location} --pat= ``` - `--pat` or `--ssh` is only needed if the repo is PRIVATE - If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token diff --git a/automation/script/docker.py b/automation/script/docker.py index 0d33391e7..c97668ac3 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -103,7 +103,7 @@ def dockerfile(self_module, input_params): deps = docker_settings.get('build_deps', []) if deps: r = self_module._run_deps( - deps, [], env, {}, {}, {}, {}, '', [], '', False, '', + deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', show_time, ' ', run_state) if r['return'] > 0: return r @@ -336,7 +336,7 @@ def docker_run(self_module, i): deps = docker_settings.get('deps', []) if deps: r = self_module._run_deps( - deps, [], env, {}, {}, {}, {}, '', [], '', False, '', + deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', show_time, ' ', run_state) if r['return'] > 0: return r @@ -394,7 +394,7 @@ def docker_run(self_module, i): container_env_string = res['container_env_string'] res = update_docker_environment( - docker_settings, env, container_env_string) + docker_settings, env, self_module.host_env_keys, container_env_string) if res['return'] > 0: return res diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index cb733f93a..8fe73d47f 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -40,6 +40,10 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd, run_state): if 'mounts' in docker_settings: mounts.extend(docker_settings['mounts']) + for key in ["MLC_INPUT", "MLC_OUTPUT", "MLC_OUTDIRNAME"]: + if "${{ " + key + " }}:${{ " + key + " }}" not in mounts: + mounts.append("${{ " + key + " }}:${{ " + key + " }}") + docker_input_mapping = docker_settings.get("input_mapping", {}) container_env_string = "" @@ -182,7 +186,8 @@ def prepare_docker_inputs(input_params, docker_settings, return docker_inputs, dockerfile_path -def update_docker_environment(docker_settings, env, container_env_string): +def update_docker_environment( + docker_settings, env, host_env_keys, container_env_string): """ Updates the Docker environment variables and build arguments. @@ -194,15 +199,6 @@ def update_docker_environment(docker_settings, env, container_env_string): Returns: dict: A dictionary with a return code indicating success or failure. """ - # Define proxy-related environment variable keys to propagate - proxy_keys = [ - "ftp_proxy", "FTP_PROXY", - "http_proxy", "HTTP_PROXY", - "https_proxy", "HTTPS_PROXY", - "no_proxy", "NO_PROXY", - "socks_proxy", "SOCKS_PROXY", - "GH_TOKEN" - ] # Ensure the '+ CM_DOCKER_BUILD_ARGS' key exists in the environment if '+ MLC_DOCKER_BUILD_ARGS' not in env: @@ -210,7 +206,7 @@ def update_docker_environment(docker_settings, env, container_env_string): # Add proxy environment variables to Docker build arguments and container # environment string - for proxy_key in proxy_keys: + for proxy_key in host_env_keys: proxy_value = os.environ.get(proxy_key) if proxy_value: container_env_string += f" --env.{proxy_key}={proxy_value} " @@ -343,6 +339,8 @@ def rebuild_flags( keys = sorted(command_dict.keys(), key=lambda x: x != "tags") for key in keys: + if key in ["input", "output", "outdirname"]: + continue # We have the corresponding env keys in container env string # Construct the full key with the prefix. full_key = f"{prefix}.{key}" if prefix else key diff --git a/automation/script/help.py b/automation/script/help.py new file mode 100644 index 000000000..044052ce5 --- /dev/null +++ b/automation/script/help.py @@ -0,0 +1,257 @@ +import os +from mlc import utils +from utils import * +import os +from collections import defaultdict + + +def display_help(self_module, input_params): + """ + Generates the documentation of MLC scripts. + + Args: + self_module: Reference to the current module for internal calls. + input_params: Dictionary containing input parameters. + + Returns: + Dictionary with the result of the operation. Keys: + - 'return': 0 on success, >0 on error. + - 'error': Error message (if any). + """ + + # Extract and handle basic inputs + quiet = input_params.get('quiet', False) + logger = self_module.logger + env = input_params.get('env', {}) + generic_inputs = self_module.input_flags_converted_to_env + + # Step 2: Search for scripts + search_result = self_module.search(input_params.copy()) + if search_result['return'] > 0: + return search_result + + scripts_list = search_result['list'] + if not scripts_list: + + print("") + print("Please use script tags or alias/uid to get help for a specific script") + print("") + print("Generic Inputs for all Scripts:") + print("") + print_input_descriptions(generic_inputs) + + else: + # Step 4: Iterate over scripts and generate help output + for script in sorted( + scripts_list, key=lambda x: x.meta.get('alias', '')): + metadata = script.meta + script_path = script.path + print_script_help( + metadata, + script_path, + generic_inputs, + env, + self_module) + + return {'return': 0} + + +def print_script_help(metadata, script_path, generic_inputs, env, self_module): + print("") + print(f"Script Name: {metadata.get('alias', metadata['uid'])}") + print(f"Tags: {', '.join(metadata.get('tags', []))}") + # Print the run commands + print_run_commands(metadata) + print("") + print("Script Inputs:") + print("") + + input_mapping = metadata.get('input_mapping', {}) + input_description = metadata.get('input_description', {}) + default_env = metadata.get('default_env', {}) + + print_input_details(input_mapping, input_description, default_env) + + print("") + print("Generic Inputs for all Scripts:") + print("") + print_input_descriptions(generic_inputs) + + variations = metadata.get('variations', {}) + + if variations: + print_variations_help(variations) + else: + print(" - No variations.") + + print("\n" + "=" * 60 + "\n") # Separator for clarity + + print( + f"""Script meta file path: {os.path.join(script_path, "meta.yaml")}""") + customize_path = os.path.join(script_path, "customize.py") + if os.path.exists(customize_path): + print(f"""Script customize file path: {customize_path}""") + else: + print(f"""Script customize file can be created at: {customize_path}""") + + run_script_name = self_module._get_script_name(env, script_path) + run_script_path = os.path.join(script_path, run_script_name) + if os.path.exists(run_script_path): + print(f"""Script run file path: {run_script_path}""") + + print("\n" + "=" * 60 + "\n") # Separator for clarity + + +def print_input_details(input_mapping, input_description, default_env): + + for i in input_mapping: + if i not in input_description or input_description[i].get( + 'env', '') != input_mapping[i]: + if i not in input_description: + input_description[i] = {} + input_description[i]['env_key'] = input_mapping[i] + + keys_to_delete = [ + key for key in input_description if key not in input_mapping and "." not in key] + for key in keys_to_delete: + del input_description[key] + + if input_description: + reverse_map = defaultdict(list) + for k, v in input_mapping.items(): + reverse_map[v].append(k) + + for i in input_description: + if i in input_mapping and input_mapping[i] in default_env: + input_description[i]['default'] = default_env[input_mapping[i]] + + # Add alias entries + for mapped_env, keys in reverse_map.items(): + if len(keys) > 1: + canonical = keys[0] + for alias in keys[1:]: + if alias in input_description: + input_description[alias] = {} + input_description[alias]['alias'] = canonical + input_description[alias]['desc'] = f"""Alias for --{canonical}""" + input_description[alias]['env_key'] = mapped_env + + print_input_descriptions(input_description) + + return + + +def print_input_descriptions(input_descriptions): + + if not input_descriptions: + print("\tNo inputs") + + for key in input_descriptions: + field = input_descriptions[key] + env_key = field.get('env_key', f"""MLC_TMP_{key.upper()}""") + desc = field.get('desc') + default = field.get('default', 'None') + choices = field.get("choices", "") + dtype = infer_type(field) + # Use .ljust(15) to ensure the key occupies 15 characters minimum + print(f"\t--{key.ljust(26)}: maps to --env.{env_key}") + if desc: + print(f"\t{' '.ljust(30)}Desc: {desc}") + print(f"\t{' '.ljust(30)}Default: {default}") + if choices: + print(f"\t{' '.ljust(30)}Choices: {choices}") + if dtype: + print(f"\t{' '.ljust(30)}Type: {dtype}") + print("") + + +def print_variations_help(variations): + # Data structures + aliases = {} + alias_reverse = defaultdict(list) + bases = defaultdict(list) + variation_groups = {} + main_variations = {} + + # First pass: classify and build maps + for name, attrs in variations.items(): + if "," in name: + continue # Skip composite variations + if not isinstance(attrs, dict): + main_variations[name] = {} + continue + if "alias" in attrs: + aliases[name] = attrs["alias"] + alias_reverse[attrs["alias"]].append(name) + else: + main_variations[name] = attrs + # Group + group = attrs.get("group", "ungrouped") + if isinstance(group, list): + group = group[0] if group else "ungrouped" + variation_groups[name] = group + # Base + base = attrs.get("base", []) + if isinstance(base, str): + base = [base] + bases[name] = base + + # Build grouped output in a simpler format + grouped_output = defaultdict(list) + + for var in sorted(main_variations.keys()): + group = variation_groups.get(var, "ungrouped") + output = f"{var}" + + if var.endswith(".#"): + output += " (dynamic substitution allowed)" + + if alias_reverse.get(var): + alias_str = ", ".join(sorted(alias_reverse[var])) + output += f" [Alias: {alias_str}]" + + if bases.get(var): + base_str = ", ".join(bases[var]) + output += f" [base: {base_str}]" + + if group != "ungrouped" and main_variations[var].get("default", False): + output += " [Default]" + + grouped_output[group].append(output) + + # Console output structure + print("\nVariations:\n") + for group in sorted(grouped_output): + print(f"\t{group.capitalize()} Variations:") + for line in grouped_output[group]: + print(f"\t - {line}") + print("") # Blank line between groups + + +def print_run_commands(metadata): + tags = ','.join(metadata.get('tags', [])) + input_mapping = metadata.get('input_mapping', {}) + + # Build the command using the extracted tags + command = f"mlcr {tags}" + print("\nRun Commands:\n") + print(f" $ {command}") + + print("") + print(f""" + * Any input can be appended to the run command directly or using its --env.key mapping. + * --env.key is useful to modify the input of a dependent script for which direct input may not be there in the main script. + * Any variation can be selected by adding it to the tags using the _ prefix. + For example, mlcr get,generic-python-lib,_panda turns on the panda variation for the get-generic-python-lib script. + * --adr. can be used to modify the dependency(ies) with the name dep_name. + For example, --adr.compiler.tags=gcc adds the tag 'gcc' to any dependency with the name compiler. + """) + + +def infer_type(field): + if "dtype" in field: + return field["dtype"] + elif "default" in field: + return type(field["default"]).__name__ + else: + return "str" diff --git a/automation/script/lint.py b/automation/script/lint.py new file mode 100644 index 000000000..63825c22f --- /dev/null +++ b/automation/script/lint.py @@ -0,0 +1,160 @@ +import os +import yaml +import copy +from mlc import utils +from utils import * + + +def lint_meta(self_module, input_params): + """ + Lints MLC script metadata files by fixing key order and validating structure. + + Args: + self_module: Reference to the current module for internal calls. + i: Dictionary containing input parameters. + + Returns: + Dictionary with the result of the operation. Keys: + - 'return': 0 on success, >0 on error. + - 'error': Error message (if any). + """ + + # Extract and handle basic inputs + quiet = input_params.get('quiet', False) + logger = self_module.logger + env = input_params.get('env', {}) + generic_inputs = self_module.input_flags_converted_to_env + generic_inputs = dict(sorted(generic_inputs.items())) + + # Search for scripts + search_result = self_module.search(input_params.copy()) + if search_result['return'] > 0: + return search_result + + scripts_list = search_result['list'] + if not scripts_list: + return {'return': 1, 'error': 'No scripts were found'} + + env = input_params.get('env', {}) + state_data = input_params.get('state', {}) + constant_vars = input_params.get('const', {}) + constant_state = input_params.get('const_state', {}) + tag_values = input_params.get('tags', '').split(",") + variation_tags = [tag[1:] for tag in tag_values if tag.startswith("_")] + + # Iterate over scripts + for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): + metadata = script.meta + script_directory = script.path + script_tags = metadata.get("tags", []) + script_alias = metadata.get('alias', '') + script_uid = metadata.get('uid', '') + script_input_mapping = metadata.get('input_mapping', {}) + script_input_description = metadata.get('input_description', {}) + script_repo = script.repo + + # Sort YAML keys + sort_result = sort_meta_yaml_file(script_directory, quiet) + if sort_result['return'] > 0: + if not quiet: + logger.error( + f"Failed to sort YAML keys for {script_alias}: {sort_result.get('error', '')}") + elif sort_result.get('modified', False): + if not quiet: + logger.info(f"Sorted YAML keys for {script_alias}") + elif not sort_result.get('modified', False): + if not quiet: + logger.info( + f"No input mapping or variations keys to be sorted for {script_alias}") + + return {'return': 0} + + +def sort_meta_yaml_file(script_directory, quiet=False): + """ + Sort specific keys in the meta.yaml file and save it back to disk. + + Args: + script_directory: Path to the script directory + quiet: Whether to suppress output messages + + Returns: + Dictionary with 'return' (0 on success, >0 on error), 'modified' (bool), + and 'error' (if any) + """ + + try: + # Find meta.yaml file + meta_yaml_path = None + for filename in ['meta.yaml', 'meta.yml']: + potential_path = os.path.join(script_directory, filename) + if os.path.exists(potential_path): + meta_yaml_path = potential_path + break + + if not meta_yaml_path: + return {'return': 1, 'error': 'meta.yaml file not found', + 'modified': False} + + # Read current YAML content + with open(meta_yaml_path, 'r', encoding='utf-8') as file: + data = yaml.safe_load(file) + + if not isinstance(data, dict): + return { + 'return': 1, 'error': 'YAML does not contain a dictionary', 'modified': False} + + # Store original for comparison + original_data = copy.deepcopy(data) + + # Sort input_mapping alphabetically + if 'input_mapping' in data and isinstance(data['input_mapping'], dict): + data['input_mapping'] = dict(sorted(data['input_mapping'].items())) + + # Sort variations: with 'group' first, then without 'group' + if 'variations' in data and isinstance(data['variations'], dict): + variations = data['variations'] + + # Separate variations with and without 'group' key + with_group = [] + without_group = [] + + for key, value in variations.items(): + if isinstance(value, dict) and 'group' in value: + with_group.append((key, value)) + else: + without_group.append((key, value)) + + # Sort both lists alphabetically by key + with_group.sort(key=lambda x: x[0]) + without_group.sort(key=lambda x: x[0]) + + # Combine them: with_group first, then without_group + sorted_variations = {} + for key, value in with_group + without_group: + sorted_variations[key] = value + + data['variations'] = sorted_variations + + # Check if anything changed (including order) + original_yaml = yaml.dump( + original_data, + default_flow_style=False, + sort_keys=False) + new_yaml = yaml.dump(data, default_flow_style=False, sort_keys=False) + + if original_yaml == new_yaml: + return {'return': 0, 'modified': False} + + # Write the sorted YAML back to file + with open(meta_yaml_path, 'w', encoding='utf-8') as file: + yaml.dump(data, file, default_flow_style=False, sort_keys=False, + allow_unicode=True, width=1000, indent=2) + + if not quiet: + print(f"Sorted YAML keys in {meta_yaml_path}") + + return {'return': 0, 'modified': True, 'sorted_data': data} + + except Exception as e: + return {'return': 1, 'error': str(e), 'modified': False} diff --git a/automation/script/module.py b/automation/script/module.py index b1759c1c7..298a6e2bc 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -67,6 +67,19 @@ def __init__(self, action_object, automation_file): 'MLC_GIT_*', 'MLC_RENEW_CACHE_ENTRY'] + self.host_env_keys = [ + "GH_TOKEN", + "ftp_proxy", + "FTP_PROXY", + "http_proxy", + "HTTP_PROXY", + "https_proxy", + "HTTPS_PROXY", + "no_proxy", + "NO_PROXY", + "socks_proxy", + "SOCKS_PROXY"] + self.input_flags_converted_to_tmp_env = { 'path': {'desc': 'Filesystem path to search for executable', 'default': ''}} @@ -486,19 +499,7 @@ def _run(self, i): 'append_unique': True}) # take some env from the user environment - keys = [ - "GH_TOKEN", - "ftp_proxy", - "FTP_PROXY", - "http_proxy", - "HTTP_PROXY", - "https_proxy", - "HTTPS_PROXY", - "no_proxy", - "NO_PROXY", - "socks_proxy", - "SOCKS_PROXY"] - for key in keys: + for key in self.host_env_keys: if os.environ.get(key, '') != '' and env.get(key, '') == '': env[key] = os.environ[key] @@ -529,12 +530,6 @@ def _run(self, i): if i.get(key): ii[key] = i[key] - # if cm run script without tags/artifact and with --help - if len(ii.get('parsed_artifact', [])) == 0 and ii.get( - 'tags', '') == '' and i.get('help', False): - return utils.call_internal_module( - self, __file__, 'module_help', 'print_help', {'meta': {}, 'path': ''}) - r = self.search(ii) if r['return'] > 0: return r @@ -791,8 +786,7 @@ def _run(self, i): # Check if has --help if i.get('help', False): - return utils.call_internal_module(self, __file__, 'module_help', 'print_help', { - 'meta': meta, 'path': path}) + return self.help(i) run_state['script_id'] = meta['alias'] + "," + meta['uid'] run_state['script_tags'] = script_tags @@ -1391,6 +1385,7 @@ def _run(self, i): 'script_alias': meta['alias'], 'extra_tags': ",".join(extra_cache_tags), 'tags': ','.join(tmp_tags), + 'exact_tags_match': True, 'meta': cached_meta, 'force': True} @@ -1760,9 +1755,10 @@ def _run(self, i): env['MLC_TMP_CURRENT_PATH'], env['MLC_OUTDIRNAME']) env['MLC_OUTDIRNAME'] = c_outdirname - if not os.path.exists(c_outdirname): - os.makedirs(c_outdirname) - os.chdir(c_outdirname) + if not fake_run: # prevent permission error inside docker runs + if not os.path.exists(c_outdirname): + os.makedirs(c_outdirname) + os.chdir(c_outdirname) # Check if pre-process and detect if 'preprocess' in dir(customize_code) and not fake_run: @@ -2461,6 +2457,7 @@ def _update_state_from_variations(self, i, meta, variation_tags, variations, env variation_meta = variations[variation_tag] if variation_tag_dynamic_suffix: + variation_meta = copy.deepcopy(variation_meta) self._update_variation_meta_with_dynamic_suffix( variation_meta, variation_tag_dynamic_suffix) @@ -2573,7 +2570,63 @@ def _update_state_from_variations(self, i, meta, variation_tags, variations, env return {'return': 0, 'variation_tags_string': variation_tags_string, 'explicit_variation_tags': explicit_variation_tags, 'warnings': warnings} + def _add_base_variations( + self, + variation_name, + variations, + variation_tags, + tmp_variations, + excluded_variation_tags + ): + """ + Adds base variations for a given variation_name into variation_tags + and updates tmp_variations if valid. + """ + + if "base" not in variations[variation_name]: + return {'return': 0} + + for base_variation in variations[variation_name]["base"]: + tag_to_append = None + dynamic_base_variation = False + dynamic_base_variation_already_added = False + + # Handle dynamic variation + if base_variation not in variations: + base_variation_dynamic = self._get_name_for_dynamic_variation_tag( + base_variation) + if not base_variation_dynamic or base_variation_dynamic not in variations: + return { + 'return': 1, + 'error': f'Variation "{base_variation}" specified as base variation of "{variation_name}" is not existing' + } + dynamic_base_variation = True + base_prefix = base_variation_dynamic.split(".")[0] + "." + + # We allow repeated dynamic variations like _patch.1,_patch.2,_patch.3 + # for tag in variation_tags: + # if tag.startswith(base_prefix): + # dynamic_base_variation_already_added = True + # break + + # Append if not already added + if base_variation not in variation_tags and not dynamic_base_variation_already_added: + tag_to_append = base_variation + + # Validate exclusion list + if tag_to_append: + if tag_to_append in excluded_variation_tags: + return { + 'return': 1, + 'error': f'Variation "{tag_to_append}" specified as base variation for the variation is in the excluded list "{variation_name}"' + } + variation_tags.append(tag_to_append) + tmp_variations[tag_to_append] = False + + return {'return': 0} + ########################################################################## + def _update_variation_tags_from_variations( self, variation_tags, variations, variation_groups, excluded_variation_tags): @@ -2611,40 +2664,15 @@ def _update_variation_tags_from_variations( variation_name = self._get_name_for_dynamic_variation_tag( variation_name) - # TODO: Move this to a function and apply it for combination of variations too - # base variations are automatically turned on. Only - # variations outside of any variation group can be added as - # a base_variation - if "base" in variations[variation_name]: - base_variations = variations[variation_name]["base"] - for base_variation in base_variations: - dynamic_base_variation = False - dynamic_base_variation_already_added = False - if base_variation not in variations: - base_variation_dynamic = self._get_name_for_dynamic_variation_tag( - base_variation) - if not base_variation_dynamic or base_variation_dynamic not in variations: - return {'return': 1, 'error': 'Variation "{}" specified as base variation of "{}" is not existing'.format( - base_variation, variation_name)} - else: - dynamic_base_variation = True - base_prefix = base_variation_dynamic.split(".")[ - 0] + "." - for x in variation_tags: - if x.startswith(base_prefix): - dynamic_base_variation_already_added = True - - if base_variation not in variation_tags and not dynamic_base_variation_already_added: - tag_to_append = base_variation - - if tag_to_append: - if tag_to_append in excluded_variation_tags: - return {'return': 1, 'error': 'Variation "{}" specified as base variation for the variation is in the excluded list "{}" '.format( - tag_to_append, variation_name)} - variation_tags.append(tag_to_append) - tmp_variations[tag_to_append] = False - - tag_to_append = None + result = self._add_base_variations( + variation_name, + variations, + variation_tags, + tmp_variations, + excluded_variation_tags + ) + if result.get('return', 0) > 0: + return result # default_variations dictionary specifies the # default_variation for each variation group. A default @@ -2676,6 +2704,16 @@ def _update_variation_tags_from_variations( combined_variation_meta = variations[combined_variation] tmp_combined_variations[combined_variation] = True + result = self._add_base_variations( + combined_variation, + variations, + variation_tags, + tmp_combined_variations, + excluded_variation_tags + ) + if result.get('return', 0) > 0: + return result + r = self._get_variation_tags_from_default_variations( combined_variation_meta, variations, @@ -2832,6 +2870,8 @@ def search(self, i): if i.get(key): ii[key] = i[key] + if i.get('all'): + ii['all'] = i['all'] r = super(ScriptAutomation, self).search(ii) if r['return'] > 0: return r @@ -3363,9 +3403,18 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a for t in update_tags_from_env_with_prefix: for key in update_tags_from_env_with_prefix[t]: if str(d.get('env', {}).get(key, '')).strip() != '': - d['tags'] += "," + t + str(d.get('env')[key]) + if isinstance( + d.get('env')[key], (str, int, float)): + d['tags'] += "," + t + str(d.get('env')[key]) + elif isinstance(d.get('env')[key], list): + for item in d.get('env')[key]: + d['tags'] += "," + t + str(item) elif str(env.get(key, '')).strip() != '': - d['tags'] += "," + t + str(env[key]) + if isinstance(env[key], (str, int, float)): + d['tags'] += "," + t + str(env[key]) + elif isinstance(env[key], list): + for item in env[key]: + d['tags'] += "," + t + str(item) for key in clean_env_keys_deps: if '?' in key or '*' in key: @@ -3404,8 +3453,18 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a update_tags_from_env = d.get("update_tags_from_env", []) for t in update_tags_from_env: - if env.get(t, '').strip() != '': - d['tags'] += "," + env[t] + if str(d.get('env', {}).get(t, '')).strip() != '': + if isinstance(d.get('env')[t], (str, int, float)): + d['tags'] += "," + str(d.get('env')[t]) + elif isinstance(d.get('env')[t], list): + for item in d.get('env')[t]: + d['tags'] += "," + str(item) + elif str(env.get(t, '')).strip() != '': + if isinstance(env[t], (str, int, float)): + d['tags'] += "," + str(env[t]) + elif isinstance(env[t], list): + for item in env[t]: + d['tags'] += "," + str(item) update_tags_if_env = d.get("update_tags_if_env", []) for t in update_tags_if_env: @@ -4470,6 +4529,42 @@ def doc(self, i): ############################################################ + def help(self, i): + """ + Document MLC script. + + Args: + (MLC input dict): + + (out) (str): if 'con', output to console + + parsed_artifact (list): prepared in MLC CLI or MLC access function + [ (artifact alias, artifact UID) ] or + [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] + + (repos) (str): list of repositories to search for automations + + (output_dir) (str): output directory (../docs by default) + + Returns: + (MLC return dict): + + * return (int): return code == 0 if no error and >0 if error + * (error) (str): error string if return>0 + + """ + + from script.help import display_help + return display_help(self, i) + + ############################################################ + + def lint(self, i): + from script.lint import lint_meta + return lint_meta(self, i) + + ############################################################ + def dockerfile(self, i): from script.docker import dockerfile return dockerfile(self, i) @@ -4540,6 +4635,9 @@ def clean_some_tmp_files(self, i): return {'return': 0} + def _get_script_name(self, env, path, filename="run"): + return get_script_name(env, path, filename) + def get_version_tag_from_version(version, cached_tags): tags_to_add = [] @@ -4775,10 +4873,28 @@ def find_cached_script(i): 'customize_code': customize_code, 'customize_common_input': customize_common_input } + env_tmp = copy.deepcopy(env) + path_to_cached_state_file = os.path.join(cached_script.path, + self_obj.file_with_cached_state) + + r = utils.load_json(file_name=path_to_cached_state_file) + if r['return'] > 0: + continue + + cached_meta = r.get("meta") + if not cached_meta: + continue + new_env = cached_meta.get("new_env", {}) + if new_env: + env_tmp.update(new_env) + state_tmp = copy.deepcopy(state) + new_state = cached_meta.get("new_state", {}) + if new_state: + state_tmp.update(new_state) deps = meta.get('deps') if deps: - r = self_obj._call_run_deps(deps, self_obj.local_env_keys, meta.get('local_env_keys', []), env, state, const, const_state, add_deps_recursive, + r = self_obj._call_run_deps(deps, self_obj.local_env_keys, meta.get('local_env_keys', []), env_tmp, state_tmp, const, const_state, add_deps_recursive, recursion_spaces + extra_recursion_spaces, remembered_selections, variation_tags_string, True, '', show_time, extra_recursion_spaces, {}) if r['return'] > 0: @@ -4786,7 +4902,7 @@ def find_cached_script(i): ii = { 'run_script_input': run_script_input, - 'env': env, + 'env': env_tmp, 'script_name': 'validate_cache', 'detect_version': True } diff --git a/automation/utils.py b/automation/utils.py index 98a3a3718..5881c148b 100644 --- a/automation/utils.py +++ b/automation/utils.py @@ -14,12 +14,12 @@ def get_host_os_info(i={}): Get some host platform name (currently windows or linux) and OS bits Args: - (CM input dict): + (input dict): (bits) (str): force host platform bits Returns: - (CM return dict): + (dict): * return (int): return code == 0 if no error and >0 if error * (error) (str): error string if return>0 @@ -41,7 +41,7 @@ def get_host_os_info(i={}): pbits = str(8 * struct.calcsize("P")) if platform.system().lower().startswith('win'): - platform = 'windows' + platform_name = 'windows' info['bat_ext'] = '.bat' info['set_env'] = 'set ${key}=${value}' info['env_separator'] = ';' @@ -56,9 +56,9 @@ def get_host_os_info(i={}): } else: if platform.system().lower().startswith('darwin'): - platform = 'darwin' + platform_name = 'darwin' else: - platform = 'linux' + platform_name = 'linux' info['bat_ext'] = '.sh' info['set_env'] = 'export ${key}="${value}"' @@ -72,12 +72,14 @@ def get_host_os_info(i={}): info['start_script'] = ['#!/bin/bash', ''] info['env'] = {} - info['platform'] = platform + info['platform'] = platform_name + info['env']['MLC_HOST_PLATFORM_FLAVOR'] = platform.machine() + info['env']['MLC_HOST_OS_TYPE'] = platform_name obits = i.get('bits', '') if obits == '': obits = '32' - if platform == 'windows': + if platform_name == 'windows': # Trying to get fast way to detect bits if os.environ.get('ProgramW6432', '') != '' or os.environ.get( 'ProgramFiles(x86)', '') != '': # pragma: no cover diff --git a/mkdocs.yml b/mkdocs.yml index ebd48a20a..c9c12d71d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,6 @@ site_name: MLPerf Automation Documentation repo_url: https://github.com/mlcommons/mlperf-automations +site_url: https://docs.mlcommons.org/mlperf-automations theme: name: material logo: img/logo_v2.svg diff --git a/script/add-custom-nvidia-system/meta.yaml b/script/add-custom-nvidia-system/meta.yaml index 94285d986..01d6b8674 100644 --- a/script/add-custom-nvidia-system/meta.yaml +++ b/script/add-custom-nvidia-system/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: add-custom-nvidia-system uid: b2e6c46c6e8745a3 cache: true @@ -9,7 +9,7 @@ category: "MLPerf benchmark support" docker: real_run: False -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - add - custom @@ -17,7 +17,7 @@ tags: - nvidia -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -28,7 +28,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect python3 - tags: get,python3 diff --git a/script/app-image-classification-onnx-py/customize.py b/script/app-image-classification-onnx-py/customize.py index 39ae86bba..c27248f7d 100644 --- a/script/app-image-classification-onnx-py/customize.py +++ b/script/app-image-classification-onnx-py/customize.py @@ -46,8 +46,8 @@ def postprocess(i): with open(fjson, 'w', encoding='utf-8') as f: json.dump(data, f, ensure_ascii=False, indent=4) except Exception as e: - logger.warning('CM warning: {}'.format(e)) - logger.warning('CM warning: {}'.format(e)) + logger.warning('MLC warning: {}'.format(e)) + logger.warning('MLC warning: {}'.format(e)) try: import yaml diff --git a/script/app-image-classification-onnx-py/run.sh b/script/app-image-classification-onnx-py/run.sh index 4325faf5a..f88018739 100644 --- a/script/app-image-classification-onnx-py/run.sh +++ b/script/app-image-classification-onnx-py/run.sh @@ -11,7 +11,7 @@ fi MLC_PYTHON_BIN=${MLC_PYTHON_BIN_WITH_PATH:-python3} MLC_TMP_CURRENT_SCRIPT_PATH=${MLC_TMP_CURRENT_SCRIPT_PATH:-$PWD} -# connect CM intelligent components with CK env +# connect MLC intelligent components with CK env export CK_ENV_ONNX_MODEL_ONNX_FILEPATH=${MLC_ML_MODEL_FILE_WITH_PATH} export CK_ENV_ONNX_MODEL_INPUT_LAYER_NAME="input_tensor:0" export CK_ENV_ONNX_MODEL_OUTPUT_LAYER_NAME="softmax_tensor:0" @@ -33,5 +33,5 @@ echo "" ${MLC_PYTHON_BIN} ${MLC_TMP_CURRENT_SCRIPT_PATH}/src/onnx_classify.py test $? -eq 0 || exit 1 -# Just a demo to pass environment variables from native scripts back to CM workflows +# Just a demo to pass environment variables from native scripts back to MLC workflows echo "MLC_APP_IMAGE_CLASSIFICATION_ONNX_PY=sucess" > tmp-run-env.out diff --git a/script/app-image-classification-tf-onnx-cpp/meta.yaml b/script/app-image-classification-tf-onnx-cpp/meta.yaml index 957a0d28f..b6afbd6cc 100644 --- a/script/app-image-classification-tf-onnx-cpp/meta.yaml +++ b/script/app-image-classification-tf-onnx-cpp/meta.yaml @@ -7,7 +7,7 @@ default_env: MLC_BATCH_SIZE: '1' deps: - tags: detect,os -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - tags: get,gcc - tags: get,dataset,image-classification,original - tags: get,dataset-aux,image-classification diff --git a/script/app-image-classification-torch-py/run.sh b/script/app-image-classification-torch-py/run.sh index 478332299..f5ce00034 100644 --- a/script/app-image-classification-torch-py/run.sh +++ b/script/app-image-classification-torch-py/run.sh @@ -2,7 +2,7 @@ MLC_TMP_CURRENT_SCRIPT_PATH=${MLC_TMP_CURRENT_SCRIPT_PATH:-$PWD} -# connect CM intelligent components with CK env +# connect MLC intelligent components with CK env export MLC_ML_TORCH_MODEL_NAME=resnet50 export MLC_ML_MODEL_INPUT_DATA_TYPE=float32 export MLC_ML_MODEL_IMAGE_HEIGHT=224 diff --git a/script/app-loadgen-generic-python/meta.yaml b/script/app-loadgen-generic-python/meta.yaml index ba5a3c616..16f564a59 100644 --- a/script/app-loadgen-generic-python/meta.yaml +++ b/script/app-loadgen-generic-python/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-loadgen-generic-python uid: d3d949cc361747a6 @@ -10,7 +10,7 @@ category: "Modular MLPerf inference benchmark pipeline" developers: "[Gaz Iqbal](https://www.linkedin.com/in/gaziqbal), [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Grigori Fursin](https://cKnowledge.org/gfursin)" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - loadgen @@ -49,7 +49,7 @@ input_mapping: new_env_keys: - MLC_MLPERF_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -82,7 +82,7 @@ deps: - loadgen ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC # ONNX - enable_if_env: MLC_MLPERF_BACKEND: @@ -110,7 +110,7 @@ deps: - onnx ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC # PyTorch # CPU @@ -175,7 +175,7 @@ deps: -# Customize this CM script +# Customize this MLC script variations: pytorch: diff --git a/script/app-mlperf-automotive-mlcommons-python/meta.yaml b/script/app-mlperf-automotive-mlcommons-python/meta.yaml index c574cc7c0..1a3d8d6d2 100644 --- a/script/app-mlperf-automotive-mlcommons-python/meta.yaml +++ b/script/app-mlperf-automotive-mlcommons-python/meta.yaml @@ -7,7 +7,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf inference benchmark pipeline for ABTF model" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - automotive - mlcommons @@ -65,7 +65,7 @@ input_mapping: multistream_target_latency: MLC_MLPERF_LOADGEN_MULTISTREAM_TARGET_LATENCY output: MLC_MLPERF_OUTPUT_DIR -# Duplicate CM environment variables to the ones used in native apps +# Duplicate MLC environment variables to the ones used in native apps env_key_mappings: MLC_HOST_: HOST_ MLC_ML_: ML_ @@ -87,7 +87,7 @@ new_state_keys: - mlperf-inference-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features @@ -97,7 +97,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python @@ -105,7 +105,7 @@ deps: - python - python3 - # Use cmind inside CM scripts + # Use cmind inside MLC scripts - tags: get,generic-python-lib,_package.cmind @@ -120,7 +120,7 @@ deps: ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC ## Onnx CPU Runtime - tags: get,generic-python-lib,_onnxruntime diff --git a/script/app-mlperf-automotive/customize.py b/script/app-mlperf-automotive/customize.py index 32f5b3416..45cf6da18 100644 --- a/script/app-mlperf-automotive/customize.py +++ b/script/app-mlperf-automotive/customize.py @@ -283,7 +283,7 @@ def postprocess(i): if env.get('MLC_HOST_SYSTEM_NAME', '') != '': host_info['system_name'] = env['MLC_HOST_SYSTEM_NAME'] - # Check CM automation repository + # Check MLC automation repository repo_name = 'mlcommons@mlperf-automations' repo_hash = '' r = mlc.access({'action': 'find', 'automation': 'repo', @@ -293,9 +293,6 @@ def postprocess(i): if os.path.isdir(repo_path): repo_name = os.path.basename(repo_path) - # Check dev - # if repo_name == 'cm4mlops': repo_name = 'mlcommons@cm4mlops' - r = utils.run_system_cmd({ 'path': repo_path, 'cmd': 'git rev-parse HEAD'}) diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index ee4af5c5b..d5a1bfedc 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -7,7 +7,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf automotive benchmark pipeline for ABTF models" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - app-mlperf-inference @@ -72,7 +72,7 @@ new_env_keys: new_state_keys: - mlc-mlperf-inference-results* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features @@ -82,7 +82,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python @@ -396,7 +396,7 @@ variations: tags: _cuda docker: all_gpus: 'yes' - base_image: nvcr.io/nvidia/pytorch:24.03-py3 + base_image: nvcr.io/nvidia/pytorch:24.08-py3 os_version: 22.04 v0.5: {} diff --git a/script/app-mlperf-inference-amd/meta.yaml b/script/app-mlperf-inference-amd/meta.yaml index 082408f87..e3f9c8789 100644 --- a/script/app-mlperf-inference-amd/meta.yaml +++ b/script/app-mlperf-inference-amd/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-amd uid: 467cdb20aabc4394 cache: false @@ -9,7 +9,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -68,7 +68,7 @@ new_env_keys: - MLC_SQUAD_ACCURACY_DTYPE -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -79,7 +79,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc ######################################################################## diff --git a/script/app-mlperf-inference-ctuning-cpp-tflite/meta.yaml b/script/app-mlperf-inference-ctuning-cpp-tflite/meta.yaml index 68e482375..8a7446499 100644 --- a/script/app-mlperf-inference-ctuning-cpp-tflite/meta.yaml +++ b/script/app-mlperf-inference-ctuning-cpp-tflite/meta.yaml @@ -21,7 +21,7 @@ default_env: deps: - tags: detect,os - tags: detect,cpu -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - enable_if_env: MLC_MLPERF_DEVICE: - gpu diff --git a/script/app-mlperf-inference-dummy/meta.yaml b/script/app-mlperf-inference-dummy/meta.yaml index b671994b7..28ce530aa 100644 --- a/script/app-mlperf-inference-dummy/meta.yaml +++ b/script/app-mlperf-inference-dummy/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-dummy uid: 5b71627383a94576 cache: false @@ -9,7 +9,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -68,7 +68,7 @@ new_env_keys: - MLC_SQUAD_ACCURACY_DTYPE -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -79,7 +79,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc ######################################################################## diff --git a/script/app-mlperf-inference-intel/meta.yaml b/script/app-mlperf-inference-intel/meta.yaml index 823ae9343..aba29ceef 100644 --- a/script/app-mlperf-inference-intel/meta.yaml +++ b/script/app-mlperf-inference-intel/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-intel uid: c05a90433bb04cc1 cache: false @@ -10,7 +10,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -68,7 +68,7 @@ new_state_keys: -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -79,7 +79,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Get MLPerf logging library diff --git a/script/app-mlperf-inference-mlcommons-cpp/meta.yaml b/script/app-mlperf-inference-mlcommons-cpp/meta.yaml index 0763a13e8..fa1a5a692 100644 --- a/script/app-mlperf-inference-mlcommons-cpp/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-cpp/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-mlcommons-cpp uid: bf62405e6c7a44bf @@ -9,7 +9,7 @@ category: "Modular MLPerf inference benchmark pipeline" developers: "[Thomas Zhu](https://www.linkedin.com/in/hanwen-zhu-483614189), [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Grigori Fursin](https://cKnowledge.org/gfursin)" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - mlcommons @@ -48,7 +48,7 @@ new_state_keys: - mlperf-inference-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -59,7 +59,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect CUDA if required - tags: get,cuda,_cudnn @@ -81,7 +81,7 @@ deps: - inference-src ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC - enable_if_env: MLC_MLPERF_BACKEND: - onnxruntime diff --git a/script/app-mlperf-inference-mlcommons-python/README.md b/script/app-mlperf-inference-mlcommons-python/README.md index adefcc231..58bf4738e 100644 --- a/script/app-mlperf-inference-mlcommons-python/README.md +++ b/script/app-mlperf-inference-mlcommons-python/README.md @@ -1,10 +1,10 @@ # README for app-mlperf-inference-mlcommons-python This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -136,7 +136,7 @@ mlcr app,vision,language,mlcommons,mlperf,inference,reference,ref - `llama2-70b-99` (base: llama2-70b_) - `llama2-70b-99.9` (base: llama2-70b_) - `llama3_1-405b` -- `llama3_1-8b` +- `llama3_1-8b_` - `mixtral-8x7b` - `pointpainting` - `resnet50` (default) @@ -166,6 +166,8 @@ mlcr app,vision,language,mlcommons,mlperf,inference,reference,ref - `dlrm-v2_` - `gptj_` - `llama2-70b_` +- `llama3_1-8b` (base: llama3_1-8b_) +- `llama3_1-8b-edge` (base: llama3_1-8b_) - `multistream` - `offline` - `r2.1_default` diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index c048e8f3c..ae19507ad 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -537,7 +537,7 @@ def get_run_cmd_reference( --tensor-parallel-size {env['MLC_MLPERF_INFERENCE_TP_SIZE']} \ {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \ {scenario_extra_options} {mode_extra_options} \ - --vllm""" + --vllm --lg-model-name {env['MLC_MODEL']}""" if env.get('MLC_MLPERF_INFERENCE_NUM_WORKERS', '') != '': cmd += f" --num-workers {env['MLC_MLPERF_INFERENCE_NUM_WORKERS']}" @@ -580,7 +580,7 @@ def get_run_cmd_reference( --dataset_dir {x}{env['MLC_DATASET_WHISPER_PATH']}{x} \ --manifest {x}{os.path.join(env['MLC_DATASET_WHISPER_PATH'], "data", "dev-all-repack.json")}{x} \ --log_dir {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \ - --model-path {x}{env['MLC_ML_MODEL_WHISPER_PATH']}{x} \ + --model_path {x}{env['MLC_ML_MODEL_WHISPER_PATH']}{x} \ {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \ {scenario_extra_options} {mode_extra_options}""" @@ -609,7 +609,7 @@ def get_run_cmd_reference( if env.get('MLC_MLPERF_POINTPAINTING_TIME', '') != '': cmd += f" --time {env['MLC_MLPERF_POINTPAINTING_TIME']}" - logger.info(fcmd) + logger.info(cmd) elif "deepseek-r1" in env['MLC_MODEL']: env['RUN_DIR'] = os.path.join( diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 68980d98d..3fdff095d 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-mlcommons-python uid: ff149e9781fc4b65 @@ -9,7 +9,7 @@ category: "Modular MLPerf inference benchmark pipeline" developers: "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Thomas Zhu](https://www.linkedin.com/in/hanwen-zhu-483614189), [Grigori Fursin](https://cKnowledge.org/gfursin)" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - vision @@ -70,7 +70,7 @@ input_mapping: deeplab_resnet50_path: MLC_ML_MODEL_DPLAB_RESNET50_PATH waymo_path: MLC_DATASET_WAYMO_PATH -# Duplicate CM environment variables to the ones used in native apps +# Duplicate MLC environment variables to the ones used in native apps env_key_mappings: MLC_HOST_: HOST_ MLC_ML_: ML_ @@ -89,7 +89,7 @@ new_state_keys: - mlperf-inference-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features - tags: detect,os @@ -98,7 +98,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python @@ -126,7 +126,7 @@ deps: - tensorrt ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC ## Onnx CPU Runtime - tags: get,generic-python-lib,_onnxruntime @@ -211,6 +211,8 @@ deps: - cpu - tags: install-vllm-from-src,_cpu + names: + - vllm-from-src enable_if_env: MLC_MLPERF_BACKEND: - vllm @@ -524,12 +526,13 @@ deps: - "yes" ## LLAMA3_1-8B - - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct + - tags: get,ml-model,llama3,_8b names: - llama3-8b-model enable_if_env: MLC_MODEL: - llama3_1-8b + - llama3_1-8b-edge skip_if_env: MLC_USE_MODEL_FROM_HOST: - "yes" @@ -1319,8 +1322,8 @@ variations: - tags: get,generic-python-lib,_protobuf names: - protobuf - version_max: "4.23.4" - version_max_usable: "4.23.4" + version_max1: "4.23.4" + version_max_usable1: "4.23.4" version_min: "3.20.3" enable_if_env: MLC_MLPERF_BACKEND: @@ -1447,10 +1450,9 @@ variations: env: MLC_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<>>+cpu.html" - llama3_1-8b: - group: models - env: - MLC_MODEL: llama3_1-8b + llama3_1-8b_: + group: + models deps: - tags: get,generic-python-lib,_package.transformers - tags: get,generic-python-lib,_package.sentencepiece @@ -1467,65 +1469,71 @@ variations: names: - llama3_1-8b - llama3-8b - ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_rclone + + llama3_1-8b: + base: + - llama3_1-8b_ + env: + MLC_MODEL: llama3_1-8b + deps: + ## CNNDM for Llama3 8B model - datacenter + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3 names: - - cnndm-llama3-edge - enable_if_env: - MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: - - edge + - cnndm-llama3-datacenter skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" - ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_rclone + + llama3_1-8b-edge: + base: + - llama3_1-8b_ + env: + MLC_MODEL: llama3_1-8b-edge + deps: + ## CNNDM for Llama3 8B model - edge + - tags: get,dataset,cnndm,_validation,_edge,_llama3 names: - - cnndm-llama3-datacenter - enable_if_env: - MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: - - datacenter + - cnndm-llama3-edge skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" - + whisper: group: models env: MLC_MODEL: whisper + adr: + vllm-from-src: + tags: _sha.b124e1085b1bf977e3dac96d99ffd9d8ddfdb6cc deps: - tags: get,generic-python-lib,_package.py-libnuma - tags: get,generic-sys-util,_libnuma-dev - tags: get,generic-python-lib,_package.setuptools-scm - tags: get,generic-python-lib,_package.librosa - tags: get,generic-python-lib,_package.transformers - version_max: "4.52.4" - tags: get,generic-python-lib,_package.openai-whisper # - tags: get,generic-python-lib,_package.evaluate # - tags: get,generic-python-lib,_package.absl-py # - tags: get,generic-python-lib,_package.rouge-score - - tags: get,ml-model,whisper,_rclone,_mlc + - tags: get,ml-model,whisper,_r2-downloader,_mlc skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" names: - whisper-model - - tags: get,dataset,whisper,_preprocessed,_mlc,_rclone + - tags: get,dataset,whisper,_preprocessed names: - whisper-dataset skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" - whisper,cpu: - deps: - - tags: install-vllm-from-src,_cpu - deepseek-r1: group: models env: MLC_MODEL: deepseek-r1 deps: - - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_rclone + - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2-downloader names: - deepseek-r1-preprocessed-dataset skip_if_env: diff --git a/script/app-mlperf-inference-nvidia/README.md b/script/app-mlperf-inference-nvidia/README.md index 9075672d8..261cf91ca 100644 --- a/script/app-mlperf-inference-nvidia/README.md +++ b/script/app-mlperf-inference-nvidia/README.md @@ -1,13 +1,96 @@ # README for app-mlperf-inference-nvidia -This README is automatically generated. Add custom content in [info.txt](info.txt). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. ## Run Commands ```bash mlcr reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia ``` -No script specific inputs +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--count` | | | `` | +| `--max_batchsize` | | | `` | +| `--mlperf_conf` | | | `` | +| `--mode` | | | `performance` | +| `--output_dir` | | | `` | +| `--scenario` | | | `Offline` | +| `--user_conf` | | | `` | +| `--devices` | | | `` | +| `--skip_preprocess` | | | `no` | +| `--skip_preprocessing` | Alias for skip_preprocess | | `` | +| `--target_qps` | | | `` | +| `--offline_target_qps` | | | `` | +| `--server_target_qps` | | | `` | +| `--target_latency` | | | `` | +| `--singlestream_target_latency` | | | `` | +| `--multistream_target_latency` | | | `` | +| `--use_triton` | | | `` | +| `--gpu_copy_streams` | | | `` | +| `--gpu_inference_streams` | | | `` | +| `--gpu_batch_size` | | | `` | +| `--dla_copy_streams` | | | `` | +| `--dla_inference_streams` | | | `` | +| `--dla_batch_size` | | | `` | +| `--input_format` | | | `` | +| `--performance_sample_count` | | | `` | +| `--workspace_size` | | | `` | +| `--log_dir` | | | `` | +| `--use_graphs` | | | `` | +| `--run_infer_on_copy_streams` | | | `` | +| `--start_from_device` | | | `` | +| `--end_on_device` | | | `` | +| `--max_dlas` | | | `` | +| `--power_setting` | | | `` | +| `--make_cmd` | | | `` | +| `--rerun` | | | `` | +| `--extra_run_options` | | | `` | +| `--use_deque_limit` | | | `` | +| `--deque_timeout_usec` | | | `` | +| `--use_cuda_thread_per_device` | | | `` | +| `--num_warmups` | | | `` | +| `--graphs_max_seqlen` | | | `` | +| `--num_issue_query_threads` | | | `` | +| `--soft_drop` | | | `` | +| `--use_small_tile_gemm_plugin` | | | `` | +| `--audio_buffer_num_lines` | | | `` | +| `--use_fp8` | | | `` | +| `--enable_sort` | | | `` | +| `--num_sort_segments` | | | `` | +| `--skip_postprocess` | | | `` | +| `--embedding_weights_on_gpu_part` | | | `` | +| `--sdxl_batcher_time_limit` | | | `` | ### Generic Script Inputs | Name | Description | Choices | Default | @@ -26,3 +109,128 @@ No script specific inputs | `--gh_token` | Github Token | | `` | | `--hf_token` | Huggingface Token | | `` | | `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Backend + +- `tensorrt` (default) + +### Batch-size + +- `batch_size.#` _(# can be substituted dynamically)_ + +### Batchsize-format-change + +- `pre5.0` +- `v5.0+` + +### Build-engine-options + +- `build_engine_options.#` _(# can be substituted dynamically)_ + +### Device + +- `cpu` +- `cuda` (default) + +### Device-memory + +- `gpu_memory.#` _(# can be substituted dynamically)_ +- `gpu_memory.16` +- `gpu_memory.24` +- `gpu_memory.32` +- `gpu_memory.40` +- `gpu_memory.48` +- `gpu_memory.8` +- `gpu_memory.80` + +### Dla-batch-size + +- `dla_batch_size.#` _(# can be substituted dynamically)_ + +### Gpu-connection + +- `pcie` +- `sxm` + +### Gpu-name + +- `a100` +- `a6000` +- `custom` +- `l4` +- `orin` +- `rtx_4090` +- `rtx_6000_ada` +- `t4` + +### Graphs + +- `use-graphs` + +### Loadgen-scenario + +- `multistream` +- `offline` +- `server` +- `singlestream` + +### Model + +- `3d-unet-99` (base: 3d-unet_) +- `3d-unet-99.9` (base: 3d-unet_) +- `bert-99` (base: bert_) +- `bert-99.9` (base: bert_) +- `dlrm-v2-99` (base: dlrm_) +- `dlrm-v2-99.9` (base: dlrm_) +- `gptj-99` (base: gptj_) +- `gptj-99.9` (base: gptj_) +- `llama2-70b-99` (base: llama2-70b_) +- `llama2-70b-99.9` (base: llama2-70b_) +- `resnet50` (default) +- `retinanet` +- `rnnt` +- `sdxl` + +### Num-gpus + +- `num-gpus.#` _(# can be substituted dynamically)_ +- `num-gpus.1` (default) + +### Power-mode + +- `maxn` +- `maxq` + +### Run-mode + +- `build` +- `build_engine` (alias: build-engine) +- `calibrate` +- `download_model` +- `prebuild` +- `preprocess_data` (alias: preprocess-data) +- `run_harness` (default) + +### Triton + +- `use_triton` + +### Ungrouped + +- `3d-unet_` +- `bert_` +- `default_variations` +- `dlrm_` +- `env` +- `gptj_` +- `llama2-70b_` +- `run-harness` +- `v3.1` (base: pre5.0) + +### Version + +- `v4.0` (base: pre5.0) +- `v4.1` (base: pre5.0) +- `v4.1-dev` (base: pre5.0) (default) +- `v5.0` (base: v5.0+) diff --git a/script/app-mlperf-inference-nvidia/meta.yaml b/script/app-mlperf-inference-nvidia/meta.yaml index b152945ed..6dda8d4a5 100644 --- a/script/app-mlperf-inference-nvidia/meta.yaml +++ b/script/app-mlperf-inference-nvidia/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-nvidia uid: bc3b17fb430f4732 cache: false @@ -10,7 +10,7 @@ automation_uid: 5b4e0237da074764 category: "Reproduce MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -92,7 +92,7 @@ input_mapping: embedding_weights_on_gpu_part: MLC_MLPERF_NVIDIA_HARNESS_EMBEDDING_WEIGHTS_ON_GPU_PART sdxl_batcher_time_limit: MLC_MLPERF_NVIDIA_HARNESS_SDXL_SERVER_BATCHER_TIME_LIMIT -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -103,7 +103,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Get Nvidia scratch space where data and models get downloaded - tags: get,mlperf,inference,nvidia,scratch,space diff --git a/script/app-mlperf-inference-qualcomm/meta.yaml b/script/app-mlperf-inference-qualcomm/meta.yaml index 246d89098..b4309881c 100644 --- a/script/app-mlperf-inference-qualcomm/meta.yaml +++ b/script/app-mlperf-inference-qualcomm/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-qualcomm uid: eef1aca5d7c0470e cache: false @@ -10,7 +10,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -79,7 +79,7 @@ new_env_keys: - MLC_SQUAD_ACCURACY_DTYPE -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -90,7 +90,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc - tags: get,git,repo names: @@ -226,7 +226,7 @@ deps: ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC - enable_if_env: MLC_MLPERF_BACKEND: - onnxruntime diff --git a/script/app-mlperf-inference-redhat/meta.yaml b/script/app-mlperf-inference-redhat/meta.yaml index cec71335e..452e8094e 100644 --- a/script/app-mlperf-inference-redhat/meta.yaml +++ b/script/app-mlperf-inference-redhat/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-redhat uid: 82c9bb3c222447ca cache: false @@ -9,7 +9,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -68,7 +68,7 @@ new_env_keys: - MLC_SQUAD_ACCURACY_DTYPE -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -79,7 +79,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc ######################################################################## diff --git a/script/app-mlperf-inference/README.md b/script/app-mlperf-inference/README.md index 6a5c63c29..6f8c726c9 100644 --- a/script/app-mlperf-inference/README.md +++ b/script/app-mlperf-inference/README.md @@ -1,10 +1,10 @@ # README for app-mlperf-inference This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -166,7 +166,7 @@ mlcr app,vision,language,mlcommons,mlperf,inference,generic - `llama2-70b-99` (base: llama2-70b_) - `llama2-70b-99.9` (base: llama2-70b_) - `llama3_1-405b` -- `llama3_1-8b` +- `llama3_1-8b_` - `mixtral-8x7b` (base: mixtral-8x7b) - `mobilenet` - `pointpainting` @@ -207,4 +207,6 @@ mlcr app,vision,language,mlcommons,mlperf,inference,generic - `dlrm_` - `gptj_` (alias: gptj) - `llama2-70b_` +- `llama3_1-8b` (base: llama3_1-8b_) +- `llama3_1-8b-edge` (base: llama3_1-8b_) - `power` diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index 37f5ca9b2..62c05a287 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -395,7 +395,7 @@ def postprocess(i): if env.get('MLC_HOST_SYSTEM_NAME', '') != '': host_info['system_name'] = env['MLC_HOST_SYSTEM_NAME'] - # Check CM automation repository + # Check MLC automation repository repo_name = 'mlcommons@mlperf-automations' repo_hash = '' r = mlc.access({'action': 'find', 'automation': 'repo', @@ -429,7 +429,7 @@ def postprocess(i): cmd = "" xcmd = "" - readme_init = "*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.*\n\n" + readme_init = "*Check [MLC MLPerf docs](https://docs.mlcommons.org/inference) for more details.*\n\n" readme_body = "## Host platform\n\n* OS version: {}\n* CPU version: {}\n* Python version: {}\n* MLC version: {}\n\n".format(platform.platform(), platform.processor(), sys.version, mlc_version) @@ -438,7 +438,7 @@ def postprocess(i): if repo_hash != '': x += ' --checkout=' + str(repo_hash) - readme_body += "## CM Run Command\n\nSee [CM installation guide](https://docs.mlcommons.org/inference/install/).\n\n" + \ + readme_body += "## MLC Run Command\n\nSee [MLC installation guide](https://docs.mlcommons.org/inference/install/).\n\n" + \ "```bash\npip install -U mlcflow\n\nmlc rm cache -f\n\nmlc pull repo {}\n\n{}\n```".format( x, xcmd) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index f00790c92..f9ad1283f 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference uid: d775cac873ee4231 @@ -9,7 +9,7 @@ category: "Modular MLPerf inference benchmark pipeline" developers: "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Thomas Zhu](https://www.linkedin.com/in/hanwen-zhu-483614189), [Grigori Fursin](https://cKnowledge.org/gfursin)" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - vision @@ -68,7 +68,7 @@ input_mapping: predeps: False -# Duplicate CM environment variables to the ones used in native apps +# Duplicate MLC environment variables to the ones used in native apps env_key_mappings: MLC_HOST_: HOST_ MLC_ML_: ML_ @@ -82,14 +82,14 @@ new_state_keys: - app_mlperf_inference_* - mlc-mlperf-inference-results* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features - tags: detect,os # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python @@ -223,6 +223,8 @@ variations: tags: _int32 llama3_1-405b-accuracy-script: tags: _int32 + llama3_1-8b-accuracy-script: + tags: _int32 env: MLC_MLPERF_PYTHON: 'yes' MLC_MLPERF_IMPLEMENTATION: mlcommons_python @@ -937,7 +939,7 @@ variations: - waymo-dataset - tags: get,ml-model,pointpainting enable_if_env: - MLC_USE_DATASET_FROM_HOST: + MLC_USE_ML_MODEL_FROM_HOST: - 'yes' names: - pointpainting-model @@ -955,10 +957,31 @@ variations: - mlperf-accuracy-script - waymo-accuracy-script tags: run,accuracy,mlperf,_waymo - - llama3_1-8b: + + llama3_1-8b_: group: model + adr: + numpy: + version_max: "1.26.999" + version_max_usable: "1.26.4" + docker: + base_image: ubuntu:24.04 + mounts: + - "${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}:${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}" + - "${{ MLC_DATASET_CNNDM_EVAL_PATH }}:${{ MLC_DATASET_CNNDM_EVAL_PATH }}" + deps: + - tags: get,ml-model,llama3,_mlc,_8b,_r2-downloader + enable_if_env: + MLC_USE_ML_MODEL_FROM_HOST: + - 'yes' + names: + - llama3_1-8b + - llama3-8b + + llama3_1-8b: + base: + - llama3_1-8b_ add_deps_recursive: mlperf-inference-implementation: tags: _llama3_1-8b @@ -972,44 +995,52 @@ variations: - all MLC_MLPERF_ACCURACY_RESULTS_DIR: - 'on' - MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: - - edge skip_if_env: MLC_MLPERF_IMPLEMENTATION: - nvidia names: - mlperf-accuracy-script - llama3_1-8b-accuracy-script - tags: run,accuracy,mlperf,_cnndm_llama_3,_edge + tags: run,accuracy,mlperf,_cnndm_llama_3,_datacenter + docker: + deps: + ## CNNDM for Llama3 8B model - datacenter + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader + names: + - cnndm-llama3-datacenter + enable_if_env: + MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: + - datacenter + MLC_USE_DATASET_FROM_HOST: + - 'yes' + + llama3_1-8b-edge: + base: + - llama3_1-8b_ + add_deps_recursive: + mlperf-inference-implementation: + tags: _llama3_1-8b-edge + env: + MLC_MODEL: + llama3_1-8b-edge + posthook_deps: - enable_if_env: MLC_MLPERF_LOADGEN_MODE: - accuracy - all MLC_MLPERF_ACCURACY_RESULTS_DIR: - 'on' - MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: - - datacenter skip_if_env: MLC_MLPERF_IMPLEMENTATION: - nvidia names: - mlperf-accuracy-script - llama3_1-8b-accuracy-script - tags: run,accuracy,mlperf,_cnndm_llama_3,_datacenter + tags: run,accuracy,mlperf,_cnndm_llama_3,_edge docker: - mounts: - - "${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}:${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}" - - "${{ MLC_DATASET_CNNDM_EVAL_PATH }}:${{ MLC_DATASET_CNNDM_EVAL_PATH }}" deps: - - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct - enable_if_env: - MLC_USE_ML_MODEL_FROM_HOST: - - 'yes' - names: - - llama3_1-8b - - llama3-8b ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_rclone + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge enable_if_env: @@ -1017,15 +1048,6 @@ variations: - edge MLC_USE_DATASET_FROM_HOST: - 'yes' - ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_rclone - names: - - cnndm-llama3-datacenter - enable_if_env: - MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: - - datacenter - MLC_USE_DATASET_FROM_HOST: - - 'yes' whisper: group: @@ -1052,13 +1074,13 @@ variations: tags: run,accuracy,mlperf,_librispeech_whisper,_int64 docker: deps: - - tags: get,ml-model,whisper,_rclone,_mlc + - tags: get,ml-model,whisper,_r2-downloader,_mlc enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' names: - whisper-model - - tags: get,dataset,whisper,_preprocessed,_mlc,_rclone + - tags: get,dataset,whisper,_preprocessed,_mlc,_r2-downloader names: - whisper-dataset enable_if_env: @@ -1130,7 +1152,7 @@ variations: tags: run,accuracy,mlperf,_dataset_deepseek-r1 docker: deps: - - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_rclone + - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2-downloader enable_if_env: MLC_USE_DATASET_FROM_HOST: - 'yes' @@ -1720,7 +1742,7 @@ variations: cuda,reference: docker: - base_image: nvcr.io/nvidia/pytorch:24.03-py3 + base_image: nvcr.io/nvidia/pytorch:24.08-py3 cuda: docker: @@ -2057,8 +2079,7 @@ variations: reproducibility add_deps_recursive: nvidia-inference-common-code: - version: r4.1-dev - tags: _mlcommons + tags: _mlcommons,_v4.0 nvidia-inference-server: version: r4.0 tags: _mlcommons @@ -2096,7 +2117,7 @@ variations: nvidia-mitten-git-src: tags: _sha.e5574192b3ae76910b6824a9275d14a4158d8afa pycuda: - version: "2024.1" + version_min: "2024.1" default_env: MLC_SKIP_SYS_UTILS: 'yes' MLC_REGENERATE_MEASURE_FILES: 'yes' @@ -2120,7 +2141,7 @@ variations: nvidia-mitten-git-src: tags: _sha.e5574192b3ae76910b6824a9275d14a4158d8afa pycuda: - version: "2024.1" + version_min: "2024.1" default_env: MLC_SKIP_SYS_UTILS: 'yes' MLC_REGENERATE_MEASURE_FILES: 'yes' diff --git a/script/app-mlperf-training-nvidia/meta.yaml b/script/app-mlperf-training-nvidia/meta.yaml index abf4e7dd9..f4aa6eea2 100644 --- a/script/app-mlperf-training-nvidia/meta.yaml +++ b/script/app-mlperf-training-nvidia/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-training-nvidia uid: 1e2e357618cc4674 @@ -7,7 +7,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf training benchmark pipeline" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - vision @@ -43,7 +43,7 @@ new_state_keys: - mlperf-inference-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features @@ -53,7 +53,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python diff --git a/script/app-mlperf-training-reference/meta.yaml b/script/app-mlperf-training-reference/meta.yaml index 45b0633be..ec2666af2 100644 --- a/script/app-mlperf-training-reference/meta.yaml +++ b/script/app-mlperf-training-reference/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-training-reference uid: 0c4b11bdcf494b4f @@ -7,7 +7,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf training benchmark pipeline" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - vision @@ -45,7 +45,7 @@ new_state_keys: - mlperf-inference-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features @@ -55,7 +55,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python diff --git a/script/app-stable-diffusion-onnx-py/meta.yaml b/script/app-stable-diffusion-onnx-py/meta.yaml index 4aacbe801..978d245bb 100644 --- a/script/app-stable-diffusion-onnx-py/meta.yaml +++ b/script/app-stable-diffusion-onnx-py/meta.yaml @@ -20,7 +20,7 @@ tags_help: "modular python app stable-diffusion onnx" deps: - tags: detect,os -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - names: - python - python3 diff --git a/script/authenticate-github-cli/run.sh b/script/authenticate-github-cli/run.sh index ad1472f09..62f121083 100644 --- a/script/authenticate-github-cli/run.sh +++ b/script/authenticate-github-cli/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/benchmark-any-mlperf-inference-implementation/run-template.sh b/script/benchmark-any-mlperf-inference-implementation/run-template.sh index 8e0cb42c0..b77849229 100644 --- a/script/benchmark-any-mlperf-inference-implementation/run-template.sh +++ b/script/benchmark-any-mlperf-inference-implementation/run-template.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/benchmark-program-mlperf/customize.py b/script/benchmark-program-mlperf/customize.py index bb74f7f4c..9172a690f 100644 --- a/script/benchmark-program-mlperf/customize.py +++ b/script/benchmark-program-mlperf/customize.py @@ -1,5 +1,5 @@ from mlc import utils -from utils import is_true +from utils import is_true, is_false import os diff --git a/script/benchmark-program/customize.py b/script/benchmark-program/customize.py index be97afe39..66886971a 100644 --- a/script/benchmark-program/customize.py +++ b/script/benchmark-program/customize.py @@ -106,7 +106,7 @@ def preprocess(i): # Print info logger.info( '***************************************************************************') - logger.info('CM script::benchmark-program/run.sh') + logger.info('MLC script::benchmark-program/run.sh') logger.info('') logger.info('Run Directory: {}'.format(env.get('MLC_RUN_DIR', ''))) diff --git a/script/benchmark-program/run.sh b/script/benchmark-program/run.sh index 6caace406..adad9f555 100755 --- a/script/benchmark-program/run.sh +++ b/script/benchmark-program/run.sh @@ -29,7 +29,7 @@ cd "${MLC_RUN_DIR}" if [[ "${MLC_DEBUG_SCRIPT_BENCHMARK_PROGRAM}" == "True" ]]; then echo "*****************************************************" - echo "You are now in Debug shell with pre-set CM env and can run the following command line manually:" + echo "You are now in Debug shell with pre-set MLC env and can run the following command line manually:" echo "" if [[ "${MLC_RUN_CMD0}" != "" ]]; then @@ -39,7 +39,7 @@ if [[ "${MLC_DEBUG_SCRIPT_BENCHMARK_PROGRAM}" == "True" ]]; then fi echo "" - echo "Type exit to return to CM script." + echo "Type exit to return to MLC script." echo "" # echo "You can also run . ./debug-script-benchmark-program.sh to reproduce and customize run." # echo "" diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index b240e4164..987f7ba6f 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -15,7 +15,7 @@ def preprocess(i): if env["MLC_DOCKER_OS"] not in ["ubuntu", "rhel", "arch"]: return { - 'return': 1, 'error': f"Specified docker OS: {env['MLC_DOCKER_OS']}. Currently only ubuntu, rhel and arch are supported in CM docker"} + 'return': 1, 'error': f"Specified docker OS: {env['MLC_DOCKER_OS']}. Currently only ubuntu, rhel and arch are supported in MLC docker"} path = i['run_script_input']['path'] @@ -227,7 +227,7 @@ def preprocess(i): if env.get('MLC_DOCKER_EXTRA_SYS_DEPS', '') != '': f.write('RUN ' + env['MLC_DOCKER_EXTRA_SYS_DEPS'] + EOL) - if env['MLC_DOCKER_OS'] == "ubuntu": + if env['MLC_DOCKER_OS'] == "ubuntu" and False: # Turning this off as we are using venv if int(str(env['MLC_DOCKER_OS_VERSION']).split('.')[0]) >= 23: if "--break-system-packages" not in env.get( 'MLC_DOCKER_PIP_INSTALL_EXTRA_FLAGS', ''): @@ -264,7 +264,7 @@ def preprocess(i): env['MLC_DOCKER_USE_DEFAULT_USER'] = 'yes' if docker_user and not is_true( - env.get('MLC_DOCKER_USE_DEFAULT_USER', '') and docker_user != 'root'): + env.get('MLC_DOCKER_USE_DEFAULT_USER', '')) and docker_user != 'root': f.write('RUN groupadd -g $GID -o ' + docker_group + EOL) @@ -293,7 +293,8 @@ def preprocess(i): dockerfile_env_input_string = "" for docker_env_key in dockerfile_env: dockerfile_env_input_string = dockerfile_env_input_string + " --env." + \ - docker_env_key + "=" + str(dockerfile_env[docker_env_key]) + docker_env_key + "=" + \ + str(dockerfile_env[docker_env_key]).replace("\n", "\\n") workdir = env.get('WORKDIR', '') if workdir == '': diff --git a/script/build-dockerfile/dockerinfo.json b/script/build-dockerfile/dockerinfo.json index 2b222c01c..f76510841 100644 --- a/script/build-dockerfile/dockerinfo.json +++ b/script/build-dockerfile/dockerinfo.json @@ -28,7 +28,7 @@ "package-manager-update-cmd": "apt-get update -y", "package-manager-get-cmd": "apt-get install -y", "packages": [ - "python3", "python3-pip", "git", "sudo", "wget", "python3-venv", "systemctl", "unzip" + "python3", "python3-pip", "git", "sudo", "wget", "python3-venv", "systemctl", "unzip", "ca-certificates" ], "versions": { "18.04": { diff --git a/script/build-mlperf-inference-server-nvidia/README.md b/script/build-mlperf-inference-server-nvidia/README.md index 280d286b0..0714b29f1 100644 --- a/script/build-mlperf-inference-server-nvidia/README.md +++ b/script/build-mlperf-inference-server-nvidia/README.md @@ -1,10 +1,10 @@ # README for build-mlperf-inference-server-nvidia This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/build-mlperf-inference-server-nvidia/meta.yaml b/script/build-mlperf-inference-server-nvidia/meta.yaml index 7fc8f9b71..8c57fd809 100644 --- a/script/build-mlperf-inference-server-nvidia/meta.yaml +++ b/script/build-mlperf-inference-server-nvidia/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: build-mlperf-inference-server-nvidia uid: f37403af5e9f4541 cache: true @@ -9,7 +9,7 @@ default_version: r3.1 category: "MLPerf benchmark support" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - build - mlcommons @@ -33,7 +33,7 @@ input_mapping: custom_system: MLC_CUSTOM_SYSTEM_NVIDIA clean: MLC_MAKE_CLEAN -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -44,7 +44,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect python3 - tags: get,python3 @@ -76,7 +76,7 @@ deps: # Detect gcc - tags: get,gcc - # Detect CMake + # Detect MLCake - tags: get,cmake version: "3.25.1" diff --git a/script/calibrate-model-for.qaic/run.sh b/script/calibrate-model-for.qaic/run.sh index 7da7962b9..d643e811e 100644 --- a/script/calibrate-model-for.qaic/run.sh +++ b/script/calibrate-model-for.qaic/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/clean-nvidia-mlperf-inference-scratch-space/run.sh b/script/clean-nvidia-mlperf-inference-scratch-space/run.sh index 32cf4d51e..4a22ce805 100644 --- a/script/clean-nvidia-mlperf-inference-scratch-space/run.sh +++ b/script/clean-nvidia-mlperf-inference-scratch-space/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/compile-model-for.qaic/run.sh b/script/compile-model-for.qaic/run.sh index d20c3a705..6ce29fde0 100644 --- a/script/compile-model-for.qaic/run.sh +++ b/script/compile-model-for.qaic/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/convert-csv-to-md/run.sh b/script/convert-csv-to-md/run.sh index 7da7962b9..d643e811e 100644 --- a/script/convert-csv-to-md/run.sh +++ b/script/convert-csv-to-md/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/create-custom-cache-entry/customize.py b/script/create-custom-cache-entry/customize.py index 795235bfd..0f275b0a5 100644 --- a/script/create-custom-cache-entry/customize.py +++ b/script/create-custom-cache-entry/customize.py @@ -5,7 +5,7 @@ def preprocess(i): - # CM script internal variables + # MLC script internal variables env = i['env'] logger = i['automation'].logger extra_cache_tags = [] diff --git a/script/create-fpgaconvnet-app-tinyml/run.sh b/script/create-fpgaconvnet-app-tinyml/run.sh index 35de74bab..65f2a08f9 100644 --- a/script/create-fpgaconvnet-app-tinyml/run.sh +++ b/script/create-fpgaconvnet-app-tinyml/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/create-fpgaconvnet-config-tinyml/run.sh b/script/create-fpgaconvnet-config-tinyml/run.sh index 35de74bab..65f2a08f9 100644 --- a/script/create-fpgaconvnet-config-tinyml/run.sh +++ b/script/create-fpgaconvnet-config-tinyml/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/detect-sudo/customize.py b/script/detect-sudo/customize.py index 104d8b781..56e6ae627 100644 --- a/script/detect-sudo/customize.py +++ b/script/detect-sudo/customize.py @@ -4,7 +4,6 @@ import subprocess import select import sys -import grp import threading import getpass @@ -13,6 +12,9 @@ def preprocess(i): os_info = i['os_info'] + if os_info['platform'] == 'windows': + return {'return': 0} + env = i['env'] meta = i['meta'] @@ -98,6 +100,7 @@ def prompt_retry(logger, timeout=10, default_retry=False): def is_user_in_sudo_group(logger): + import grp # noqa """Check if the current user is in the 'sudo' group.""" try: sudo_group = grp.getgrnam('sudo').gr_mem diff --git a/script/detect-sudo/run.sh b/script/detect-sudo/run.sh index 821adb3f9..ee74b85b6 100644 --- a/script/detect-sudo/run.sh +++ b/script/detect-sudo/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/download-and-extract/meta.yaml b/script/download-and-extract/meta.yaml index 02aaf1175..8a3b17d07 100644 --- a/script/download-and-extract/meta.yaml +++ b/script/download-and-extract/meta.yaml @@ -125,9 +125,9 @@ variations: download-script: tags: _wget group: download-tool - r2_downloader: + r2-downloader: add_deps: download-script: - tags: _r2_downloader + tags: _r2-downloader group: download-tool versions: {} diff --git a/script/download-file/README.md b/script/download-file/README.md index 5caa972b5..dcc6f866c 100644 --- a/script/download-file/README.md +++ b/script/download-file/README.md @@ -1,10 +1,10 @@ # README for download-file This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -72,7 +72,7 @@ mlcr download file - `curl` - `gdown` - `mlcutil` (alias: cmutil) (default) -- `r2_downloader` +- `r2-downloader` - `rclone` - `wget` diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 3e0665c79..6a2eb01e9 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -203,7 +203,7 @@ def preprocess(i): env['MLC_DOWNLOAD_CMD'] += f" || (({del_cmd} {env['MLC_DOWNLOAD_FILENAME']} || true) && wget -nc {extra_download_options} {url})" logger.info(f"{env['MLC_DOWNLOAD_CMD']}") - elif tool == "r2_downloader": + elif tool == "r2-downloader": env['MLC_DOWNLOAD_CMD'] = f"bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) " if env["MLC_HOST_OS_TYPE"] == "windows": # have to modify the variable from url to temp_url if it is diff --git a/script/download-file/meta.yaml b/script/download-file/meta.yaml index e195b078f..aaf9bf1dd 100644 --- a/script/download-file/meta.yaml +++ b/script/download-file/meta.yaml @@ -77,9 +77,9 @@ variations: env: MLC_DOWNLOAD_TOOL: wget group: download-tool - r2_downloader: + r2-downloader: env: - MLC_DOWNLOAD_TOOL: r2_downloader + MLC_DOWNLOAD_TOOL: r2-downloader group: download-tool versions: {} diff --git a/script/draw-graph-from-json-data/run.sh b/script/draw-graph-from-json-data/run.sh index 32cf4d51e..4a22ce805 100644 --- a/script/draw-graph-from-json-data/run.sh +++ b/script/draw-graph-from-json-data/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/dump-pip-freeze/run.sh b/script/dump-pip-freeze/run.sh index 0e379ea14..a6b4b8dea 100644 --- a/script/dump-pip-freeze/run.sh +++ b/script/dump-pip-freeze/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index ddefc0757..f6b185fe0 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -36,7 +36,7 @@ def preprocess(i): env['MLC_EXTRACT_FILENAME'] = filename - # Check if extract to some path outside CM cache (to reuse large files + # Check if extract to some path outside MLC cache (to reuse large files # later if cache is cleaned) extract_path = env.get('MLC_EXTRACT_PATH', '') if extract_path != '': diff --git a/script/fail/customize.py b/script/fail/customize.py index 6d3eda24d..840aabb82 100644 --- a/script/fail/customize.py +++ b/script/fail/customize.py @@ -19,7 +19,7 @@ def preprocess(i): if is_true(env.get('MLC_FAIL_WINDOWS', '')): if os_info['platform'] == 'windows': return {'return': 1, - 'error': 'CM detected fail condition: running on Windows'} + 'error': 'MLC detected fail condition: running on Windows'} return {'return': 0} diff --git a/script/generate-mlperf-inference-submission/README.md b/script/generate-mlperf-inference-submission/README.md index 9e66372e8..603dc5d8d 100644 --- a/script/generate-mlperf-inference-submission/README.md +++ b/script/generate-mlperf-inference-submission/README.md @@ -1,10 +1,10 @@ # README for generate-mlperf-inference-submission This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -41,6 +41,7 @@ mlcr generate,submission,mlperf,mlperf-inference,inference,mlcommons,inference-s | Name | Description | Choices | Default | |------|-------------|---------|------| | `--analyzer_settings_file` | | | `` | +| `--status` | | | `` | | `--category` | | | `` | | `--clean` | | | `` | | `--dashboard` | | | `` | diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 875a12906..8ca22b596 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -401,11 +401,48 @@ def generate_submission(env, state, inp, submission_division, logger): # we check for the existance of mlperf_log_summary.txt # mlperf_log_detail.txt to consider a result folder as valid. # Rest of the checks are done later by the submission checker - files_to_check = [ + files_to_check_in_perf_dir = [ "mlperf_log_summary.txt", "mlperf_log_detail.txt"] - if not all([os.path.exists(os.path.join( - result_scenario_path, "performance", "run_1", f)) for f in files_to_check]): + perf_run_dir = os.path.join( + result_scenario_path, "performance", "run_1") + missing = [ + f for f in files_to_check_in_perf_dir + if not os.path.exists(os.path.join(perf_run_dir, f)) + ] + if missing: + logger.warning( + f"""Missing file(s) in {perf_run_dir}: {', '.join(missing)}, Skipping directory: {result_scenario_path}""") + continue + + files_to_check_in_acc_dir = [ + "mlperf_log_summary.txt", + "mlperf_log_detail.txt", + "mlperf_log_accuracy.json", + "accuracy.txt"] + acc_run_dir = os.path.join( + result_scenario_path, "accuracy") + missing = [ + f for f in files_to_check_in_acc_dir + if not os.path.exists(os.path.join(acc_run_dir, f)) + ] + if missing: + logger.warning( + f"""Missing file(s) in {acc_run_dir}: {', '.join(missing)}, Skipping directory: {result_scenario_path}""") + continue + + if not os.path.exists(os.path.join(perf_run_dir, "user.conf")) and not os.path.exists( + os.path.join(result_scenario_path, "user.conf")): + logger.warning( + f"""Missing user.conf in both {os.path.join(perf_run_dir, "user.conf")} and {os.path.join(result_scenario_path, "user.conf")}. Skipping directory: {result_scenario_path}""" + ) + continue + + if not os.path.exists(os.path.join(perf_run_dir, "measurements.json")) and not os.path.exists( + os.path.join(result_scenario_path, "measurements.json")): + logger.warning( + f"""Missing measurements.json in both {os.path.join(perf_run_dir, "measurements.json")} and {os.path.join(result_scenario_path, "measurements.json")}. Skipping directory: {result_scenario_path}""" + ) continue if not os.path.isdir(measurement_scenario_path): @@ -488,7 +525,7 @@ def generate_submission(env, state, inp, submission_division, logger): result_mode_path, "system_meta.json") else: logger.error( - "WARNING: system_meta.json was not found in the SUT root or mode directory inside the results folder. CM is automatically creating one using the system defaults. Please modify them as required.") + "WARNING: system_meta.json was not found in the SUT root or mode directory inside the results folder. MLC is automatically creating one using the system defaults. Please modify them as required.") if os.path.exists(saved_system_meta_file_path): with open(saved_system_meta_file_path, "r") as f: saved_system_meta = json.load(f) diff --git a/script/generate-mlperf-inference-submission/meta.yaml b/script/generate-mlperf-inference-submission/meta.yaml index 4060c6058..4125b7eb7 100644 --- a/script/generate-mlperf-inference-submission/meta.yaml +++ b/script/generate-mlperf-inference-submission/meta.yaml @@ -124,6 +124,8 @@ variations: tags: _wg-inference mlperf-inference-submission-checker: tags: _wg-inference + default_env: + MLC_MLPERF_SUBMISSION_CHECKER_VERSION: v5.1 deps: - names: - inference-src @@ -140,6 +142,8 @@ variations: tags: _wg-automotive mlperf-inference-submission-checker: tags: _wg-automotive + default_env: + MLC_MLPERF_SUBMISSION_CHECKER_VERSION: v0.5 deps: - names: - automotive-src diff --git a/script/generate-mlperf-inference-user-conf/meta.yaml b/script/generate-mlperf-inference-user-conf/meta.yaml index b500b8f94..086de0bb9 100644 --- a/script/generate-mlperf-inference-user-conf/meta.yaml +++ b/script/generate-mlperf-inference-user-conf/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: generate-mlperf-inference-user-conf uid: 3af4475745964b93 @@ -9,7 +9,7 @@ category: "MLPerf benchmark support" developers: "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Thomas Zhu](https://www.linkedin.com/in/hanwen-zhu-483614189), [Grigori Fursin](https://cKnowledge.org/gfursin)" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - generate - mlperf @@ -63,7 +63,7 @@ new_env_keys: new_state_keys: - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features diff --git a/script/generate-mlperf-tiny-report/meta.yaml b/script/generate-mlperf-tiny-report/meta.yaml index 467226c1b..39de344f6 100644 --- a/script/generate-mlperf-tiny-report/meta.yaml +++ b/script/generate-mlperf-tiny-report/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: generate-mlperf-tiny-report uid: 709c3f3f9b3e4783 @@ -12,7 +12,7 @@ developers: "[Grigori Fursin](https://cKnowledge.org/gfursin)" default_env: MLC_IMPORT_TINYMLPERF_REPO_TAGS: "1.1-private" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - generate - mlperf @@ -23,14 +23,14 @@ tags: input_mapping: repo_tags: MLC_IMPORT_TINYMLPERF_REPO_TAGS -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features - tags: detect,os # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect python3 - tags: get,python3 diff --git a/script/generate-nvidia-engine/meta.yaml b/script/generate-nvidia-engine/meta.yaml index b63ba77e1..d6c50605f 100644 --- a/script/generate-nvidia-engine/meta.yaml +++ b/script/generate-nvidia-engine/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: generate-nvidia-engine uid: 0eef9f05b272401f @@ -8,7 +8,7 @@ automation_uid: 5b4e0237da074764 category: "MLPerf benchmark support" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - generate - engine @@ -32,7 +32,7 @@ new_env_keys: - MLC_MLPERF_* - MLC_DATASET_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -43,7 +43,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect CUDA - tags: get,cuda,_cudnn diff --git a/script/get-android-sdk/customize.py b/script/get-android-sdk/customize.py index eedcb31e6..8087f07e3 100644 --- a/script/get-android-sdk/customize.py +++ b/script/get-android-sdk/customize.py @@ -20,7 +20,7 @@ def preprocess(i): # Check if ANDROID_HOME is already set android_home = os.environ.get('ANDROID_HOME', '').strip() - # We are inside CM cache entry + # We are inside MLC cache entry cur_dir = os.getcwd() if android_home == '': diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index d5890e103..e76f8123b 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -73,7 +73,7 @@ def preprocess(i): def detect_version(i): logger = i['automation'].logger - r = i['automation'].parse_version({'match_text': r'CLANG:\sAOCC_([\d.]+-Build#[\d]+)', + r = i['automation'].parse_version({'match_text': r'CLANG:\sAOCC_([\d.]+(?:-[\w]+)?-Build#[\d]+)', 'group_number': 1, 'env_key': 'MLC_AOCC_VERSION', 'which_env': i['env']}) diff --git a/script/get-apptainer/COPYRIGHT.md b/script/get-apptainer/COPYRIGHT.md new file mode 100644 index 000000000..2d6a2775e --- /dev/null +++ b/script/get-apptainer/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +Š 2023-2025 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-apptainer/customize.py b/script/get-apptainer/customize.py new file mode 100644 index 000000000..6750eede3 --- /dev/null +++ b/script/get-apptainer/customize.py @@ -0,0 +1,85 @@ +from mlc import utils +import os + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + automation = i['automation'] + + recursion_spaces = i['recursion_spaces'] + + file_name_apptainer = 'apptainer.exe' if os_info['platform'] == 'windows' else 'apptainer' + + if 'MLC_APPTAINER_BIN_WITH_PATH' not in env: + env['FILE_NAME'] = file_name_apptainer + env['CONTAINER_TOOL_NAME'] = "apptainer" + r = i['automation'].find_artifact({'file_name': file_name_apptainer, + 'env': env, + 'os_info': os_info, + 'default_path_env_key': 'PATH', + 'detect_version': True, + 'env_path_key': 'MLC_APPTAINER_BIN_WITH_PATH', + 'run_script_input': i['run_script_input'], + 'recursion_spaces': recursion_spaces}) + if r['return'] > 0: + if r['return'] == 16: + run_file_name = "install" + r = automation.run_native_script( + {'run_script_input': i['run_script_input'], 'env': env, 'script_name': run_file_name}) + if r['return'] > 0: + return r + else: + return r + + return {'return': 0} + + +def detect_version(i): + r = i['automation'].parse_version({ + 'match_text': r'[Aa]pptainer version\s*([\d.]+)', + 'group_number': 1, + 'env_key': 'MLC_APPTAINER_VERSION', + 'which_env': i['env'] + }) + if r['return'] > 0: + return r + + logger = i['automation'].logger + + version = r['version'] + + tool = "apptainer" + + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) + return {'return': 0, 'version': version, "tool": tool} + + +def postprocess(i): + env = i['env'] + + r = detect_version(i) + + if r['return'] > 0: + return r + + version = r['version'] + tool = r['tool'] + found_file_path = env['MLC_APPTAINER_BIN_WITH_PATH'] + + found_path = os.path.dirname(found_file_path) + env['MLC_APPTAINER_INSTALLED_PATH'] = found_path + env['+PATH'] = [found_path] + + env['MLC_APPTAINER_CACHE_TAGS'] = 'version-' + version + + env['MLC_APPTAINER_VERSION'] = version + + env['MLC_CONTAINER_TOOL'] = tool + + return {'return': 0, 'version': version} diff --git a/script/get-apptainer/install-centos.sh b/script/get-apptainer/install-centos.sh new file mode 100644 index 000000000..be9879bf2 --- /dev/null +++ b/script/get-apptainer/install-centos.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e +${MLC_SUDO} yum install -y epel-release +${MLC_SUDO} yum install -y apptainer +test $? -eq 0 || exit $? + +# Print version +apptainer --version diff --git a/script/get-apptainer/install-ubuntu.sh b/script/get-apptainer/install-ubuntu.sh new file mode 100644 index 000000000..8a7036669 --- /dev/null +++ b/script/get-apptainer/install-ubuntu.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +${MLC_SUDO} apt-get update +${MLC_SUDO} apt-get install -y software-properties-common +${MLC_SUDO} add-apt-repository -y ppa:apptainer/ppa +${MLC_SUDO} apt-get update +${MLC_SUDO} apt-get install -y apptainer +test $? -eq 0 || exit $? + +# Print version +apptainer --version + diff --git a/script/get-apptainer/install.bat b/script/get-apptainer/install.bat new file mode 100644 index 000000000..dfde606b6 --- /dev/null +++ b/script/get-apptainer/install.bat @@ -0,0 +1,2 @@ +echo "Please install apptainer to continue" +exit 1 diff --git a/script/get-apptainer/install.sh b/script/get-apptainer/install.sh new file mode 100644 index 000000000..dfde606b6 --- /dev/null +++ b/script/get-apptainer/install.sh @@ -0,0 +1,2 @@ +echo "Please install apptainer to continue" +exit 1 diff --git a/script/get-apptainer/meta.yaml b/script/get-apptainer/meta.yaml new file mode 100644 index 000000000..39743ec39 --- /dev/null +++ b/script/get-apptainer/meta.yaml @@ -0,0 +1,22 @@ +alias: get-apptainer +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +category: Detection or installation of tools and artifacts +deps: +- tags: detect,os +- tags: detect,sudo +input_description: {} +input_mapping: {} +new_env_keys: +- MLC_APPTAINER_VERSION +- MLC_CONTAINER_TOOL +new_state_keys: [] +post_deps: [] +posthook_deps: [] +prehook_deps: [] +tags: +- get-apptainer +uid: 37c5bc01c11f49fa +variations: {} +versions: {} diff --git a/script/get-apptainer/run.bat b/script/get-apptainer/run.bat new file mode 100644 index 000000000..a90c05a64 --- /dev/null +++ b/script/get-apptainer/run.bat @@ -0,0 +1,3 @@ +@echo off +apptainer --version > tmp-ver.out +if %errorlevel% neq 0 exit /b 1 diff --git a/script/get-apptainer/run.sh b/script/get-apptainer/run.sh new file mode 100644 index 000000000..38a153a65 --- /dev/null +++ b/script/get-apptainer/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash +apptainer --version > tmp-ver.out +test $? -eq 0 || exit 1 diff --git a/script/get-aria2/customize.py b/script/get-aria2/customize.py index 2f7459cde..dfbdafc36 100644 --- a/script/get-aria2/customize.py +++ b/script/get-aria2/customize.py @@ -5,7 +5,7 @@ def preprocess(i): - # Pre-set by CM + # Pre-set by MLC os_info = i['os_info'] env = i['env'] logger = i['automation'].logger diff --git a/script/get-cache-dir/README.md b/script/get-cache-dir/README.md new file mode 100644 index 000000000..b9f650517 --- /dev/null +++ b/script/get-cache-dir/README.md @@ -0,0 +1,62 @@ +# README for get-cache-dir +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,cache,dir,directory +``` + +No script specific inputs +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Ungrouped + +- `name.#` _(# can be substituted dynamically)_ diff --git a/script/get-cache-dir/meta.yaml b/script/get-cache-dir/meta.yaml index e02b9a7cb..25aad5eae 100644 --- a/script/get-cache-dir/meta.yaml +++ b/script/get-cache-dir/meta.yaml @@ -2,7 +2,7 @@ alias: get-cache-dir automation_alias: script automation_uid: 5b4e0237da074764 cache: true -category: CM Interface +category: MLC Interface deps: [] docker: run: false diff --git a/script/get-cmsis_5/meta.yaml b/script/get-cmsis_5/meta.yaml index 95ac1ef3a..b35872112 100644 --- a/script/get-cmsis_5/meta.yaml +++ b/script/get-cmsis_5/meta.yaml @@ -11,7 +11,7 @@ default_version: custom deps: - tags: detect,os new_env_keys: -- CMSIS* +- MLCSIS* tags: - get - cmsis diff --git a/script/get-cmsis_5/run.sh b/script/get-cmsis_5/run.sh index 47d1e2554..af857a9a5 100644 --- a/script/get-cmsis_5/run.sh +++ b/script/get-cmsis_5/run.sh @@ -7,11 +7,11 @@ echo "******************************************************" if [ ! -d "cmsis" ]; then if [ -z ${MLC_GIT_SHA} ]; then - echo "Cloning CMSIS_5 from ${MLC_GIT_URL} with branch ${MLC_GIT_CHECKOUT} ${MLC_GIT_DEPTH} ${MLC_GIT_RECURSE_SUBMODULES}..." + echo "Cloning MLCSIS_5 from ${MLC_GIT_URL} with branch ${MLC_GIT_CHECKOUT} ${MLC_GIT_DEPTH} ${MLC_GIT_RECURSE_SUBMODULES}..." git clone ${MLC_GIT_RECURSE_SUBMODULES} -b "${MLC_GIT_CHECKOUT}" ${MLC_GIT_URL} ${MLC_GIT_DEPTH} cmsis if [ "${?}" != "0" ]; then exit 1; fi else - echo "Cloning CMSIS_5 from ${MLC_GIT_URL} with default branch and checkout ${MLC_GIT_CHECKOUT} ${MLC_GIT_DEPTH} ${MLC_GIT_RECURSE_SUBMODULES}..." + echo "Cloning MLCSIS_5 from ${MLC_GIT_URL} with default branch and checkout ${MLC_GIT_CHECKOUT} ${MLC_GIT_DEPTH} ${MLC_GIT_RECURSE_SUBMODULES}..." git clone ${MLC_GIT_RECURSE_SUBMODULES} ${MLC_GIT_URL} ${MLC_GIT_DEPTH} cmsis if [ "${?}" != "0" ]; then exit 1; fi cd cmsis diff --git a/script/get-cuda-devices/run.sh b/script/get-cuda-devices/run.sh index 2ee43d856..c802ec930 100644 --- a/script/get-cuda-devices/run.sh +++ b/script/get-cuda-devices/run.sh @@ -23,7 +23,7 @@ cd ${MLC_TMP_CURRENT_SCRIPT_PATH} ${MLC_NVCC_BIN_WITH_PATH} -allow-unsupported-compiler print_cuda_devices.cu test $? -eq 0 || exit 1 -# Return to the original path obtained in CM +# Return to the original path obtained in MLC echo "" echo "Running program ..." diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index cf70343e3..8214fe123 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -6,7 +6,7 @@ category: AI/ML datasets default_env: MLC_DATASET_CALIBRATION: 'no' deps: -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - names: - python - python3 @@ -39,6 +39,34 @@ deps: skip_if_env: MLC_TMP_ML_MODEL: - llama3_1-8b +prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + MLC_CNNDM_DOWNLOAD_TOOL: + - rclone + - tags: get,rclone-config,_mlc-inference + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + MLC_CNNDM_DOWNLOAD_TOOL: + - rclone + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_RCLONE_COPY_USING: copy + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL env: MLC_DATASET: CNNDM tags: @@ -58,25 +86,19 @@ variations: datacenter: group: category rclone: - prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_mlc-inference - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes + env: + MLC_CNNDM_DOWNLOAD_TOOL: rclone group: download-tool add_deps_recursive: dae: tags: _rclone - r2_downloader: + r2-downloader: + env: + MLC_CNNDM_DOWNLOAD_TOOL: r2-downloader group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader dry-run: group: run-mode env: @@ -84,7 +106,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -126,21 +148,25 @@ variations: - MLC_DATASET_PATH - MLC_DATASET_EVAL_PATH - MLC_DATASET_CNNDM_EVAL_PATH + validation,llama3: + default-variations: + download-tool: r2-downloader + download-src: mlc validation,edge,llama3,mlc,rclone: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,edge env: - MLC_DATASET_CNNDM_FILENAME: sample_cnn_eval_5000.json - MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> + MLC_DATASET_CNNDM_FILENAME: cnn_eval_5000.json + MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/datasets/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH - validation,edge,llama3,mlc,r2_downloader: + validation,edge,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,edge env: - MLC_DATASET_CNNDM_FILENAME: sample_cnn_eval_5000.json + MLC_DATASET_CNNDM_FILENAME: cnn_eval_5000.json MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/llama3-1-8b-sample-cnn-eval-5000.uri MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH @@ -150,10 +176,10 @@ variations: extra_cache_tags: cnndm,dataset,llama3,val,datacenter env: MLC_DATASET_CNNDM_FILENAME: cnn_eval.json - MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> + MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/datasets/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH - validation,datacenter,llama3,mlc,r2_downlaoder: + validation,datacenter,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,datacenter @@ -171,7 +197,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_CALIBRATION_DATASET_CNNDM_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_CALIBRATION_DATASET_CNNDM_PATH - calibation,llama3,mlc,r2_downloader: + calibation,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,calib @@ -185,7 +211,7 @@ tests: - variations_list: # - validation,edge,rclone,llama3,mlc,dry-run # - validation,datacenter,rclone,llama3,mlc,dry-run - - validation,edge,r2_downloader,llama3,mlc,dry-run - - validation,datacenter,r2_downloader,llama3,mlc,dry-run - - calibration,r2_downloader,llama3,mlc,dry-run - # - calibration,rclone,llama3,mlc,dry-run \ No newline at end of file + - validation,edge,r2-downloader,llama3,mlc,dry-run + - validation,datacenter,r2-downloader,llama3,mlc,dry-run + - calibration,r2-downloader,llama3,mlc,dry-run + # - calibration,rclone,llama3,mlc,dry-run diff --git a/script/get-dataset-coco/customize.py b/script/get-dataset-coco/customize.py index e8798fbaf..8e904e264 100644 --- a/script/get-dataset-coco/customize.py +++ b/script/get-dataset-coco/customize.py @@ -6,7 +6,7 @@ def preprocess(i): - # CM script internal variables + # MLC script internal variables variation_tags = i.get('variation_tags', []) automation = i['automation'] env = i['env'] diff --git a/script/get-dataset-cognata-mlcommons/meta.yaml b/script/get-dataset-cognata-mlcommons/meta.yaml index ecb10799a..0b24c75e9 100644 --- a/script/get-dataset-cognata-mlcommons/meta.yaml +++ b/script/get-dataset-cognata-mlcommons/meta.yaml @@ -43,7 +43,7 @@ env: deps: -# Prepare dummy CM cache entry to manage dataset +# Prepare dummy MLC cache entry to manage dataset - names: - custom-cache-entry-mlcommons-cognata-dataset tags: create,custom,cache,entry diff --git a/script/get-dataset-cognata-mlcommons/run.sh b/script/get-dataset-cognata-mlcommons/run.sh index 5563a95da..309cd36ba 100644 --- a/script/get-dataset-cognata-mlcommons/run.sh +++ b/script/get-dataset-cognata-mlcommons/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-dataset-igbh/run.sh b/script/get-dataset-igbh/run.sh index edb705045..ba4a04a7c 100644 --- a/script/get-dataset-igbh/run.sh +++ b/script/get-dataset-igbh/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-dataset-imagenet-train/customize.py b/script/get-dataset-imagenet-train/customize.py index 1fe0b8b43..34241f2c9 100644 --- a/script/get-dataset-imagenet-train/customize.py +++ b/script/get-dataset-imagenet-train/customize.py @@ -27,7 +27,7 @@ def preprocess(i): return {'return': 0} else: - return {'return': 1, 'error': 'Please rerun the last CM command with --env.IMAGENET_TRAIN_PATH={path the folder containing full ImageNet training images} or envoke mlcr "get train dataset imagenet" --input={path to the folder containing ImageNet training images}'} + return {'return': 1, 'error': 'Please rerun the last MLC command with --env.IMAGENET_TRAIN_PATH={path the folder containing full ImageNet training images} or envoke mlcr "get train dataset imagenet" --input={path to the folder containing ImageNet training images}'} elif not os.path.isdir(path): if path.endswith(".tar"): diff --git a/script/get-dataset-imagenet-val/customize.py b/script/get-dataset-imagenet-val/customize.py index 9b4569d15..3d4506131 100644 --- a/script/get-dataset-imagenet-val/customize.py +++ b/script/get-dataset-imagenet-val/customize.py @@ -40,7 +40,7 @@ def preprocess(i): env['MLC_DATASET_IMAGENET_VAL_REQUIRE_DAE'] = 'yes' return {'return': 0} - # return {'return':1, 'error':'Please rerun the last CM command + # return {'return':1, 'error':'Please rerun the last MLC command # with --env.IMAGENET_PATH={path the folder containing full # ImageNet images} or envoke mlcr "get val dataset # imagenet" --input={path to the folder containing ImageNet diff --git a/script/get-dataset-librispeech/meta.yaml b/script/get-dataset-librispeech/meta.yaml index c696b3e25..817c0f119 100644 --- a/script/get-dataset-librispeech/meta.yaml +++ b/script/get-dataset-librispeech/meta.yaml @@ -7,7 +7,7 @@ default_version: dev-clean deps: - names: - sys-utils - tags: get,sys-utils-cm + tags: get,sys-utils-mlc env: MLC_DATASET: LIBRISPEECH MLC_WGET_URL: http://www.openslr.org/resources/12/<<>> diff --git a/script/get-dataset-mlperf-inference-llama3/meta.yaml b/script/get-dataset-mlperf-inference-llama3/meta.yaml index 9e8762f35..24aef9e20 100644 --- a/script/get-dataset-mlperf-inference-llama3/meta.yaml +++ b/script/get-dataset-mlperf-inference-llama3/meta.yaml @@ -36,12 +36,28 @@ variations: MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl MLC_DATASET_TYPE: validation MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + validation,rclone: + env: + MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + validation,r2-downloader: + env: + MLC_RCLONE_URL: "" + MLC_DATASET_FILE_NAME: "" calibration: group: dataset-type env: MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl MLC_DATASET_TYPE: calibration MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + calibration,rclone: + env: + MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + calibration,r2-downloader: + env: + MLC_RCLONE_URL: "" + MLC_DATASET_FILE_NAME: "" rclone: add_deps_recursive: dae: @@ -52,5 +68,13 @@ variations: MLC_DOWNLOAD_URL: <<>> MLC_RCLONE_CONFIG_NAME: mlc-inference group: download-tool + r2-downloader: + add_deps_recursive: + dae: + tags: _r2-downloader + env: + MLC_DOWNLOAD_FILENAME: checkpoint + MLC_RCLONE_CONFIG_NAME: mlc-inference + group: download-tool print_env_at_the_end: MLC_DATASET_LLAMA3_PATH: Path to the dataset diff --git a/script/get-dataset-nuscenes/run.sh b/script/get-dataset-nuscenes/run.sh index abe5a17c7..ca6f48e58 100644 --- a/script/get-dataset-nuscenes/run.sh +++ b/script/get-dataset-nuscenes/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-dataset-squad/meta.yaml b/script/get-dataset-squad/meta.yaml index cc55e3b50..10a34b711 100644 --- a/script/get-dataset-squad/meta.yaml +++ b/script/get-dataset-squad/meta.yaml @@ -5,7 +5,7 @@ cache: true category: AI/ML datasets default_version: '1.1' deps: -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc env: MLC_DATASET: SQUAD new_env_keys: diff --git a/script/get-dataset-waymo/meta.yaml b/script/get-dataset-waymo/meta.yaml index ef56e4a2e..e90315017 100644 --- a/script/get-dataset-waymo/meta.yaml +++ b/script/get-dataset-waymo/meta.yaml @@ -21,22 +21,12 @@ variations: group: download-src default: true prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_waymo - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_WAYMO_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_WAYMO_PATH - MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format extra_cache_tags: waymo,dataset force_cache: true names: @@ -54,6 +44,21 @@ variations: add_deps_recursive: dae: tags: _rclone + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_waymo + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader default: true dry-run: group: run-mode @@ -62,3 +67,12 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + rclone,mlc: + env: + MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/dataset.uri diff --git a/script/get-dataset-waymo/run.sh b/script/get-dataset-waymo/run.sh index ba1412ac9..0af8d4a79 100644 --- a/script/get-dataset-waymo/run.sh +++ b/script/get-dataset-waymo/run.sh @@ -1,14 +1,14 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out #${MLC_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency -if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "true" ]]; then - cd "${MLC_DATASET_WAYMO_PATH}/kitti_format/training" || exit +if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then + cd "${MLC_DATASET_WAYMO_PATH}/training" || exit for f in *.tar.gz; do tar -xzvf "$f"; done cd - || exit fi \ No newline at end of file diff --git a/script/get-dataset-whisper/meta.yaml b/script/get-dataset-whisper/meta.yaml index 12fddf9e4..a436e6ef5 100644 --- a/script/get-dataset-whisper/meta.yaml +++ b/script/get-dataset-whisper/meta.yaml @@ -14,7 +14,7 @@ tests: run_inputs: - variations_list: - rclone,preprocessed,mlc,dry-run - - r2_downloader,preprocessed,mlc,dry-run + - r2-downloader,preprocessed,mlc,dry-run variations: preprocessed: group: dataset-type @@ -45,7 +45,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -59,10 +59,10 @@ variations: tags: _rclone default: true group: download-tool - r2_downloader: + r2-downloader: add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader group: download-tool rclone,preprocessed: env: @@ -95,6 +95,6 @@ variations: update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL - r2_downloader,preprocessed: + r2-downloader,preprocessed: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/whisper-dataset.uri diff --git a/script/get-dlrm-data-mlperf-inference/run.sh b/script/get-dlrm-data-mlperf-inference/run.sh index 180056e2f..58b536f0c 100644 --- a/script/get-dlrm-data-mlperf-inference/run.sh +++ b/script/get-dlrm-data-mlperf-inference/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-gcc/customize.py b/script/get-gcc/customize.py index 25e39a781..9cc1c472a 100644 --- a/script/get-gcc/customize.py +++ b/script/get-gcc/customize.py @@ -83,9 +83,12 @@ def postprocess(i): found_file_path = env['MLC_GCC_BIN_WITH_PATH'] - found_path = os.path.dirname(found_file_path) + # found_path = os.path.dirname(os.path.realpath(found_file_path)) //Need + # to use this if we don't want to cache specific to softlinks + found_path = os.path.dirname(os.path.abspath(found_file_path)) - env['MLC_GCC_INSTALLED_PATH'] = found_path + env['MLC_GCC_INSTALLED_PATH'] = os.path.dirname( + found_path) # /usr in case of /usr/bin/gcc file_name_c = os.path.basename(found_file_path) # G: changed next line to handle cases like gcc-8 diff --git a/script/get-gcc/run.sh b/script/get-gcc/run.sh index e5b397bf6..f6311cbd4 100644 --- a/script/get-gcc/run.sh +++ b/script/get-gcc/run.sh @@ -3,6 +3,6 @@ gcc_bin=${MLC_GCC_BIN_WITH_PATH} echo "${gcc_bin} --version" ${gcc_bin} --version > tmp-ver.out -test $? -eq 0 || exit 1 +test $? -eq 0 || exit $? cat tmp-ver.out diff --git a/script/get-gcc/validate_cache_off.sh b/script/get-gcc/validate_cache_off.sh new file mode 100644 index 000000000..9907116d2 --- /dev/null +++ b/script/get-gcc/validate_cache_off.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [[ -n "$MLC_GCC_DIR_PATH" && -e "$MLC_GCC_DIR_PATH" ]]; then + if [[ "$(realpath "$MLC_GCC_DIR_PATH")" != "$(realpath "$MLC_GCC_INSTALLED_PATH")" ]]; then + echo "$MLC_GCC_DIR_PATH" has changed from the cached entry. Invalidating the cache entry. + exit 1 + fi +fi + +${MLC_GCC_INSTALLED_PATH}/bin/gcc --version > tmp-ver.out +test $? -eq 0 || exit $? diff --git a/script/get-generic-sys-util/README.md b/script/get-generic-sys-util/README.md new file mode 100644 index 000000000..45892d2fb --- /dev/null +++ b/script/get-generic-sys-util/README.md @@ -0,0 +1,137 @@ +# README for get-generic-sys-util +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,sys-util,generic,generic-sys-util +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--fail_safe` | | | `` | +| `--ignore_missing` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Mode + +- `detect` (default) +- `install` + +### Ungrouped + +- `autoconf` +- `bzip2` +- `cmake` +- `crossbuild-essential-arm64` +- `dmidecode` +- `ffmpeg` +- `flex` +- `g++-11` +- `g++-12` +- `g++-9` +- `gcc-11` +- `gcc-9` +- `gflags-dev` +- `git-lfs` +- `glog-dev` +- `ipmitool` +- `libboost-all-dev` +- `libbz2-dev` +- `libev-dev` +- `libffi` +- `libffi-dev` +- `libffi7` +- `libffi8` +- `libgdbm-dev` +- `libgl` +- `libgl1-mesa-glx` +- `libgmock-dev` +- `liblzma-dev` +- `libmkl-dev` +- `libmpfr-dev` +- `libncurses-dev` +- `libnuma-dev` +- `libpci-dev` +- `libpng-dev` +- `libre2-dev` +- `libreadline-dev` +- `libsm6` +- `libsqlite3-dev` +- `libssl-dev` +- `libudev-dev` +- `libxext6` +- `linux-tools` +- `md5sha1sum` +- `ninja-build` +- `nlohmann-json3-dev` +- `ntpdate` +- `numactl` +- `nvidia-cuda-toolkit` +- `pkg-config` +- `postfix` +- `psmisc` +- `rapidjson-dev` +- `rsync` +- `screen` +- `sox` +- `systemd` +- `tk-dev` +- `transmission` +- `unzip` +- `vim-common` +- `wget` +- `wkhtmltopdf` +- `xfonts-base` +- `xz` +- `zlib` +- `zlib1g-dev` diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index d14be0b41..74d023872 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -8,6 +8,7 @@ default_env: MLC_SUDO: sudo deps: - tags: detect,os +- tags: detect,sudo env: MLC_GENERIC_SYS_UTIL_INSTALL_NEEDED: 'no' MLC_SYS_UTIL_VERSION_CMD: '' @@ -573,8 +574,6 @@ variations: dnf: libudev-devl yum: libudev-devel linux-tools: - deps: - - tags: detect,os env: MLC_SYS_UTIL_NAME: linux-tools new_env_keys: @@ -629,6 +628,13 @@ variations: brew: '' dnf: ntpdate yum: ntpdate + crossbuild-essential-arm64: + env: + MLC_SYS_UTIL_NAME: crossbuild-essential-arm64 + MLC_SYS_UTIL_CHECK_CMD: 'command -v aarch64-linux-gnu-gcc >/dev/null 2>&1' + state: + crossbuild-essential-arm64: + apt: crossbuild-essential-arm64 numactl: deps: - enable_if_env: @@ -641,13 +647,13 @@ variations: tags: install,numactl,from.src env: MLC_SYS_UTIL_NAME: numactl - MLC_TMP_VERSION_DETECT_GROUP_NUMBER: 0 + MLC_SYS_UTIL_CHECK_CMD: 'numactl -s' new_env_keys: - MLC_NUMACTL_VERSION state: numactl: apt: numactl - dnf: numactl-devel + dnf: numactl yum: numactl-devel nvidia-cuda-toolkit: env: diff --git a/script/get-git-repo/README.md b/script/get-git-repo/README.md new file mode 100644 index 000000000..f79fe3b3a --- /dev/null +++ b/script/get-git-repo/README.md @@ -0,0 +1,97 @@ +# README for get-git-repo +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,git,repo,repository,clone +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--branch` | | | `` | +| `--depth` | | | `--depth 4` | +| `--env_key` | | | `` | +| `--folder` | | | `repo` | +| `--patch` | | | `no` | +| `--pull` | | | `` | +| `--submodules` | | | ` --recurse-submodules` | +| `--update` | Alias for pull | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Checkout + +- `branch.#` _(# can be substituted dynamically)_ +- `tag.#` _(# can be substituted dynamically)_ + +### Git-history + +- `full-history` +- `short-history` (default) + +### Post-checkout + +- `sha.#` _(# can be substituted dynamically)_ + +### Repo + +- `repo.#` _(# can be substituted dynamically)_ + +### Ungrouped + +- `cherrypicks.#` _(# can be substituted dynamically)_ +- `lfs` +- `no-recurse-submodules` +- `patch` +- `patch.#` _(# can be substituted dynamically)_ +- `pr-to-apply.#` _(# can be substituted dynamically)_ +- `submodules.#` _(# can be substituted dynamically)_ diff --git a/script/get-git-repo/meta.yaml b/script/get-git-repo/meta.yaml index 468468bb5..e52110035 100644 --- a/script/get-git-repo/meta.yaml +++ b/script/get-git-repo/meta.yaml @@ -67,6 +67,11 @@ variations: patch: env: MLC_GIT_PATCH: 'yes' + patch.#: + env: + MLC_GIT_PATCH: 'yes' + +,MLC_GIT_PATCH_FILEPATHS: + - '#' pr-to-apply.#: env: MLC_GIT_PR_TO_APPLY: '#' diff --git a/script/get-ipol-src/meta.yaml b/script/get-ipol-src/meta.yaml index ac64d8971..92e388799 100644 --- a/script/get-ipol-src/meta.yaml +++ b/script/get-ipol-src/meta.yaml @@ -13,8 +13,10 @@ extra_cache_tags_from_env: - env: MLC_IPOL_YEAR prefix: year- input_description: - number: IPOL publication number - year: IPOL publication year + number: + desc: IPOL publication number + year: + desc: IPOL publication year input_mapping: number: MLC_IPOL_NUMBER year: MLC_IPOL_YEAR diff --git a/script/get-lib-jemalloc/README.md b/script/get-lib-jemalloc/README.md index f61ba4247..f6b90d0f5 100644 --- a/script/get-lib-jemalloc/README.md +++ b/script/get-lib-jemalloc/README.md @@ -1,6 +1,35 @@ # README for get-lib-jemalloc -This README is automatically generated. Add custom content in [info.txt](info.txt). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. ## Run Commands ```bash @@ -26,3 +55,21 @@ No script specific inputs | `--gh_token` | Github Token | | `` | | `--hf_token` | Huggingface Token | | `` | | `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Ungrouped + +- `branch.#` _(# can be substituted dynamically)_ +- `config.#` _(# can be substituted dynamically)_ +- `enable-prof` +- `enable-stats` +- `lg-hugepage.#` _(# can be substituted dynamically)_ +- `lg-page.#` _(# can be substituted dynamically)_ +- `lg-quantum.#` _(# can be substituted dynamically)_ +- `sha.#` _(# can be substituted dynamically)_ +- `version.official` (base: url.official) + +### Version + +- `url.#` _(# can be substituted dynamically)_ +- `url.official` (default) diff --git a/script/get-lib-jemalloc/customize.py b/script/get-lib-jemalloc/customize.py index 0fb3b7de8..4771bc813 100644 --- a/script/get-lib-jemalloc/customize.py +++ b/script/get-lib-jemalloc/customize.py @@ -1,3 +1,4 @@ +from utils import * from mlc import utils import os import subprocess @@ -15,6 +16,13 @@ def preprocess(i): configure_command += f""" --with-lg-quantum={env['MLC_JEMALLOC_LG_QUANTUM']} """ if env.get('MLC_JEMALLOC_LG_PAGE', '') != '': configure_command += f""" --with-lg-page={env['MLC_JEMALLOC_LG_PAGE']} """ + + if is_true(env.get('MLC_JEMALLOC_STATS')): + configure_command += " --enable-stats " + + if is_true(env.get('MLC_JEMALLOC_PROF')): + configure_command += " --enable-prof " + if env.get('MLC_JEMALLOC_CONFIG', '') != '': configure_command += f""" {env['MLC_JEMALLOC_CONFIG'].replace("'", "")} """ diff --git a/script/get-lib-jemalloc/meta.yaml b/script/get-lib-jemalloc/meta.yaml index a856e9e4b..10c73be91 100644 --- a/script/get-lib-jemalloc/meta.yaml +++ b/script/get-lib-jemalloc/meta.yaml @@ -65,9 +65,18 @@ variations: config.#: env: MLC_JEMALLOC_CONFIG: '#' + enable-stats: + env: + MLC_JEMALLOC_STATS: true + enable-prof: + env: + MLC_JEMALLOC_PROF: true lg-page.#: env: MLC_JEMALLOC_LG_PAGE: '#' + lg-hugepage.#: + env: + MLC_JEMALLOC_LG_PAGE: '#' lg-quantum.#: env: MLC_JEMALLOC_LG_QUANTUM: '#' diff --git a/script/get-ml-model-abtf-ssd-pytorch/README.md b/script/get-ml-model-abtf-ssd-pytorch/README.md index 3ef5cbbdc..6a6e5d631 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/README.md +++ b/script/get-ml-model-abtf-ssd-pytorch/README.md @@ -1,10 +1,10 @@ # README for get-ml-model-abtf-ssd-pytorch This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/get-ml-model-deeplabv3_plus/README.md b/script/get-ml-model-deeplabv3_plus/README.md index 90f6525bc..438f35648 100644 --- a/script/get-ml-model-deeplabv3_plus/README.md +++ b/script/get-ml-model-deeplabv3_plus/README.md @@ -1,10 +1,10 @@ # README for get-ml-model-deeplabv3-plus This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/get-ml-model-gptj/meta.yaml b/script/get-ml-model-gptj/meta.yaml index b54e3adeb..e20408f11 100644 --- a/script/get-ml-model-gptj/meta.yaml +++ b/script/get-ml-model-gptj/meta.yaml @@ -154,7 +154,6 @@ variations: - names: - cuda tags: get,cuda - - tags: get,nvidia,scratch,space - tags: get,cuda-devices,_with-pycuda enable_if_env: MLC_HOST_OS_FLAVOR: @@ -163,6 +162,7 @@ variations: skip_if_env: MLC_HOST_OS_FLAVOR: - ubuntu + - tags: get,nvidia,scratch,space - env: {} force_new_env_keys: - GPTJ_CHECKPOINT_PATH diff --git a/script/get-ml-model-llama2/customize.py b/script/get-ml-model-llama2/customize.py index 494b7c9cf..491e76248 100644 --- a/script/get-ml-model-llama2/customize.py +++ b/script/get-ml-model-llama2/customize.py @@ -29,12 +29,6 @@ def preprocess(i): else: if path == '' or not os.path.exists(path): env['MLC_TMP_REQUIRE_DOWNLOAD'] = 'yes' - if env['MLC_DOWNLOAD_SRC'] == "mlcommons": - i['run_script_input']['script_name'] = 'run-rclone' - if env.get('MLC_OUTDIRNAME', '') != '': - env['LLAMA2_CHECKPOINT_PATH'] = env['MLC_OUTDIRNAME'] - else: - env['LLAMA2_CHECKPOINT_PATH'] = os.getcwd() return {'return': 0} @@ -42,11 +36,12 @@ def preprocess(i): def postprocess(i): env = i['env'] - if env.get('LLAMA2_CHECKPOINT_PATH', '') == '': - env['LLAMA2_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] - else: - env['MLC_ML_MODEL_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] - env['MLC_ML_MODEL_LLAMA2_FILE_WITH_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_PATH'] + if env.get('MLC_DOWNLOAD_MODE', '') != "dry": + if env.get('LLAMA2_CHECKPOINT_PATH', '') == '': + env['LLAMA2_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] + else: + env['MLC_ML_MODEL_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] + env['MLC_ML_MODEL_LLAMA2_FILE_WITH_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_PATH'] return {'return': 0} diff --git a/script/get-ml-model-llama2/meta.yaml b/script/get-ml-model-llama2/meta.yaml index d0e14cf77..a2b0c3c1e 100644 --- a/script/get-ml-model-llama2/meta.yaml +++ b/script/get-ml-model-llama2/meta.yaml @@ -41,6 +41,12 @@ tags: - llama2-70b - text-summarization uid: 5db97be9f61244c6 +tests: + needs_pat: true + run_inputs: + - variations_list: + - r2-downloader,70b,mlc,dry-run + - r2-downloader,7b,mlc,dry-run variations: L40s: env: @@ -84,11 +90,11 @@ variations: MLC_ML_MODEL_PRECISION: int8 MLC_ML_MODEL_WEIGHT_DATA_TYPES: int8 group: precision - mlc: - group: download-source - default: true - env: - MLC_DOWNLOAD_SRC: mlcommons + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone prehook_deps: - tags: get,rclone enable_if_env: @@ -99,6 +105,55 @@ variations: enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - yes + r2-downloader: + group: download-tool + default: true + add_deps_recursive: + dae: + tags: _r2-downloader + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + mlc: + group: download-source + default: true + env: + MLC_DOWNLOAD_SRC: mlcommons + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: LLAMA2_CHECKPOINT_PATH + MLC_EXTRACT_FINAL_ENV_NAME: LLAMA2_CHECKPOINT_PATH + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + mlc,rclone,70b: + env: + MLC_DOWNLOAD_URL: mlc-llama2:Llama-2-70b-chat-hf + mlc,rclone,7b: + env: + MLC_DOWNLOAD_URL: mlc-llama2:Llama-2-7b-chat-hf + mlc,r2-downloader,70b: + env: + MLC_DOWNLOAD_URL: https://llama2.mlcommons-storage.org/metadata/llama-2-70b-chat-hf.uri + mlc,r2-downloader,7b: + env: + MLC_DOWNLOAD_URL: https://llama2.mlcommons-storage.org/metadata/llama-2-7b-chat-hf.uri hf: group: download-source env: diff --git a/script/get-ml-model-llama2/run-rclone.sh b/script/get-ml-model-llama2/run-rclone.sh deleted file mode 100644 index 1fc602a9f..000000000 --- a/script/get-ml-model-llama2/run-rclone.sh +++ /dev/null @@ -1,4 +0,0 @@ -cmd="rclone sync mlc-llama2:${MLC_GIT_CHECKOUT_FOLDER} ${LLAMA2_CHECKPOINT_PATH}/${MLC_GIT_CHECKOUT_FOLDER} -P" -echo $cmd -eval $cmd -test $? -eq 0 || exit $? diff --git a/script/get-ml-model-llama3/meta.yaml b/script/get-ml-model-llama3/meta.yaml index fd79fdc33..b10665d20 100644 --- a/script/get-ml-model-llama3/meta.yaml +++ b/script/get-ml-model-llama3/meta.yaml @@ -38,8 +38,8 @@ tests: run_inputs: - variations_list: - rclone,405b,mlc,dry-run - - r2_downloader,405b,mlc,dry-run - - r2_downloader,8b,mlc,dry-run + - r2-downloader,405b,mlc,dry-run + - r2-downloader,8b,mlc,dry-run variations: fp16: default: true @@ -59,6 +59,27 @@ variations: env: MLC_ML_MODEL_NAME: Llama-3.1-8b-Instruct MLC_ML_MODEL_R2_HOSTED_NAME: llama3-1-8b-instruct + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_mlperf-llama3-1 + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + default: true + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader mlc: group: download-src default: true @@ -72,6 +93,7 @@ variations: force_cache: true names: - dae + - llama3-model-download tags: download-and-extract force_env_keys: - MLC_OUTDIRNAME @@ -84,35 +106,14 @@ variations: env: MLC_DOWNLOAD_URL: mlc-llama3-1:inference/<<>> adr: - dae: + llama3-model-download: extra_cache_tags: llama3,dataset,rclone - mlc,r2_downloader: + mlc,r2-downloader: env: MLC_DOWNLOAD_URL: https://llama3-1.mlcommons-storage.org/metadata/<<>>.uri adr: - dae: + llama3-model-download: extra_cache_tags: llama3,dataset,rclone - rclone: - group: download-tool - add_deps_recursive: - dae: - tags: _rclone - prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_mlperf-llama3-1 - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - default: true - r2_downloader: - group: download-tool - add_deps_recursive: - dae: - tags: _r2_downloader dry-run: group: run-mode env: @@ -120,7 +121,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x hf: diff --git a/script/get-ml-model-pointpainting/customize.py b/script/get-ml-model-pointpainting/customize.py index 07b18602d..7e617fca5 100644 --- a/script/get-ml-model-pointpainting/customize.py +++ b/script/get-ml-model-pointpainting/customize.py @@ -12,9 +12,9 @@ def preprocess(i): return {'return': 1, 'error': 'Script not supported in windows yet!'} if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') != '': - if not os.path.exists(env['MLC_ML_MODEL_POINT_PAINTING']): + if not os.path.exists(env['MLC_ML_MODEL_POINT_PAINTING_PATH']): return { - 'return': 1, 'error': f"Provided model path {env['MLC_ML_MODEL_POINT_PAINTING']} does not exist."} + 'return': 1, 'error': f"Provided model path {env['MLC_ML_MODEL_POINT_PAINTING_PATH']} does not exist."} if env.get('MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') != '': if not os.path.exists(env['MLC_ML_MODEL_DPLAB_RESNET50_PATH']): @@ -24,13 +24,6 @@ def preprocess(i): if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') == '' or env.get( 'MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" - if env['MLC_DOWNLOAD_SRC'] == "mlcommons": - i['run_script_input']['script_name'] = 'run-rclone' - if env.get('MLC_OUTDIRNAME', '') != '': - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'] = env['MLC_OUTDIRNAME'] - else: - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'] = os.path.join( - os.getcwd(), "model") return {'return': 0} @@ -39,21 +32,22 @@ def postprocess(i): env = i['env'] - if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') == '': - if env['MLC_ML_MODEL_PP_FORMAT'] == "onnx": - env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp.onnx") - else: - env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp_ep36.pth") - - if env.get('MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': - if env['MLC_ML_MODEL_DPLAB_RESNET50_FORMAT'] == "onnx": - env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "deeplabv3+.onnx") - else: - env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], - "best_deeplabv3plus_resnet50_waymo_os16.pth") + if env.get('MLC_DOWNLOAD_MODE', '') != "dry": + if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') == '': + if env['MLC_ML_MODEL_PP_FORMAT'] == "onnx": + env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp.onnx") + else: + env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp_ep36.pth") + + if env.get('MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': + if env['MLC_ML_MODEL_DPLAB_RESNET50_FORMAT'] == "onnx": + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "deeplabv3+.onnx") + else: + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], + "best_deeplabv3plus_resnet50_waymo_os16.pth") return {'return': 0} diff --git a/script/get-ml-model-pointpainting/meta.yaml b/script/get-ml-model-pointpainting/meta.yaml index b811d58d6..8047b6188 100644 --- a/script/get-ml-model-pointpainting/meta.yaml +++ b/script/get-ml-model-pointpainting/meta.yaml @@ -27,9 +27,11 @@ variations: env: MLC_ML_MODEL_PP_FORMAT: onnx MLC_ML_MODEL_DPLAB_RESNET50_FORMAT: onnx - mlc: - group: download-src - default: true + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone prehook_deps: - tags: get,rclone enable_if_env: @@ -40,6 +42,47 @@ variations: enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - yes + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader + default: true + mlc: + group: download-src + default: true + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_POINT_PAINTING_TMP_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_POINT_PAINTING_TMP_PATH + extra_cache_tags: waymo,dataset + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL env: MLC_DOWNLOAD_SRC: mlcommons - + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + rclone,mlc: + env: + MLC_DOWNLOAD_URL: mlc-waymo:waymo_preprocessed_dataset/model + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/model.uri diff --git a/script/get-ml-model-pointpainting/run-rclone.sh b/script/get-ml-model-pointpainting/run-rclone.sh deleted file mode 100644 index 9b76a1511..000000000 --- a/script/get-ml-model-pointpainting/run-rclone.sh +++ /dev/null @@ -1,4 +0,0 @@ -cmd="rclone sync mlc-waymo:waymo_preprocessed_dataset/model ${MLC_ML_MODEL_POINT_PAINTING_TMP_PATH} -P" -echo $cmd -eval $cmd -test $? -eq 0 || exit $? \ No newline at end of file diff --git a/script/get-ml-model-pointpainting/run.sh b/script/get-ml-model-pointpainting/run.sh index 3197bb8ad..238a89269 100644 --- a/script/get-ml-model-pointpainting/run.sh +++ b/script/get-ml-model-pointpainting/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-ml-model-resnet50/README.md b/script/get-ml-model-resnet50/README.md new file mode 100644 index 000000000..13012c78b --- /dev/null +++ b/script/get-ml-model-resnet50/README.md @@ -0,0 +1,90 @@ +# README for get-ml-model-resnet50 +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,raw,ml-model,resnet50,ml-model-resnet50,image-classification +``` + +No script specific inputs +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Framework + +- `ncnn` +- `onnx` (alias: onnxruntime) (default) +- `pytorch` +- `tensorflow` (alias: tf) +- `tflite` + +### Model-output + +- `argmax` (default) +- `no-argmax` + +### Opset-version + +- `opset-11` +- `opset-8` + +### Precision + +- `fp32` (default) +- `int8` +- `uint8` + +### Ungrouped + +- `batch_size.#` _(# can be substituted dynamically)_ +- `batch_size.1` +- `fix-input-shape` +- `from-tf` +- `huggingface_default` diff --git a/script/get-ml-model-resnet50/meta.yaml b/script/get-ml-model-resnet50/meta.yaml index a752747ff..69ab60401 100644 --- a/script/get-ml-model-resnet50/meta.yaml +++ b/script/get-ml-model-resnet50/meta.yaml @@ -132,12 +132,12 @@ variations: env: MLC_DOWNLOAD_CHECKSUM: f6a4da60cd5f084d97efc2c1ddb10beb MLC_PACKAGE_URL: https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx - MLC_PACKAGE_URL1: https://zenodo.org/record/4735647/files/resnet50_v1.onnx + MLC_DOWNLOAD_URL1: https://zenodo.org/record/4735647/files/resnet50_v1.onnx onnx,opset-8: env: MLC_DOWNLOAD_CHECKSUM: a638cf028b5870da29e09ccc2f7182e7 MLC_PACKAGE_URL: https://armi.in/files/resnet50_v1.onnx - MLC_PACKAGE_URL1: https://zenodo.org/record/2592612/files/resnet50_v1.onnx + MLC_DOWNLOAD_URL1: https://zenodo.org/record/2592612/files/resnet50_v1.onnx onnxruntime: alias: onnx opset-11: diff --git a/script/get-ml-model-retinanet/run-no-nms.sh b/script/get-ml-model-retinanet/run-no-nms.sh index 82e546f62..aa0d60151 100644 --- a/script/get-ml-model-retinanet/run-no-nms.sh +++ b/script/get-ml-model-retinanet/run-no-nms.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-ml-model-whisper/customize.py b/script/get-ml-model-whisper/customize.py index 1a1ef7555..0eb19e75e 100644 --- a/script/get-ml-model-whisper/customize.py +++ b/script/get-ml-model-whisper/customize.py @@ -12,8 +12,26 @@ def preprocess(i): if os_info['platform'] == "windows": return {'return': 1, 'error': 'Script not supported in windows yet!'} - if env.get('MLC_ML_MODEL_WHISPER_PATH', '') == '': + checkpoint_path = env.get('MLC_ML_MODEL_WHISPER_PATH', '').strip() + + if checkpoint_path == '': env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + else: + # Normalize and expand the path + checkpoint_path = os.path.abspath(os.path.expanduser(checkpoint_path)) + env['MLC_ML_MODEL_WHISPER_PATH'] = checkpoint_path + + if not os.path.exists(checkpoint_path): + return { + 'return': 1, + 'error': f"Provided Whisper model path '{checkpoint_path}' does not exist." + } + + if not os.path.isdir(checkpoint_path): + return { + 'return': 1, + 'error': f"Provided Whisper model path '{checkpoint_path}' is not a directory." + } return {'return': 0} diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index bd7f4eaef..c5cc34084 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -2,6 +2,12 @@ alias: get-ml-model-whisper automation_alias: script automation_uid: 5b4e0237da074764 cache: true +category: AI/ML models +env: + MLC_ML_MODEL_WEIGHT_TRANSFORMATIONS: 'no' + MLC_ML_MODEL_DATASET: whisper +input_mapping: + checkpoint: MLC_ML_MODEL_WHISPER_PATH new_env_keys: - MLC_ML_MODEL_WHISPER_PATH - MLC_ML_MODEL_FILE_WITH_PATH @@ -12,13 +18,62 @@ tags: - get - ml-model - whisper +- speech-recognition tests: run_inputs: - variations_list: - rclone,mlc,dry-run - - r2_downloader,mlc,dry-run + - r2-downloader,mlc,dry-run + # - huggingface + uid: 3bea2356e97f47b1 + +prehook_deps: +- enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + MLC_DOWNLOAD_SRC: + - 'huggingface' + env: {} + extra_cache_tags: whisper + force_env_keys: + - MLC_GIT_CHECKOUT_FOLDER + names: + - hf-zoo + tags: get,ml-model,huggingface,zoo,_clone-repo + variations: + fp32: + default: true + env: + MLC_ML_MODEL_INPUT_DATA_TYPES: fp32 + MLC_ML_MODEL_PRECISION: fp32 + MLC_ML_MODEL_WEIGHT_DATA_TYPES: fp32 + group: precision + + huggingface: + group: download-src + env: + MLC_DOWNLOAD_SRC: huggingface + + large-v3: + env: + MLC_GIT_CHECKOUT_FOLDER: whisper-large-v3 + group: model-size + default: true + default_variations: + huggingface-stub: openai/whisper-large-v3 + + openai/whisper-large-v3: + base: + - large-v3 + adr: + hf-zoo: + tags: _model-stub.openai/whisper-large-v3 + env: + MLC_MODEL_ZOO_ENV_KEY: WHISPER + group: huggingface-stub + dry-run: env: MLC_DOWNLOAD_MODE: dry @@ -26,7 +81,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -46,35 +101,35 @@ variations: force_env_keys: - MLC_OUTDIRNAME names: - - dae + - whisper-model-dae tags: download-and-extract update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL rclone: - add_deps_recursive: - dae: + adr: + whisper-model-dae: tags: _rclone env: MLC_DOWNLOAD_URL: 'mlc-inference:mlcommons-inference-wg-public/Whisper/model/' prehook_deps: - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - true - tags: get,rclone - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - true - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC - force_cache: true - tags: get,rclone-config,_mlc-inference + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - true + tags: get,rclone + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - true + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC + force_cache: true + tags: get,rclone-config,_mlc-inference default: true group: download-tool - r2_downloader: - add_deps_recursive: - dae: - tags: _r2_downloader + r2-downloader: + adr: + whisper-model-dae: + tags: _r2-downloader env: MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' - group: download-tool \ No newline at end of file + group: download-tool diff --git a/script/get-mlperf-automotive-scratch-space/run.sh b/script/get-mlperf-automotive-scratch-space/run.sh index 821adb3f9..ee74b85b6 100644 --- a/script/get-mlperf-automotive-scratch-space/run.sh +++ b/script/get-mlperf-automotive-scratch-space/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-mlperf-inference-intel-scratch-space/run.sh b/script/get-mlperf-inference-intel-scratch-space/run.sh index 4260c33a9..52d73d4c1 100644 --- a/script/get-mlperf-inference-intel-scratch-space/run.sh +++ b/script/get-mlperf-inference-intel-scratch-space/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-mlperf-inference-nvidia-common-code/meta.yaml b/script/get-mlperf-inference-nvidia-common-code/meta.yaml index e5d4a42dc..76fa3b666 100644 --- a/script/get-mlperf-inference-nvidia-common-code/meta.yaml +++ b/script/get-mlperf-inference-nvidia-common-code/meta.yaml @@ -41,6 +41,10 @@ variations: adr: mlperf-inference-results: tags: _nvidia-only + v4.0: + add_deps_recursive: + mlperf-inference-results: + tags: _code-only,_v4.0 v5.0: add_deps_recursive: mlperf-inference-results: diff --git a/script/get-mlperf-inference-nvidia-scratch-space/run.sh b/script/get-mlperf-inference-nvidia-scratch-space/run.sh index 4260c33a9..52d73d4c1 100644 --- a/script/get-mlperf-inference-nvidia-scratch-space/run.sh +++ b/script/get-mlperf-inference-nvidia-scratch-space/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-mlperf-inference-results/meta.yaml b/script/get-mlperf-inference-results/meta.yaml index d3b886de3..f669e7477 100644 --- a/script/get-mlperf-inference-results/meta.yaml +++ b/script/get-mlperf-inference-results/meta.yaml @@ -69,6 +69,12 @@ variations: GITHUB_REPO_OWNER: GATEOverflow NVIDIA_ONLY: 'yes' group: source-repo + v4.0: + group: version + env: + MLC_GIT_URL: https://github.com/<<>>/inference_results_v4.0.git + MLC_MLPERF_INFERENCE_RESULTS_VERSION_NAME: v4.0 + MLC_VERSION: "v4.0" v5.0: group: version env: diff --git a/script/get-oneapi/README.md b/script/get-oneapi/README.md index 7d8dedc9c..84cc32210 100644 --- a/script/get-oneapi/README.md +++ b/script/get-oneapi/README.md @@ -1,13 +1,46 @@ # README for get-one-api -This README is automatically generated. Add custom content in [info.txt](info.txt). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. ## Run Commands ```bash mlcr get,oneapi,compiler,get-oneapi ``` -No script specific inputs +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--oneapi_dir` | | | `` | ### Generic Script Inputs | Name | Description | Choices | Default | @@ -26,3 +59,9 @@ No script specific inputs | `--gh_token` | Github Token | | `` | | `--hf_token` | Huggingface Token | | `` | | `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Ungrouped + +- `fortran` +- `path.#` _(# can be substituted dynamically)_ diff --git a/script/get-oneapi/install.sh b/script/get-oneapi/install.sh index 54f9178b2..a299a37f3 100644 --- a/script/get-oneapi/install.sh +++ b/script/get-oneapi/install.sh @@ -28,8 +28,8 @@ CMD="bash ./${MLC_ONEAPI_INSTALL_FILENAME} -a --silent --cli --eula accept --in run "$CMD" if [[ ${MLC_ONEAPI_FORTRAN} == 'yes' ]] then - wget -nc https://registrationcenter-download.intel.com/akdlm/IRC_NAS/2238465b-cfc7-4bf8-ad04-e55cb6577cba/intel-fortran-essentials-2025.1.1.8_offline.sh + wget -nc ${MLC_ONEAPI_FORTRAN_COMPILER_URL_BASE}/${MLC_ONEAPI_FORTRAN_COMPILER_FILENAME} test $? -eq 0 || exit $? - CMD="bash ./intel-fortran-essentials-2025.1.1.8_offline.sh -a --silent --cli --eula accept --install-dir ${PWD}/install" + CMD="bash ./${MLC_ONEAPI_FORTRAN_COMPILER_FILENAME} -a --silent --cli --eula accept --install-dir ${PWD}/install" run "$CMD" fi diff --git a/script/get-oneapi/meta.yaml b/script/get-oneapi/meta.yaml index 69bcd435f..cec4b7aa5 100644 --- a/script/get-oneapi/meta.yaml +++ b/script/get-oneapi/meta.yaml @@ -4,7 +4,7 @@ automation_uid: 5b4e0237da074764 cache: true category: Compiler automation clean_files: [] -default_version: 2025.1.1 +default_version: 2025.2.1 deps: - tags: detect,os name: Detect or install OneAPI compiler @@ -35,13 +35,31 @@ tests: - fortran versions: - 2025.1.1: + 2025.2.1: + env: + MLC_ONEAPI_INSTALL_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/3b7a16b3-a7b0-460f-be16-de0d64fa6b1e + MLC_ONEAPI_INSTALL_FILENAME: intel-oneapi-base-toolkit-2025.2.1.44_offline.sh + MLC_ONEAPI_FORTRAN_COMPILER_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a31a1a8e-4b6f-4548-a554-fe3ac1c8b230 + MLC_ONEAPI_FORTRAN_COMPILER_FILENAME: intel-fortran-essentials-2025.2.1.5_offline.sh + MLC_ONEAPI_INSTALL_VERSION_PREFIX: '2025.2' + MLC_VERSION: '2025.2.1' + + 2025.2.0: + env: + MLC_ONEAPI_INSTALL_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/bd1d0273-a931-4f7e-ab76-6a2a67d646c7 + MLC_ONEAPI_INSTALL_FILENAME: intel-oneapi-base-toolkit-2025.2.0.592.sh + MLC_ONEAPI_INSTALL_VERSION_PREFIX: '2025.2' + MLC_VERSION: '2025.2.0' + + 2025.1.0: env: MLC_ONEAPI_INSTALL_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/6bfca885-4156-491e-849b-1cd7da9cc760 MLC_ONEAPI_INSTALL_FILENAME: intel-oneapi-base-toolkit-2025.1.1.36_offline.sh + MLC_ONEAPI_FORTRAN_COMPILER_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/2238465b-cfc7-4bf8-ad04-e55cb6577cba + MLC_ONEAPI_FORTRAN_COMPILER_FILENAME: intel-fortran-essentials-2025.1.1.8_offline.sh MLC_ONEAPI_INSTALL_VERSION_PREFIX: '2025.1' MLC_VERSION: '2025.1.1' - + variations: path.#: env: diff --git a/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml b/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml index 8c0da0989..0cc5f359c 100644 --- a/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml +++ b/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml @@ -28,18 +28,18 @@ variations: validation,rclone: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/deepseek_r1/mlperf_deepseek_r1_dataset_4388_fp8_eval.pkl - validation,r2_downloader: + validation,r2-downloader: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/deepseek-r1-dataset-4388-fp8-eval.uri calibration: group: dataset-type env: MLC_PREPROCESSED_DATASET_TYPE: calibration - r2_downloader: + r2-downloader: group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader rclone: group: download-tool add_deps_recursive: @@ -66,7 +66,7 @@ variations: calibration,rclone: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/deepseek_r1/mlperf_deepseek_r1_calibration_dataset_500_fp8_eval.pkl - calibration,r2_downloader: + calibration,r2-downloader: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/deepseek-r1-calibration-dataset-500-fp8-eval.uri mlc: @@ -94,8 +94,8 @@ variations: tests: run_inputs: - variations_list: - - calibration,r2_downloader,mlc,dry-run - - validation,r2_downloader,mlc,dry-run + - calibration,r2-downloader,mlc,dry-run + - validation,r2-downloader,mlc,dry-run # - validation,rclone,mlc,dry-run # - calibration,rclone,mlc,dry-run diff --git a/script/get-preprocessed-dataset-openorca/meta.yaml b/script/get-preprocessed-dataset-openorca/meta.yaml index 8ddb5f049..5e9f1a83d 100644 --- a/script/get-preprocessed-dataset-openorca/meta.yaml +++ b/script/get-preprocessed-dataset-openorca/meta.yaml @@ -7,7 +7,7 @@ category_sort: 8500 default_env: MLC_DATASET_CALIBRATION: 'no' deps: -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - names: - python - python3 diff --git a/script/get-preprocessed-dataset-squad/run-packed.sh b/script/get-preprocessed-dataset-squad/run-packed.sh index f42dedcf8..6fd16c7a2 100644 --- a/script/get-preprocessed-dataset-squad/run-packed.sh +++ b/script/get-preprocessed-dataset-squad/run-packed.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-preprocessed-dataset-squad/run.sh b/script/get-preprocessed-dataset-squad/run.sh index 32c02c034..79e8a78d5 100644 --- a/script/get-preprocessed-dataset-squad/run.sh +++ b/script/get-preprocessed-dataset-squad/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-python3/meta.yaml b/script/get-python3/meta.yaml index b187346d2..434a00f0e 100644 --- a/script/get-python3/meta.yaml +++ b/script/get-python3/meta.yaml @@ -21,9 +21,6 @@ prehook_deps: inherit_variation_tags: true reuse_version: true tags: install,python,src -print_env_at_the_end: - MLC_PYTHON_BIN_WITH_PATH: Path to Python - MLC_PYTHON_VERSION: Python version tags: - get - python diff --git a/script/get-rclone-config/run.sh b/script/get-rclone-config/run.sh index 32cf4d51e..4a22ce805 100644 --- a/script/get-rclone-config/run.sh +++ b/script/get-rclone-config/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-rclone/customize.py b/script/get-rclone/customize.py index 91fa9b685..eebd76c71 100644 --- a/script/get-rclone/customize.py +++ b/script/get-rclone/customize.py @@ -146,7 +146,7 @@ def postprocess(i): cur_dir = os.getcwd() path_bin = os.path.join(cur_dir, file_name) if os.path.isfile(path_bin): - # Was downloaded and extracted by CM + # Was downloaded and extracted by MLC env['MLC_RCLONE_BIN_WITH_PATH'] = path_bin env['+PATH'] = [cur_dir] diff --git a/script/get-rclone/meta.yaml b/script/get-rclone/meta.yaml index b0727d484..75403386a 100644 --- a/script/get-rclone/meta.yaml +++ b/script/get-rclone/meta.yaml @@ -27,4 +27,4 @@ variations: env: MLC_RCLONE_SYSTEM: 'yes' warnings: - - This CM script will install rclone using sudo/brew! + - This MLC script will install rclone using sudo/brew! diff --git a/script/get-spec-ptd/meta.yaml b/script/get-spec-ptd/meta.yaml index f642f7053..8c6087600 100644 --- a/script/get-spec-ptd/meta.yaml +++ b/script/get-spec-ptd/meta.yaml @@ -24,7 +24,8 @@ deps: - MLC_GIT_* tags: get,git,repo,_repo.https://github.com/mlcommons/power input_description: - input: Path to SPEC PTDaemon (Optional) + input: + desc: Path to SPEC PTDaemon (Optional) input_mapping: input: MLC_INPUT new_env_keys: diff --git a/script/get-sys-utils-cm/do_pip_installs.sh.old b/script/get-sys-utils-cm/do_pip_installs.sh.old deleted file mode 100644 index 441f884dc..000000000 --- a/script/get-sys-utils-cm/do_pip_installs.sh.old +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -PIP_EXTRA=`python3 -c "import pkg_resources; print(' --break-system-packages ' if int(pkg_resources.get_distribution('pip').version.split('.')[0]) >= 23 else '')"` -cmd="python3 -m pip install -r ${MLC_TMP_CURRENT_SCRIPT_PATH}/requirements.txt ${MLC_PYTHON_PIP_USER} ${MLC_PYTHON_PIP_COMMON_EXTRA} ${PIP_EXTRA}" -echo $cmd -eval $cmd diff --git a/script/get-sys-utils-cm/COPYRIGHT.md b/script/get-sys-utils-mlc/COPYRIGHT.md similarity index 100% rename from script/get-sys-utils-cm/COPYRIGHT.md rename to script/get-sys-utils-mlc/COPYRIGHT.md diff --git a/script/get-sys-utils-cm/customize.py b/script/get-sys-utils-mlc/customize.py similarity index 99% rename from script/get-sys-utils-cm/customize.py rename to script/get-sys-utils-mlc/customize.py index ed72dac02..7a5c4ff83 100644 --- a/script/get-sys-utils-cm/customize.py +++ b/script/get-sys-utils-mlc/customize.py @@ -87,7 +87,7 @@ def preprocess(i): logger.info( '***********************************************************************') logger.info( - 'This script will attempt to install minimal system dependencies for CM.') + 'This script will attempt to install minimal system dependencies for MLC.') logger.info('Note that you may be asked for your SUDO password ...') logger.info( '***********************************************************************') diff --git a/script/get-sys-utils-cm/do_pip_installs.sh b/script/get-sys-utils-mlc/do_pip_installs.sh similarity index 100% rename from script/get-sys-utils-cm/do_pip_installs.sh rename to script/get-sys-utils-mlc/do_pip_installs.sh diff --git a/script/get-sys-utils-cm/meta.yaml b/script/get-sys-utils-mlc/meta.yaml similarity index 95% rename from script/get-sys-utils-cm/meta.yaml rename to script/get-sys-utils-mlc/meta.yaml index 83e39e9a4..dfce36f85 100644 --- a/script/get-sys-utils-cm/meta.yaml +++ b/script/get-sys-utils-mlc/meta.yaml @@ -1,4 +1,4 @@ -alias: get-sys-utils-cm +alias: get-sys-utils-mlc uid: bc90993277e84b8e automation_alias: script diff --git a/script/get-sys-utils-cm/requirements.txt b/script/get-sys-utils-mlc/requirements.txt similarity index 100% rename from script/get-sys-utils-cm/requirements.txt rename to script/get-sys-utils-mlc/requirements.txt diff --git a/script/get-sys-utils-cm/run-arch.sh b/script/get-sys-utils-mlc/run-arch.sh similarity index 100% rename from script/get-sys-utils-cm/run-arch.sh rename to script/get-sys-utils-mlc/run-arch.sh diff --git a/script/get-sys-utils-cm/run-debian.sh b/script/get-sys-utils-mlc/run-debian.sh similarity index 100% rename from script/get-sys-utils-cm/run-debian.sh rename to script/get-sys-utils-mlc/run-debian.sh diff --git a/script/get-sys-utils-cm/run-macos.sh b/script/get-sys-utils-mlc/run-macos.sh similarity index 100% rename from script/get-sys-utils-cm/run-macos.sh rename to script/get-sys-utils-mlc/run-macos.sh diff --git a/script/get-sys-utils-cm/run-rhel.sh b/script/get-sys-utils-mlc/run-rhel.sh similarity index 96% rename from script/get-sys-utils-cm/run-rhel.sh rename to script/get-sys-utils-mlc/run-rhel.sh index e91d70d07..0d61d5ccb 100644 --- a/script/get-sys-utils-cm/run-rhel.sh +++ b/script/get-sys-utils-mlc/run-rhel.sh @@ -24,7 +24,8 @@ ${MLC_SUDO} ${MLC_PACKAGE_TOOL} update && \ ca-certificates curl cmake \ gcc git g++ \ libtool libffi-devel libssl-devel\ - zlib-devel \ + procps-ng \ + zlib-devel \ libbz2-devel \ openssh-client \ make mesa-libGL \ diff --git a/script/get-sys-utils-cm/run-sles.sh b/script/get-sys-utils-mlc/run-sles.sh similarity index 100% rename from script/get-sys-utils-cm/run-sles.sh rename to script/get-sys-utils-mlc/run-sles.sh diff --git a/script/get-sys-utils-cm/run-ubuntu.sh b/script/get-sys-utils-mlc/run-ubuntu.sh similarity index 100% rename from script/get-sys-utils-cm/run-ubuntu.sh rename to script/get-sys-utils-mlc/run-ubuntu.sh diff --git a/script/get-tensorrt/meta.yaml b/script/get-tensorrt/meta.yaml index 5370e37d4..1263e4158 100644 --- a/script/get-tensorrt/meta.yaml +++ b/script/get-tensorrt/meta.yaml @@ -13,9 +13,10 @@ deps: tags: get,python3 docker: {} input_description: - input: Full path to the installed TensorRT library (nvinfer) - tar_file: Full path to the TensorRT Tar file downloaded from the Nvidia website - (https://developer.nvidia.com/tensorrt) + input: + desc: Full path to the installed TensorRT library (nvinfer) + tar_file: + desc: Full path to the TensorRT Tar file downloaded from the Nvidia website (https://developer.nvidia.com/tensorrt) input_mapping: input: MLC_INPUT tar_file: MLC_TENSORRT_TAR_FILE_PATH diff --git a/script/get-xilinx-sdk/run.sh b/script/get-xilinx-sdk/run.sh index 821adb3f9..ee74b85b6 100644 --- a/script/get-xilinx-sdk/run.sh +++ b/script/get-xilinx-sdk/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/install-cuda-prebuilt/meta.yaml b/script/install-cuda-prebuilt/meta.yaml index a5f06022e..2d0f1ce1e 100644 --- a/script/install-cuda-prebuilt/meta.yaml +++ b/script/install-cuda-prebuilt/meta.yaml @@ -8,6 +8,7 @@ default_env: default_version: 11.8.0 deps: - tags: detect,os +- tags: detect,sudo docker: run: true input_mapping: @@ -15,6 +16,7 @@ input_mapping: local_run_file_path: CUDA_RUN_FILE_LOCAL_PATH override-driver-check: MLC_CUDA_DRIVER_INSTALL_OVERRIDE skip_sudo: CUDA_SKIP_SUDO +sudo_install: True new_env_keys: - MLC_CUDA_* - MLC_NVCC_* @@ -89,3 +91,9 @@ versions: 12.6.1: env: MLC_CUDA_LINUX_FILENAME: cuda_12.6.1_560.35.03_linux.run + 12.8.1: + env: + MLC_CUDA_LINUX_FILENAME: cuda_12.8.1_570.124.06_linux.run + 12.9.1: + env: + MLC_CUDA_LINUX_FILENAME: cuda_12.9.1_575.57.08_linux.run diff --git a/script/install-gcc-src/customize.py b/script/install-gcc-src/customize.py index 7edf5103b..576e180bc 100644 --- a/script/install-gcc-src/customize.py +++ b/script/install-gcc-src/customize.py @@ -17,6 +17,35 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] + if env.get('MLC_GCC_TARGET', '') != '': + env['MLC_GCC_TARGET_STRING'] = f""" --target={env['MLC_GCC_TARGET']} """ + else: + env['MLC_GCC_TARGET_STRING'] = '' + + if env.get('MLC_GCC_HOST', '') != '': + env['MLC_GCC_HOST_STRING'] = f""" --host={env['MLC_GCC_HOST']} """ + else: + env['MLC_GCC_HOST_STRING'] = '' + + if env.get('MLC_GCC_BUILD', '') != '': + env['MLC_GCC_BUILD_STRING'] = f""" --build={env['MLC_GCC_BUILD']} """ + else: + env['MLC_GCC_BUILD_STRING'] = '' + + if env.get('MLC_GCC_WITH_SYSROOT', '') != '': + env['MLC_GCC_SYSROOT_STRING'] = f""" --with-sysroot={env['MLC_GCC_WITH_SYSROOT']} """ + else: + env['MLC_GCC_SYSROOT_STRING'] = '' + env['MLC_GCC_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install', 'bin') return {'return': 0} + + +def postprocess(i): + + env = i['env'] + if env.get('MLC_GIT_REPO_CURRENT_HASH', '') != '': + env['MLC_GCC_SRC_REPO_COMMIT'] = env['MLC_GIT_REPO_CURRENT_HASH'] + + return {'return': 0} diff --git a/script/install-gcc-src/meta.yaml b/script/install-gcc-src/meta.yaml index 741ab39b6..8879d3645 100644 --- a/script/install-gcc-src/meta.yaml +++ b/script/install-gcc-src/meta.yaml @@ -24,8 +24,15 @@ deps: - MLC_GIT_CHECKOUT_SHA _tag.: - MLC_GIT_CHECKOUT_TAG -env: - MLC_GIT_URL: git://gcc.gnu.org/git/gcc.git +input_mapping: + targets: MLC_GCC_TARGET + host: MLC_GCC_HOST + build: MLC_GCC_BUILD + extra: MLC_GCC_EXTRA_CONFIGURE_STRING + + +new_env_keys: + - MLC_GCC_* post_deps: - skip_if_env: MLC_REQUIRE_INSTALL: @@ -37,13 +44,28 @@ tags: - gcc - src-gcc uid: faae0ebd6e1242db +docker: + skip_run_cmd: true + pre_run_cmds: + - mlc pull repo tests: run_inputs: - {} variations: + mainline: + group: repo + default: true + default_variations: + version: master + env: + MLC_GIT_URL: git://gcc.gnu.org/git/gcc.git + darwin: + group: repo + env: + MLC_GIT_URL: https://github.com/iains/gcc-darwin-arm64 + MLC_GCC_WITH_SYSROOT: '`xcrun --show-sdk-path`' master: group: version - default: true env: MLC_GIT_CHECKOUT: master version.#: diff --git a/script/install-gcc-src/run.sh b/script/install-gcc-src/run.sh index c8b1ca291..bf0a78b39 100644 --- a/script/install-gcc-src/run.sh +++ b/script/install-gcc-src/run.sh @@ -11,6 +11,8 @@ if [ ! -d "src" ]; then test $? -eq 0 || exit $? fi +rm -rf install +rm -rf build mkdir -p install mkdir -p build @@ -22,7 +24,10 @@ cd src ./contrib/download_prerequisites cd ../build -../src/configure --prefix="${INSTALL_DIR}" --with-gcc-major-version-only --disable-multilib + +cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} ${MLC_GCC_HOST_STRING} ${MLC_GCC_BUILD_STRING} ${MLC_GCC_SYSROOT_STRING} ${MLC_GCC_EXTRA_CONFIGURE_STRING} --with-gcc-major-version-only --disable-multilib" +echo $cmd +eval $cmd test $? -eq 0 || exit $? diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index 2f0c25412..30b6f67f7 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -51,8 +51,9 @@ def preprocess(i): llvm_build_type = env['MLC_LLVM_BUILD_TYPE'] targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD', 'X86') + cross_compile_options = env.get('MLC_LLVM_CROSS_COMPILE_FLAGS', '') - cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type } -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {extra_cmake_options}""" + cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type} -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {cross_compile_options} {extra_cmake_options}""" env['MLC_LLVM_CMAKE_CMD'] = cmake_cmd @@ -83,4 +84,7 @@ def postprocess(i): if os.path.isdir(path_include): env['+C_INCLUDE_PATH'] = [path_include] + if env.get('MLC_GIT_REPO_CURRENT_HASH', '') != '': + env['MLC_LLVM_SRC_REPO_COMMIT'] = env['MLC_GIT_REPO_CURRENT_HASH'] + return {'return': 0} diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index c59f0a5c2..cef05d5a7 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -36,6 +36,9 @@ deps: - MLC_GIT_CHECKOUT_TAG _tag.llvmorg-: - MLC_VERSION +input_mapping: + targets: MLC_LLVM_TARGETS_TO_BUILD + env: MLC_GIT_URL: https://github.com/llvm/llvm-project name: Build LLVM compiler from sources (can take >30 min) @@ -107,6 +110,11 @@ variations: env: MLC_LLVM_INSTALLED_PATH: '#' MLC_LLVM_USE_INSTALLED_DIR: yes + cross-compile-x86-aarch64: + deps: + - tags: get,generic-sys-util,_crossbuild-essential-arm64 + env: + MLC_LLVM_CROSS_COMPILE_FLAGS: "-DCMAKE_SYSTEM_NAME=Linux -DCMAKE_SYSTEM_PROCESSOR=aarch64 -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-linux-gnu -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++" for-intel-mlperf-inference-v3.1-bert: adr: conda-package: @@ -247,6 +255,10 @@ variations: - full-history env: MLC_GIT_CHECKOUT_TAG: '#' +docker: + skip_run_cmd: true + pre_run_cmds: + - mlc pull repo versions: {} tests: run_inputs: diff --git a/script/install-mlperf-logging-from-src/run.sh b/script/install-mlperf-logging-from-src/run.sh index 0d5d73c4c..0aa4de475 100644 --- a/script/install-mlperf-logging-from-src/run.sh +++ b/script/install-mlperf-logging-from-src/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/install-nccl-libs/run.sh b/script/install-nccl-libs/run.sh index 821adb3f9..ee74b85b6 100644 --- a/script/install-nccl-libs/run.sh +++ b/script/install-nccl-libs/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/install-numactl-from-src/meta.yaml b/script/install-numactl-from-src/meta.yaml index a03250cdf..0da55b589 100644 --- a/script/install-numactl-from-src/meta.yaml +++ b/script/install-numactl-from-src/meta.yaml @@ -58,4 +58,4 @@ variations: env: MLC_GIT_CHECKOUT_TAG: '#' warnings: -- This CM script will need sudo to install numactl! +- This MLC script will need sudo to install numactl! diff --git a/script/prepare-training-data-bert/run-nvidia.sh b/script/prepare-training-data-bert/run-nvidia.sh index 016f6e07d..ad17be414 100644 --- a/script/prepare-training-data-bert/run-nvidia.sh +++ b/script/prepare-training-data-bert/run-nvidia.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/prepare-training-data-bert/run-reference.sh b/script/prepare-training-data-bert/run-reference.sh index a72f3adfd..543114a5b 100644 --- a/script/prepare-training-data-bert/run-reference.sh +++ b/script/prepare-training-data-bert/run-reference.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/prepare-training-data-bert/run.sh b/script/prepare-training-data-bert/run.sh index 194b53e80..30655574c 100644 --- a/script/prepare-training-data-bert/run.sh +++ b/script/prepare-training-data-bert/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/prepare-training-data-resnet/run-nvidia.sh b/script/prepare-training-data-resnet/run-nvidia.sh index 18fa70634..a78820a82 100644 --- a/script/prepare-training-data-resnet/run-nvidia.sh +++ b/script/prepare-training-data-resnet/run-nvidia.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/prepare-training-data-resnet/run-reference.sh b/script/prepare-training-data-resnet/run-reference.sh index a2f4a5eda..cdb890d58 100644 --- a/script/prepare-training-data-resnet/run-reference.sh +++ b/script/prepare-training-data-resnet/run-reference.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/preprocess-mlperf-inference-submission/meta.yaml b/script/preprocess-mlperf-inference-submission/meta.yaml index f0b3c105a..81abf1a6e 100644 --- a/script/preprocess-mlperf-inference-submission/meta.yaml +++ b/script/preprocess-mlperf-inference-submission/meta.yaml @@ -38,8 +38,8 @@ input_mapping: submitter: MLC_MLPERF_SUBMITTER submission_preprocessor_args: MLC_MLPERF_PREPROCESS_SUBMISSION_EXTRA_ARGS default_env: - MLC_MLPERF_NOINFER_LOW_ACCURACY_RESULTS: True - MLC_MLPERF_NOINFER_SCENARIO_RESULTS: True + MLC_MLPERF_NOINFER_LOW_ACCURACY_RESULTS: False + MLC_MLPERF_NOINFER_SCENARIO_RESULTS: False tags: - run - mlc diff --git a/script/process-mlperf-accuracy/meta.yaml b/script/process-mlperf-accuracy/meta.yaml index 8f25f68fb..24ff6fdd0 100644 --- a/script/process-mlperf-accuracy/meta.yaml +++ b/script/process-mlperf-accuracy/meta.yaml @@ -353,7 +353,7 @@ variations: group: dataset cnndm_llama_3,edge: deps: - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_rclone + - tags: get,dataset,cnndm,_validation,_edge,_llama3 names: - cnndm-llama3-edge skip_if_env: @@ -361,7 +361,7 @@ variations: - "yes" cnndm_llama_3,datacenter: deps: - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_rclone + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3 names: - cnndm-llama3-datacenter skip_if_env: diff --git a/script/prune-bert-models/customize.py b/script/prune-bert-models/customize.py index e69cde689..8a36d1664 100644 --- a/script/prune-bert-models/customize.py +++ b/script/prune-bert-models/customize.py @@ -33,7 +33,7 @@ def preprocess(i): logger.info('') print( - 'Local CM cache path to the updated BERT pruner src from NeurIPS 2022: ' + + 'Local MLC cache path to the updated BERT pruner src from NeurIPS 2022: ' + env['MLC_GIT_REPO_BERT_PRUNER_NEURIPS_2022_CHECKOUT_PATH']) logger.info('') diff --git a/script/push-mlperf-inference-results-to-github/customize.py b/script/push-mlperf-inference-results-to-github/customize.py index 9b2df4d0f..6b123bb64 100644 --- a/script/push-mlperf-inference-results-to-github/customize.py +++ b/script/push-mlperf-inference-results-to-github/customize.py @@ -42,3 +42,20 @@ def preprocess(i): env['MLC_GIT_PUSH_CMD'] = "git push" return {'return': 0} + + +def postprocess(i): + + os_info = i['os_info'] + env = i['env'] + meta = i['meta'] + automation = i['automation'] + + env['MLC_MLPERF_RESULTS_SYNC_DIR'] = env['MLC_GIT_REPO_CHECKOUT_PATH'] + + logger = automation.logger + + logger.info( + f"\n\nPath to the locally synced submission directory: {env['MLC_MLPERF_RESULTS_SYNC_DIR']}\n\n") + + return {'return': 0} diff --git a/script/push-mlperf-inference-results-to-github/meta.yaml b/script/push-mlperf-inference-results-to-github/meta.yaml index 2e92bf7c8..3535b31a3 100644 --- a/script/push-mlperf-inference-results-to-github/meta.yaml +++ b/script/push-mlperf-inference-results-to-github/meta.yaml @@ -4,6 +4,10 @@ automation_uid: 5b4e0237da074764 category: MLPerf benchmark support default_env: MLC_MLPERF_RESULTS_GIT_REPO_URL: https://github.com/mlcommons/mlperf_inference_submissions_v4.0 +new_env_keys: + - MLC_MLPERF_RESULTS_SYNC_DIR +print_env_at_the_end: + MLC_MLPERF_RESULTS_SYNC_DIR: Path to the locally synced submission directory deps: - names: - python3 diff --git a/script/reproduce-mlperf-octoml-tinyml-results/meta.yaml b/script/reproduce-mlperf-octoml-tinyml-results/meta.yaml index 055bb03dd..d04ad19f5 100644 --- a/script/reproduce-mlperf-octoml-tinyml-results/meta.yaml +++ b/script/reproduce-mlperf-octoml-tinyml-results/meta.yaml @@ -7,7 +7,7 @@ default_version: r1.0 deps: - tags: detect,os - tags: detect,cpu -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - names: - python3 - python diff --git a/script/reproduce-mlperf-training-nvidia/meta.yaml b/script/reproduce-mlperf-training-nvidia/meta.yaml index 263ab9d63..cb2c8e926 100644 --- a/script/reproduce-mlperf-training-nvidia/meta.yaml +++ b/script/reproduce-mlperf-training-nvidia/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: reproduce-mlperf-training-nvidia uid: f183628f292341e2 cache: false @@ -9,7 +9,7 @@ automation_uid: 5b4e0237da074764 category: "Reproduce MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -29,7 +29,7 @@ new_state_keys: - mlperf-training-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: diff --git a/script/run-all-mlperf-models/run-bert-macos.sh b/script/run-all-mlperf-models/run-bert-macos.sh index bc7a75747..9b103d0c9 100644 --- a/script/run-all-mlperf-models/run-bert-macos.sh +++ b/script/run-all-mlperf-models/run-bert-macos.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-bert.sh b/script/run-all-mlperf-models/run-bert.sh index 815e144ec..f3ffa1222 100644 --- a/script/run-all-mlperf-models/run-bert.sh +++ b/script/run-all-mlperf-models/run-bert.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-cpp-implementation.sh b/script/run-all-mlperf-models/run-cpp-implementation.sh index 3d4c7b93d..0e974b3d9 100644 --- a/script/run-all-mlperf-models/run-cpp-implementation.sh +++ b/script/run-all-mlperf-models/run-cpp-implementation.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-mobilenet-models.sh b/script/run-all-mlperf-models/run-mobilenet-models.sh index 849bb48ec..f0500c418 100644 --- a/script/run-all-mlperf-models/run-mobilenet-models.sh +++ b/script/run-all-mlperf-models/run-mobilenet-models.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out @@ -22,8 +22,8 @@ function run() { exit_if_error fi } -POWER=" --power=yes --adr.mlperf-power-client.power_server=192.168.0.15 --adr.mlperf-power-client.port=4940 --adr.mlperf-power-client.ntp_server=time.google.com " -POWER="" +POWER=" --power=yes --adr.mlperf-power-client.power_server=192.168.1.79 --adr.mlperf-power-client.port=4950 --adr.mlperf-power-client.ntp_server=time.google.com " +#POWER="" #extra_option=" --minimize_disk_usage=yes" extra_option=" --minimize_disk_usage=no" extra_tags="" @@ -35,11 +35,8 @@ extra_tags="" # run "$MLC_RUN_CMD" run "mlcr run,mobilenet-models,_tflite$extra_tags \ --adr.compiler.tags=gcc \ -${extra_option} " +${extra_option} $POWER" -run "mlcr run,mobilenet-models,_tflite,_armnn,_neon$extra_tags \ ---adr.compiler.tags=gcc \ -${extra_option} " extra_option=" $extra_option --adr.mlperf-inference-implementation.compressed_dataset=on" diff --git a/script/run-all-mlperf-models/run-nvidia-4090.sh b/script/run-all-mlperf-models/run-nvidia-4090.sh index 135a270d0..05184c117 100644 --- a/script/run-all-mlperf-models/run-nvidia-4090.sh +++ b/script/run-all-mlperf-models/run-nvidia-4090.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-nvidia-a100.sh b/script/run-all-mlperf-models/run-nvidia-a100.sh index a3489e7d2..715d28077 100644 --- a/script/run-all-mlperf-models/run-nvidia-a100.sh +++ b/script/run-all-mlperf-models/run-nvidia-a100.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-nvidia-t4.sh b/script/run-all-mlperf-models/run-nvidia-t4.sh index adde34344..adb1eef9e 100644 --- a/script/run-all-mlperf-models/run-nvidia-t4.sh +++ b/script/run-all-mlperf-models/run-nvidia-t4.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-reference-models.sh b/script/run-all-mlperf-models/run-reference-models.sh index 01766158a..2a0948892 100644 --- a/script/run-all-mlperf-models/run-reference-models.sh +++ b/script/run-all-mlperf-models/run-reference-models.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-resnet50-macos.sh b/script/run-all-mlperf-models/run-resnet50-macos.sh index 5b83b9a9b..9b49434f8 100644 --- a/script/run-all-mlperf-models/run-resnet50-macos.sh +++ b/script/run-all-mlperf-models/run-resnet50-macos.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-resnet50.sh b/script/run-all-mlperf-models/run-resnet50.sh index a2144bf0a..797f174f5 100644 --- a/script/run-all-mlperf-models/run-resnet50.sh +++ b/script/run-all-mlperf-models/run-resnet50.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-retinanet-sh b/script/run-all-mlperf-models/run-retinanet-sh index 3f10a88ee..abf29a632 100644 --- a/script/run-all-mlperf-models/run-retinanet-sh +++ b/script/run-all-mlperf-models/run-retinanet-sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/template.sh b/script/run-all-mlperf-models/template.sh index 9a5fb1893..f0359ced4 100644 --- a/script/run-all-mlperf-models/template.sh +++ b/script/run-all-mlperf-models/template.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 62eb7f535..3c40bac93 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -33,7 +33,7 @@ def preprocess(i): 'tags': env['MLC_DOCKER_RUN_SCRIPT_TAGS']}) if len(r['list']) < 1: raise Exception( - 'CM script with tags ' + + 'MLC script with tags ' + env['MLC_DOCKER_RUN_SCRIPT_TAGS'] + ' not found!') @@ -62,7 +62,8 @@ def preprocess(i): if os_info['platform'] == 'windows': CMD += " 2> nul" else: - CMD += " 2> /dev/null" + CMD += " 2> /dev/null || true " + logger.info(' ' + CMD) logger.info('') @@ -100,7 +101,7 @@ def preprocess(i): if os_info['platform'] == 'windows': CMD += " 2> nul" else: - CMD += " 2> /dev/null" + CMD += " 2> /dev/null || true" logger.info('') logger.info('Checking Docker images:') diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 8cf0fbe1c..a0c82e593 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -331,9 +331,9 @@ def preprocess(i): if state.get("mlc-mlperf-inference-results"): # print(state["mlc-mlperf-inference-results"]) for sut in state["mlc-mlperf-inference-results"]: # only one sut will be there - # Better to do this in a stand alone CM script with proper deps but + # Better to do this in a stand alone MLC script with proper deps but # currently we manage this by modifying the sys path of the python - # executing CM + # executing MLC from tabulate import tabulate # noqa logger.info(f"{sut}") diff --git a/script/run-mlperf-inference-submission-checker/README.md b/script/run-mlperf-inference-submission-checker/README.md index 626aa9fcd..ce18e987b 100644 --- a/script/run-mlperf-inference-submission-checker/README.md +++ b/script/run-mlperf-inference-submission-checker/README.md @@ -1,10 +1,10 @@ # README for run-mlperf-inference-submission-checker This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -90,3 +90,7 @@ mlcr run,mlc,mlcommons,mlperf,inference,mlperf-inference,submission,checker,subm ### Ungrouped - `short-run` + +### Version + +- `version.master` diff --git a/script/run-mlperf-inference-submission-checker/customize.py b/script/run-mlperf-inference-submission-checker/customize.py index 91b0d94c1..a427431e9 100644 --- a/script/run-mlperf-inference-submission-checker/customize.py +++ b/script/run-mlperf-inference-submission-checker/customize.py @@ -65,6 +65,10 @@ def preprocess(i): extra_args = ' ' + env.get('MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS', '') + if (is_true(env.get('MLC_TAR_SUBMISSION_DIR')) or env.get('MLC_MLPERF_SUBMITTER_ID', + '') != '') and "skip-extra-files-in-root-check" not in extra_args: + extra_args += " --skip-extra-files-in-root-check " + x_submitter = ' --submitter ' + q + submitter + q if submitter != '' else '' x_version = ' --version ' + version + ' ' if version != '' else '' diff --git a/script/runtime-system-infos/meta.yaml b/script/runtime-system-infos/meta.yaml index 3e3186c4b..37c383814 100644 --- a/script/runtime-system-infos/meta.yaml +++ b/script/runtime-system-infos/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: runtime-system-infos uid: 755cf27627784001 cache: false @@ -9,7 +9,7 @@ category: "MLPerf benchmark support" docker: real_run: False -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - runtime - system @@ -21,7 +21,7 @@ input_mapping: interval: MLC_SYSTEM_INFO_MEASUREMENT_INTERVAL -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -32,7 +32,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect python3 - tags: get,python3 diff --git a/script/set-cpu-frequency/run.sh b/script/set-cpu-frequency/run.sh index 57b22872a..1bf863667 100644 --- a/script/set-cpu-frequency/run.sh +++ b/script/set-cpu-frequency/run.sh @@ -24,6 +24,9 @@ case "$DRIVER_KEY" in echo "→ intel_pstate: disabling turbo, setting performance governor" echo 0 | ${MLC_SUDO} tee /sys/devices/system/cpu/intel_pstate/no_turbo >/dev/null ${MLC_SUDO} cpupower frequency-set -g performance + echo "" + echo "Note: intel_pstate does _not_ support a userspace/fixed frequency mode." + echo "If you need a precise kHz, switch back to acpi-cpufreq in your kernel cmdline." ;; amd-pstate) diff --git a/script/set-device-settings-qaic/run.sh b/script/set-device-settings-qaic/run.sh index dab1a87d4..2220d36c6 100644 --- a/script/set-device-settings-qaic/run.sh +++ b/script/set-device-settings-qaic/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/set-performance-mode/run-ubuntu.sh b/script/set-performance-mode/run-ubuntu.sh index 71b8ce802..c5889cfa9 100644 --- a/script/set-performance-mode/run-ubuntu.sh +++ b/script/set-performance-mode/run-ubuntu.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/set-performance-mode/run.sh b/script/set-performance-mode/run.sh index 821adb3f9..ee74b85b6 100644 --- a/script/set-performance-mode/run.sh +++ b/script/set-performance-mode/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/set-user-limits/run.sh b/script/set-user-limits/run.sh index 32cf4d51e..4a22ce805 100644 --- a/script/set-user-limits/run.sh +++ b/script/set-user-limits/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/set-venv/customize.py b/script/set-venv/customize.py index daf23ea4f..64b6d3607 100644 --- a/script/set-venv/customize.py +++ b/script/set-venv/customize.py @@ -80,10 +80,10 @@ def preprocess(i): shell = shell.replace('MLC_SET_VENV_WORK', 'work') if shell == '': shell = 'cmd' - cmd = 'cd {} & call {} & set MLC_REPOS=%CD%\\{}\\CM & {}\n'.format( + cmd = 'cd {} & call {} & set MLC_REPOS=%CD%\\{}\\MLC & {}\n'.format( name, activate_script, name, shell) else: - cmd = '#!/bin/bash\n\ncd {} ; source {} ; export MLC_REPOS=$PWD/CM ; cd work\n'.format( + cmd = '#!/bin/bash\n\ncd {} ; source {} ; export MLC_REPOS=$PWD/MLC ; cd work\n'.format( name, activate_script) with open(script_file, 'w') as f: diff --git a/script/submit-mlperf-results/README.md b/script/submit-mlperf-results/README.md index 6a8108ee0..f1909fca1 100644 --- a/script/submit-mlperf-results/README.md +++ b/script/submit-mlperf-results/README.md @@ -1,10 +1,10 @@ # README for submit-mlperf-results This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/submit-mlperf-results/customize.py b/script/submit-mlperf-results/customize.py index 1c2c6b619..b6ec51754 100644 --- a/script/submit-mlperf-results/customize.py +++ b/script/submit-mlperf-results/customize.py @@ -2,6 +2,8 @@ from mlc import utils import os import json +import shutil +import tempfile def preprocess(i): @@ -25,26 +27,33 @@ def preprocess(i): env['MLC_TMP_CURRENT_PATH'], file_path)) - r = get_signed_url( - server, - benchmark, - submitter_id, - submitter_name, - file_path) - if r['return'] > 0: - return r - - signed_url = r['signed_url'] - submission_id = r['submission_id'] - - r = upload_file_to_signed_url(file_path, signed_url, logger) - if r['return'] > 0: - return r - - r = trigger_submission_checker( - server, submitter_id, benchmark, submission_id, logger) - if r['return'] > 0: - return r + with tempfile.TemporaryDirectory() as tmpdir: + tmp_file_path = os.path.join(tmpdir, "submission.tar.gz") + + # use tmp_file_path to prevent long file paths which causes issue + # during upload + shutil.copy(file_path, tmp_file_path) + + r = get_signed_url( + server, + benchmark, + submitter_id, + submitter_name, + tmp_file_path) + if r['return'] > 0: + return r + + signed_url = r['signed_url'] + submission_id = r['submission_id'] + + r = upload_file_to_signed_url(tmp_file_path, signed_url, logger) + if r['return'] > 0: + return r + + r = trigger_submission_checker( + server, submitter_id, benchmark, submission_id, logger) + if r['return'] > 0: + return r return {'return': 0}