-
Notifications
You must be signed in to change notification settings - Fork 47
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #277 from firesim/cached-br
Cached Buildroot Images
- Loading branch information
Showing
9 changed files
with
266 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| name: weekly-build | ||
|
|
||
| on: | ||
| schedule: | ||
| # run at 00:00 on sunday | ||
| - cron: "0 0 * * 0" | ||
|
|
||
| defaults: | ||
| run: | ||
| shell: bash -leo pipefail {0} | ||
|
|
||
| env: | ||
| REMOTE_WORK_DIR: /scratch/buildbot/firemarshal-ci-shared/firemarshal-${{ github.sha }} | ||
| PERSONAL_ACCESS_TOKEN: ${{ secrets.FIREMARSHAL_BR_UPLOAD_GH_PERSONAL_ACCESS_KEY }} | ||
|
|
||
| jobs: | ||
| setup-repo: | ||
| name: setup-repo | ||
| needs: change-filters | ||
| if: needs.change-filters.outputs.run-core == 'true' | ||
| runs-on: firemarshal | ||
| steps: | ||
| - name: Delete old checkout | ||
| run: | | ||
| ls -alh . | ||
| rm -rf ${{ env.REMOTE_WORK_DIR }}/* || true | ||
| rm -rf ${{ env.REMOTE_WORK_DIR }}/.* || true | ||
| rm -rf ${{ github.workspace }}/* || true | ||
| rm -rf ${{ github.workspace }}/.* || true | ||
| - uses: actions/checkout@v3 | ||
| - name: Setup repo copy | ||
| run: | | ||
| git clone $GITHUB_WORKSPACE ${{ env.REMOTE_WORK_DIR }} | ||
| - name: Setup conda (install all deps) | ||
| run: | | ||
| cd ${{ env.REMOTE_WORK_DIR }} | ||
| conda env create -f ./conda-reqs.yaml -p ./.conda-env | ||
| eval "$(conda shell.bash hook)" | ||
| conda install -y -p $PWD/.conda-env -c ucb-bar riscv-tools=1.0.4 | ||
| - name: Initialize all submodules | ||
| run: | | ||
| cd ${{ env.REMOTE_WORK_DIR }} | ||
| eval "$(conda shell.bash hook)" | ||
| conda activate $PWD/.conda-env | ||
| ./init-submodules.sh | ||
| - name: Verify open file limits | ||
| run: | | ||
| HARD_LIMIT=$(ulimit -Hn) | ||
| REQUIRED_LIMIT=16384 | ||
| if [ "$HARD_LIMIT" -lt "$REQUIRED_LIMIT" ]; then | ||
| echo "ERROR: Your system does not support an open files limit (the output of 'ulimit -Sn' and 'ulimit -Hn') of at least $REQUIRED_LIMIT, which is required to workaround a bug in buildroot. You will not be able to build a Linux distro with FireMarshal until this is addressed." | ||
| exit 1 | ||
| fi | ||
| ulimit -Sn $(ulimit -Hn) | ||
| build-upload-br-image: | ||
| name: build-upload-br-image | ||
| needs: [setup-repo] | ||
| runs-on: firemarshal | ||
| steps: | ||
| - name: Build buildroot image | ||
| run: | | ||
| cd ${{ env.REMOTE_WORK_DIR }} | ||
| eval "$(conda shell.bash hook)" | ||
| conda activate $PWD/.conda-env | ||
| ./marshal -v build br-base.json | ||
| ./scripts/upload-br-image.py | ||
| cleanup: | ||
| name: cleanup | ||
| needs: [setup-repo, build-upload-br-image] | ||
| runs-on: firemarshal | ||
| if: ${{ always() }} | ||
| steps: | ||
| - name: Delete repo copy and conda env | ||
| run: | | ||
| rm -rf ${{ env.REMOTE_WORK_DIR }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| from pathlib import Path | ||
| import os | ||
| from github import Github | ||
| import time | ||
| import zipfile | ||
|
|
||
| script_dir = Path(os.path.dirname(os.path.realpath(__file__))) | ||
| fm_dir = script_dir.parent | ||
|
|
||
| GH_REPO = 'firemarshal-public-br-images' | ||
| GH_ORG = 'firesim' | ||
| URL_PREFIX = f"https://raw.githubusercontent.com/{GH_ORG}/{GH_REPO}" | ||
|
|
||
|
|
||
| # taken from https://stackoverflow.com/questions/63427607/python-upload-files-directly-to-github-using-pygithub | ||
| # IMPORTANT: only works for binary files! (i.e. tar.gz or zip files) | ||
| def upload_binary_file(local_file_path, gh_file_path): | ||
| print(f":DEBUG: Attempting to upload {local_file_path} to {gh_file_path}") | ||
|
|
||
| g = Github(os.environ['PERSONAL_ACCESS_TOKEN']) | ||
|
|
||
| repo = g.get_repo(f'{GH_ORG}/{GH_REPO}') | ||
| all_files = [] | ||
| contents = repo.get_contents("") | ||
| while contents: | ||
| file_content = contents.pop(0) | ||
| if file_content.type == "dir": | ||
| contents.extend(repo.get_contents(file_content.path)) | ||
| else: | ||
| file = file_content | ||
| all_files.append(str(file).replace('ContentFile(path="', '').replace('")', '')) | ||
|
|
||
| with open(local_file_path, 'rb') as file: | ||
| content = file.read() | ||
|
|
||
| tries = 10 | ||
| delay = 15 | ||
| msg = f"Committing files from {os.environ['GITHUB_SHA']}" | ||
| upload_branch = 'main' | ||
| r = None | ||
|
|
||
| # Upload to github | ||
| git_file = gh_file_path | ||
| if git_file in all_files: | ||
| contents = repo.get_contents(git_file) | ||
| for n in range(tries): | ||
| try: | ||
| r = repo.update_file(contents.path, msg, content, contents.sha, branch=upload_branch) | ||
| break | ||
| except Exception as e: | ||
| print(f"Got exception: {e}") | ||
| time.sleep(delay) | ||
| assert r is not None, f"Unable to poll 'update_file' API {tries} times" | ||
| print(f"Updated: {git_file}") | ||
| else: | ||
| for n in range(tries): | ||
| try: | ||
| r = repo.create_file(git_file, msg, content, branch=upload_branch) | ||
| break | ||
| except Exception as e: | ||
| print(f"Got exception: {e}") | ||
| time.sleep(delay) | ||
| assert r is not None, f"Unable to poll 'create_file' API {tries} times" | ||
| print(f"Created: {git_file}") | ||
|
|
||
| return r['commit'].sha | ||
|
|
||
|
|
||
| def make_relative(path): | ||
| path_str = str(path) | ||
| return path_str.replace(f"{fm_dir}/", "") | ||
|
|
||
|
|
||
| # only caches firechip board br images | ||
| for e in (fm_dir / 'images' / 'firechip').iterdir(): | ||
| if not e.is_file(): | ||
| for ie in e.iterdir(): | ||
| if ie.is_file() and ie.suffix == '.img' and "br." in ie.name: | ||
| ie_zip = str(ie) + ".zip" | ||
| with zipfile.ZipFile(ie_zip, 'w', zipfile.ZIP_BZIP2, compresslevel=9) as zip_ref: | ||
| zip_ref.write(ie, ie.name) | ||
| upload_binary_file(ie_zip, make_relative(ie_zip)) | ||
| os.remove(ie_zip) |
Oops, something went wrong.