From 14873806ebcf46b56b104995d3d6b2e7975c735c Mon Sep 17 00:00:00 2001 From: Austin Bozowski Date: Thu, 14 Jul 2022 22:50:00 -0700 Subject: [PATCH] Chef CI/CD Docs (#20711) * Update README and make bundle generic and gating * Format * Format * More semantic naming * Spelling * Spelling * Restyle again --- examples/chef/README.md | 147 ++++++++++++++++++++++++++++++++++++++-- examples/chef/chef.py | 13 ++-- 2 files changed, 147 insertions(+), 13 deletions(-) diff --git a/examples/chef/README.md b/examples/chef/README.md index 07579f3791947a..b9641be4d82ac8 100644 --- a/examples/chef/README.md +++ b/examples/chef/README.md @@ -9,9 +9,9 @@ Chef uses the shell app a starting point, but processes the data model defined on ZAP files during build time. This procedure is handled by its unified build script: `chef.py`. -As it incorporates the processing of ZAP files as part of the build process, it -does not use `zzz_generated`, but rather places the auto-generated zap artifacts -under its `out` temporary folder. +When processing ZAP files as part of the build process, Chef places the +auto-generated zap artifacts under its `out` temporary folder. Chef uses +artifacts from `zzz_generated` for CI/CD. All device types available (.zap files) are found inside the `devices` folder. @@ -83,9 +83,142 @@ All device types available (.zap files) are found inside the `devices` folder. All CI jobs for chef can be found in `.github/workflows/chef.yaml`. -These jobs use a platform-specific image with base `chip-build`. +These jobs use a platform-specific image with base `chip-build`. Such images +contain the toolchain for the respective platform under `/opt`. -CI jobs call chef with the options `--ci -t `. The `--ci` option will -execute builds for all devices specified in `cicd_config["ci_allow_list"]` -defined in `chef.py` (so long as these devices are also in `/devices`) on the +CI jobs call chef with the options `--ci -t $PLATFORM`. The `--ci` option will +execute builds for all devices specified in `ci_allow_list` defined in +`cicd_config.json` (so long as these devices are also in `/devices`) on the specified platform. + +CI jobs also call the function `bundle_$PLATFORM` at the end of each example +build. This function should copy or move build output files from the build +output location into `_CD_STAGING_DIR`. Typically, the set of files touched is +the minimal set of files needed to flash a device. See the function +`bundle_esp32` for reference. + +### Adding a platform + +First, implement a `bundle_$PLATFORM` function. + +Next, ensure that the examples in `ci_allow_list` build in a container using the +relevant platform image. You can simulate the workflow locally by mounting your +CHIP repo into a container and executing the CI command: + +```shell +docker run -it --mount source=$(pwd),target=/workspace,type=bind connectedhomeip/chip-build-$PLATFORM:$VERSION +``` + +In the container: + +```shell +chown -R $(whoami) /workspace +cd /workspace +source ./scripts/bootstrap.sh +source ./scripts/activate.sh +./examples/chef/chef.py --ci -t $PLATFORM +``` + +Once you are confident the CI examples build and bundle in a container, add a +new job to the chef workflow. + +Replace all instances of `$PLATFORM` with the new platform. Replace `$VERSION` +with the image version used in the rest of the workflows, or update the image +version for all images in the workflow as needed. + +```yaml +chef_$PLATFORM: + name: Chef - $PLATFORM CI Examples + runs-on: ubuntu-latest + if: github.actor != 'restyled-io[bot]' + + container: + image: connectedhomeip/chip-build-$PLATFORM:$VERSION + options: --user root + + steps: + - uses: Wandalen/wretry.action@v1.0.15 + name: Checkout + with: + action: actions/checkout@v3 + with: | + token: ${{ github.token }} + attempt_limit: 3 + attempt_delay: 2000 + - name: Checkout submodules + run: scripts/checkout_submodules.py --shallow --platform $PLATFORM + - name: Bootstrap + timeout-minutes: 10 + run: scripts/build/gn_bootstrap.sh + - name: CI Examples $PLATFORM + shell: bash + run: | + ./scripts/run_in_build_env.sh "./examples/chef/chef.py --ci -t $PLATFORM" +``` + +## CD + +Once CI is enabled for a platform, the platform may also be integrated into +`integrations/cloudbuild/`, where chef builds are defined in `chef.yaml`. See +the `README` in this path for more information. + +Note that the image used in `chef.yaml` is `chip-build-vscode`. See +`docker/images/chip-build-vscode/Dockerfile` for the source of this image. This +image is a combination of the individual toolchain images. Therefore, before a +platform is integrated into chef CD, the toolchain should be copied into +`chip-build-vscode` and `chef.yaml` should be updated to use the new image +version. + +Finally, add the new platform to `cd_platforms` in `cicd_config.json`. The +configuration should follow the following schema: + +```json +"$PLATFORM": { + "output_archive_prefix_1": ["option_1", "option_2"], + "output_archive_prefix_2": [], +} +``` + +Take note of the configuration for `linux`: + +```json +"linux": { + "linux_x86": ["--cpu_type", "x64"], + "linux_arm64_ipv6only": ["--cpu_type", "arm64", "--ipv6only"] +}, +``` + +This will produce output archives prefixed `linux_x86` and +`linux_arm_64_ipv6only` and will append the respective options to each build +command for these targets. + +To test your configuration locally, you may employ a similar strategy as in CI: + +```shell +docker run -it --mount source=$(pwd),target=/workspace,type=bind connectedhomeip/chip-build-vscode:$VERSION +``` + +In the container: + +```shell +chown -R $(whoami) /workspace +cd /workspace +source ./scripts/bootstrap.sh +source ./scripts/activate.sh +./examples/chef/chef.py --build_all --keep_going +``` + +You may also use the Google Cloud Build local builder as detailed in the +`README` of `integrations/cloudbuild/`. + +## Adding new devices + +To add new devices for chef: + +- Execute `python sample_app_util.py zap --rename-file` to rename + the example and place the new file in `examples/chef/devices`. + - See the `README` in `examples/chef/sample_app_util/` for more info. +- Execute `scripts/tools/zap_regen_all.py`, commit `zzz_generated` and + `examples/chef/devices`. + - This is gated by the workflow in `.github/workflows/zap_templates.yaml`. +- All devices added to the repository are built in CD. diff --git a/examples/chef/chef.py b/examples/chef/chef.py index b3703264bd46bc..c687296305d397 100755 --- a/examples/chef/chef.py +++ b/examples/chef/chef.py @@ -134,18 +134,19 @@ def bundle(platform: str, device_name: str) -> None: platform: The platform to bundle. device_name: The example to bundle. """ + bundler_name = f"bundle_{platform}" matter_file = f"{device_name}.matter" zap_file = os.path.join(_DEVICE_FOLDER, f"{device_name}.zap") flush_print(f"Bundling {platform}", with_border=True) flush_print(f"Cleaning {_CD_STAGING_DIR}") shutil.rmtree(_CD_STAGING_DIR, ignore_errors=True) os.mkdir(_CD_STAGING_DIR) - if platform == "linux": - bundle_linux(device_name) - elif platform == "nrfconnect": - bundle_nrfconnect(device_name) - elif platform == "esp32": - bundle_esp32(device_name) + flush_print(f"Checking for {bundler_name}") + chef_module = sys.modules[__name__] + if hasattr(chef_module, bundler_name): + flush_print(f"Found {bundler_name}") + bundler = getattr(chef_module, bundler_name) + bundler(device_name) else: flush_print(f"No bundle function for {platform}!") exit(1)