From 8405492a24bb90013e4bfb2497b49ad0380353b8 Mon Sep 17 00:00:00 2001 From: busticated Date: Mon, 11 Apr 2022 11:26:16 -0700 Subject: [PATCH] workbench / local compiler CI checks run in parallel by platform --- .circleci/config.yml | 176 +++++++++++++++++++++++++++++++++-- .workbench/manifest.json | 7 +- scripts/test-build-tasks.ps1 | 31 ++++-- scripts/test-build-tasks.sh | 28 ++++-- 4 files changed, 214 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8512fba066..23eb764a6c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,9 +12,77 @@ tag_filters: &tag_filters orbs: win: circleci/windows@2.4.0 +aliases: + - &tasks ["compile:all clean:all", "compile:user clean:user", "compile:debug clean:debug"] + - &platforms ["photon", "p1", "electron", "argon", "boron", "bsom", "b5som", "tracker"] + # Define a job to be invoked later in a workflow. # See: https://circleci.com/docs/2.0/configuration-reference/#jobs jobs: + setup-workspace: + machine: + image: ubuntu-2004:current + steps: + - run: + name: "Create Tested Platforms Files in Workspace" + command: | + mkdir -p $HOME/workspace/tested-platforms + ls -la $HOME/workspace + ls -la $HOME/workspace/tested-platforms + echo ":::: done!" + - persist_to_workspace: + root: ~/workspace + paths: + - tested-platforms + validate-toolchain-platforms: + machine: + image: ubuntu-2004:current + steps: + - attach_workspace: + at: ~/workspace + - checkout + - install-prtcl-ubuntu + - run: + name: "Check if all toolchain platforms were tested" + command: | + ls -la $HOME/workspace + ls -la $HOME/workspace/tested-platforms + echo + deviceOSPath=~/project + deviceOSSource="source:${deviceOSPath}" + toolchain="$(prtcl toolchain:view ${deviceOSSource} --json)" + platformsStr=$(echo $toolchain | jq '.platforms' | jq 'map(.name)') + platforms=() + while read p; do + platforms+=($(echo ${p} | tr -d '"')) + done < <(echo $platformsStr | jq -c '.[]') + echo ":::: Found toolchain platforms: ${platforms[@]}" + oses=( + 'darwin-x64' + 'ubuntu-x64' + 'windows-x64' + ) + for os in "${oses[@]}"; do + for platform in "${platforms[@]}"; do + dir="${HOME}/workspace/tested-platforms" + prefix="${os}-${platform}" + pattern="${prefix}-*" + echo ":::: Checking for results matching '${dir}/${pattern}'" + results=$(find $dir -maxdepth 1 -name "${pattern}" -type f) + count=0 + if [ -n "${results}" ]; then + count=$(echo "$results" | wc -l) + fi + echo ":::: Found ${count} results:" + echo "$results" + echo + if [ $count -lt 1 ]; then + echo ":::: Missing tests for ${platform}!" + exit 1 + fi + done + done + echo ":::: done!" build-for-windows: executor: name: win/default @@ -22,36 +90,57 @@ jobs: parameters: tasks: type: string + platform: + type: string steps: - checkout - run: git submodule update --init --recursive - install-prtcl-windows - test-localcompiler-windows: tasks: << parameters.tasks >> + platform: << parameters.platform >> + - save-platform-to-workspace: + os: windows-x64 + tasks: << parameters.tasks >> + platform: << parameters.platform >> build-for-darwin: macos: xcode: "13.2.1" parameters: tasks: type: string + platform: + type: string steps: - checkout - run: git submodule update --init --recursive - install-prtcl-darwin - test-localcompiler-nix: tasks: << parameters.tasks >> + platform: << parameters.platform >> + - save-platform-to-workspace: + os: darwin-x64 + tasks: << parameters.tasks >> + platform: << parameters.platform >> build-for-ubuntu: machine: image: ubuntu-2004:current parameters: tasks: type: string + platform: + type: string steps: - checkout - run: git submodule update --init --recursive - install-prtcl-ubuntu - test-localcompiler-nix: tasks: << parameters.tasks >> + platform: << parameters.platform >> + - save-platform-to-workspace: + os: ubuntu-x64 + tasks: << parameters.tasks >> + platform: << parameters.platform >> combine-binaries: docker: - image: alpine:latest @@ -246,43 +335,118 @@ commands: parameters: tasks: type: string + platform: + type: string steps: - run: name: "Test Local Compilation Tasks" no_output_timeout: "10m" - command: scripts/test-build-tasks.sh ~/project "<< parameters.tasks >>" + command: scripts/test-build-tasks.sh ~/project "<< parameters.platform >>" "<< parameters.tasks >>" environment: PRTCL_DISABLE_AUTOUPDATE: "1" test-localcompiler-windows: parameters: tasks: type: string + platform: + type: string steps: - run: name: "Test Local Compilation Tasks" no_output_timeout: "20m" - command: pwsh scripts/test-build-tasks.ps1 $home\project "<< parameters.tasks >>" + command: pwsh scripts/test-build-tasks.ps1 $home\project "<< parameters.platform >>" "<< parameters.tasks >>" environment: PRTCL_DISABLE_AUTOUPDATE: "1" - + save-platform-to-workspace: + parameters: + os: + type: string + tasks: + type: string + platform: + type: string + steps: + - attach_workspace: + at: ~/workspace + - when: + condition: + or: + - equal: [ 'ubuntu-x64', << parameters.os >> ] + - equal: [ 'darwin-x64', << parameters.os >> ] + steps: + - run: + name: "Save Platform to Workspace (*nix)" + command: | + os="<< parameters.os >>" + tasks="<< parameters.tasks >>" + platform="<< parameters.platform >>" + dir="${HOME}/workspace/tested-platforms" + filename="${os}-${platform}-$(echo $tasks | tr -d ' ":')" + filepath="${dir}/${filename}" + echo ":::: Adding: ${filepath}" + touch $filepath + ls -la $dir + echo ":::: done!" + - when: + condition: + and: + - equal: [ 'windows-x64', << parameters.os >> ] + steps: + - run: + name: "Save Platform to Workspace (windows)" + shell: powershell.exe + command: | + $ErrorActionPreference = "Stop" + $os = "<< parameters.os >>" + $tasks = "<< parameters.tasks >>" + $platform = "<< parameters.platform >>" + $dir = "$($home)\workspace\tested-platforms" + $filename = "$($os)-$($platform)-$($tasks -replace '[ :"]','')" + $filepath = "$($dir)\$($filename)" + echo ":::: Adding: $($filepath)" + New-Item $filepath -type file + Get-ChildItem -Force $dir + echo ":::: done!" + - persist_to_workspace: + root: ~/workspace + paths: + - tested-platforms workflows: build-and-test: jobs: + - setup-workspace: + <<: *tag_filters - build-for-windows: <<: *tag_filters matrix: parameters: - tasks: ["compile:all clean:all", "compile:user clean:user", "compile:debug clean:debug"] + tasks: *tasks + platform: *platforms + requires: + - setup-workspace - build-for-ubuntu: <<: *tag_filters matrix: parameters: - tasks: ["compile:all clean:all", "compile:user clean:user", "compile:debug clean:debug"] + tasks: *tasks + platform: *platforms + requires: + - setup-workspace - build-for-darwin: <<: *tag_filters matrix: parameters: - tasks: ["compile:all clean:all", "compile:user clean:user", "compile:debug clean:debug"] + tasks: *tasks + platform: *platforms + requires: + - setup-workspace + - validate-toolchain-platforms: + <<: *tag_filters + requires: + - setup-workspace + - build-for-windows + - build-for-ubuntu + - build-for-darwin - build-and-test: <<: *tag_filters context: diff --git a/.workbench/manifest.json b/.workbench/manifest.json index 4569445693..a106ea1e04 100644 --- a/.workbench/manifest.json +++ b/.workbench/manifest.json @@ -2,7 +2,7 @@ "version": "1.0.0", "toolchains": [ { - "platforms": [6, 8, 10, 12, 13, 22, 23, 25, 26], + "platforms": [6, 8, 10, 12, 13, 23, 25, 26], "firmware": "deviceOS@source", "compilers": "gcc-arm@10.2.1", "tools": "buildtools@1.1.1", @@ -36,11 +36,6 @@ "name": "boron", "generation": 3 }, - { - "id": 22, - "name": "asom", - "generation": 3 - }, { "id": 23, "name": "bsom", diff --git a/scripts/test-build-tasks.ps1 b/scripts/test-build-tasks.ps1 index a7a869019f..f8c0be34c8 100644 --- a/scripts/test-build-tasks.ps1 +++ b/scripts/test-build-tasks.ps1 @@ -1,6 +1,7 @@ param ( [Parameter(Mandatory = $true, Position = 0)][string]$deviceOSPath, - [Parameter(Mandatory = $false, Position = 1)][string]$taksList + [Parameter(Mandatory = $false, Position = 1)][string]$platformList, + [Parameter(Mandatory = $false, Position = 2)][string]$taksList ) function run(){ @@ -22,10 +23,22 @@ function run(){ } } +# setup variables $ErrorActionPreference = "Stop" -$deviceOSSource="source:${deviceOSPath}" -$tinkerPath="$($deviceOSPath)\user\applications\tinker\" -$toolchain = run prtcl toolchain:view $deviceOSSource --json +$deviceOSSource = "source:${deviceOSPath}" +$tinkerPath = "$($deviceOSPath)\user\applications\tinker\" +$platforms = @() +$tasks = @() + +if ($platformList){ + $platforms = $platformList.split(' ') +} else { + $toolchain = run prtcl toolchain:view $deviceOSSource --json + $json = $toolchain | ConvertFrom-Json + foreach ($p in $json.platforms){ + $platforms += $p.name + } +} if ($taksList){ $tasks = $taksList.split(' ') @@ -42,16 +55,18 @@ if ($taksList){ echo ":::: Using prtcl $(prtcl version)" echo ":::: Using Device OS at $($deviceOSPath)" +echo ":::: Targeting platforms: $($platforms)" +echo ":::: Running tasks: $($tasks)" +# install toolchain and run specified tasks run prtcl toolchain:install $deviceOSSource --quiet -$json = $toolchain | ConvertFrom-Json -foreach ($platform in $json.platforms){ - echo ":::: Testing build tasks for $($platform.name)" +foreach ($platform in $platforms){ + echo ":::: Testing build tasks for $($platform)" echo "" foreach ($task in $tasks){ - run prtcl $task $deviceOSSource $platform.name $tinkerPath --quiet + run prtcl $task $deviceOSSource $platform $tinkerPath --quiet if ($LASTEXITCODE -ne 0){ throw "Failure! Exit code is $LASTEXITCODE" diff --git a/scripts/test-build-tasks.sh b/scripts/test-build-tasks.sh index 374fffef76..635f867e28 100755 --- a/scripts/test-build-tasks.sh +++ b/scripts/test-build-tasks.sh @@ -10,7 +10,7 @@ hasCmd(){ if [ $# -lt 1 ]; then echo ":::: Error: Missing script arguments!" echo ":::: Fix: \`deviceOSPath\` is required - e.g. \`~/path/to/device-os\`" - echo ":::: Usage: