From 7bb8b2865f1a733ac0799b2857d2a780bc405fe8 Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Tue, 19 Jan 2021 16:50:48 +0100 Subject: [PATCH 1/3] Change workflow to bump version releases Do it automatically via GitHub Actions --- .ci/all-options-enabled-cache.cmake | 26 +++ .github/workflows/ci.yml | 4 +- .github/workflows/update-latest-releases.yml | 53 ++++++ doc/developers-faqs.md | 8 +- releases/latest.releases.yaml | 173 +++++++++++++++++++ scripts/robotologyUpdateLatestReleases.sh | 83 +++++++++ 6 files changed, 345 insertions(+), 2 deletions(-) create mode 100644 .ci/all-options-enabled-cache.cmake create mode 100644 .github/workflows/update-latest-releases.yml create mode 100644 releases/latest.releases.yaml create mode 100644 scripts/robotologyUpdateLatestReleases.sh diff --git a/.ci/all-options-enabled-cache.cmake b/.ci/all-options-enabled-cache.cmake new file mode 100644 index 000000000..8a98f7806 --- /dev/null +++ b/.ci/all-options-enabled-cache.cmake @@ -0,0 +1,26 @@ +# Dependencies options +set(ROBOTOLOGY_USES_GAZEBO ON CACHE BOOL "") +set(ROBOTOLOGY_USES_OCTAVE ON CACHE BOOL "") +set(ROBOTOLOGY_USES_PYTHON ON CACHE BOOL "") +set(ROBOTOLOGY_USES_MATLAB ON CACHE BOOL "") +set(ROBOTOLOGY_USES_OCULUS_SDK ON CACHE BOOL "") +set(ROBOTOLOGY_USES_CYBERITH_SDK ON CACHE BOOL "") +set(ROBOTOLOGY_USES_CFW2CAN ON CACHE BOOL "") +set(ROBOTOLOGY_USES_ESDCAN ON CACHE BOOL "") +set(ROBOTOLOGY_USES_XSENS_MVN_SDK ON CACHE BOOL "") + +# Profiles options +set(ROBOTOLOGY_ENABLE_ROBOT_TESTING ON CACHE BOOL "") +set(ROBOTOLOGY_ENABLE_DYNAMICS ON CACHE BOOL "") +set(ROBOTOLOGY_ENABLE_DYNAMICS_FULL_DEPS ON CACHE BOOL "") +set(ROBOTOLOGY_ENABLE_HUMAN_DYNAMICS ON CACHE BOOL "") +set(ROBOTOLOGY_ENABLE_ICUB_HEAD ON CACHE BOOL "") +set(ROBOTOLOGY_ENABLE_ICUB_BASIC_DEMOS ON CACHE BOOL "") +set(ROBOTOLOGY_ENABLE_TELEOPERATION ON CACHE BOOL "") +set(ROBOTOLOGY_ENABLE_EVENT_DRIVEN ON CACHE BOOL "") + + + + + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5eefaf9a4..551750354 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,7 +167,7 @@ jobs: matrix: build_type: [Release] os: [ubuntu-18.04, ubuntu-20.04, macos-10.15, macos-11.0, windows-2019] - project_tags: [Default, Unstable, Release202102] + project_tags: [Default, Unstable, LatestReleases, Release202102] include: - os: ubuntu-18.04 build_type: Release @@ -194,6 +194,8 @@ jobs: project_tags_cmake_options: "" - project_tags: Unstable project_tags_cmake_options: "-DROBOTOLOGY_PROJECT_TAGS=Unstable" + - project_tags: LatestReleases + project_tags_cmake_options: "-DROBOTOLOGY_PROJECT_TAGS=Custom -DROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE=${GITHUB_WORKSPACE}/releases/latest.releases.yaml" - project_tags: Release202102 project_tags_cmake_options: "-DROBOTOLOGY_PROJECT_TAGS=Custom -DROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE=${GITHUB_WORKSPACE}/releases/2021.02.yaml" steps: diff --git a/.github/workflows/update-latest-releases.yml b/.github/workflows/update-latest-releases.yml new file mode 100644 index 000000000..4c46ef883 --- /dev/null +++ b/.github/workflows/update-latest-releases.yml @@ -0,0 +1,53 @@ +name: Update latest releases +# This action checks if the releases/latest.releases.yaml file needs to be updated +# Check doc/developer-faqs.md for more info + +on: + workflow_dispatch: + schedule: + # Run the job once a week + - cron: '0 0 * * 0' + +jobs: + update-latest-releases: + name: "Update latest releases" + runs-on: [ubuntu-20.04] + steps: + - uses: actions/checkout@v2 + + - name: Install dependencies + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/v4.4.1/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + + - name: Download repositories + run: | + mkdir -p build + cd build + cmake -C ${GITHUB_WORKSPACE}/.ci/all-options-enabled-cache.cmake -DYCM_BOOTSTRAP_VERBOSE=ON -DNON_INTERACTIVE_BUILD:BOOL=TRUE .. + make update-all + + - name: Run script + run: | + bash ./scripts/robotologyUpdateLatestReleases.sh + + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v3 + with: + commit-message: Automatic update of latest releases versions + committer: GitHub + branch: automatic-update-latest-releases-versions + delete-branch: true + title: 'Automatic update of latest releases versions' + body: | + This is the **periodic automatic update** of the latest releases versions. + + Before merging, wait for the continuous integration outcome as it is possible that the automatic update of a given project created problems in the compilation of the overall superbuild. + + For more info, check the [developer FAQs documentation of the robotology-superbuild](https://github.com/robotology/robotology-superbuild/blob/master/doc/developers-faqs.md). + + - name: Check outputs + run: | + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/doc/developers-faqs.md b/doc/developers-faqs.md index 9c8c7d1ef..b3cbb1491 100644 --- a/doc/developers-faqs.md +++ b/doc/developers-faqs.md @@ -45,8 +45,14 @@ endif() * Add the profile documentation in [`doc/profile.md#profile-specific-documentation`](profiles.md#profile-specific-documentation). Take inspiration from the documentation of existing profiles. If the profile need a specific enviroment variable to be set of a value to be appended (such as `YARP_DATA_DIRS`), document it in the documentation and add it in the templates in https://github.com/robotology/robotology-superbuild/blob/master/cmake/template and in [`doc/environment-variables-configuration.md`](environment-variables-configuration.md). * Add the profile option in the `.ci/initial-cache.ci.cmake`, so it will be tested in the Continuous Integration and binaries will be generated for it. +## How to bump the version of a subproject +* The superbuild contains a `releases/latest.releases.yaml` file that is meant to contain the latest release of each subproject of the superbuild. +* This file is automatically updated by the `update-latest-releases.yml` GitHub action, that periodically checks the default branches of the repo (the one used if `ROBOTOLOGY_PROJECT_TAGS` is set to `Stable`) and extract the latest tag done on that branch. The action can also be run manually using the [`workflow_dispatch` event](https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/). +* If you have any project for which you want to manually manage the release used in the `releases/latest.releases.yaml` file, +you can disable the automatical update of the tags by adding its CMake name in the `projects_to_skip` array in the `scripts/robotologyUpdateLatestReleases.sh` script. + ## How to do a new release -* Sometime before the release, add a `yyyy.mm.yaml` file in https://github.com/robotology/robotology-superbuild/tree/master/releases, containing the version of package contained in the new release. +* Sometime before the release, copy the `releases/latest.releases.yaml` file in `releases/yyyy.mm.yaml` file, containing the version of package contained in the new release. * Modify the CI scripts to start testing the `yyyy.mm.yaml` * Once the release is ready to be made, create a `releases/yyyy.mm` branch from `master` * On the branch `releases/yyyy.mm` modify the default value of the `ROBOTOLOGY_PROJECT_TAGS` CMake option to be `Custom` and of the `ROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE` to the point to the `yyyy.mm.yaml` file. diff --git a/releases/latest.releases.yaml b/releases/latest.releases.yaml new file mode 100644 index 000000000..bfeaf33e5 --- /dev/null +++ b/releases/latest.releases.yaml @@ -0,0 +1,173 @@ +repositories: + qpOASES: + type: git + url: https://github.com/robotology-dependencies/qpOASES.git + version: v3.2.0.1 + osqp: + type: git + url: https://github.com/oxfordcontrol/osqp.git + version: v0.6.0 + manif: + type: git + url: https://github.com/artivis/manif.git + version: 44bdfebff0fbc56cb189f680212257dc7f20ea58 + qhull: + type: git + url: https://github.com/qhull/qhull.git + version: v8.0.2 + CppAD: + type: git + url: https://github.com/coin-or/CppAD.git + version: 4aac52664de911af02cfade06131a9a6f6b48d7e + casadi: + type: git + url: https://github.com/GiulioRomualdi/casadi.git + version: a26cd8ffba99052b74553eec1daeff640eea7e79 + YCM: + type: git + url: https://github.com/robotology/ycm.git + version: v0.12.1 + YARP: + type: git + url: https://github.com/robotology/yarp.git + version: 513cf7a32c83c9c96abb51a9535cf8714e027e6b + ICUB: + type: git + url: https://github.com/robotology/icub-main.git + version: v1.18.0 + ICUBcontrib: + type: git + url: https://github.com/robotology/icub-contrib-common.git + version: v1.18.0 + robots-configuration: + type: git + url: https://github.com/robotology/robots-configuration.git + version: v1.18.0 + GazeboYARPPlugins: + type: git + url: https://github.com/robotology/gazebo-yarp-plugins.git + version: v3.5.1 + icub-gazebo: + type: git + url: https://github.com/robotology/icub-gazebo.git + version: v1.18.0 + icub-models: + type: git + url: https://github.com/robotology/icub-models.git + version: v1.18.1 + yarp-matlab-bindings: + type: git + url: https://github.com/robotology/yarp-matlab-bindings.git + version: v3.4.0 + RobotTestingFramework: + type: git + url: https://github.com/robotology/robot-testing-framework.git + version: v2.0.1 + icub-tests: + type: git + url: https://github.com/robotology/icub-tests.git + version: v1.18.0 + blocktestcore: + type: git + url: https://github.com/robotology/blocktest.git + version: v2.3.0 + blocktest-yarp-plugins: + type: git + url: https://github.com/robotology/blocktest-yarp-plugins.git + version: v1.1.0 + iDynTree: + type: git + url: https://github.com/robotology/idyntree.git + version: v2.0.1 + BlockFactory: + type: git + url: https://github.com/robotology/blockfactory.git + version: v0.8.1 + WBToolbox: + type: git + url: https://github.com/robotology/wb-toolbox.git + version: v5.3 + OsqpEigen: + type: git + url: https://github.com/robotology/osqp-eigen.git + version: v0.6.2 + UnicyclePlanner: + type: git + url: https://github.com/robotology/unicycle-footstep-planner.git + version: v0.3.0 + walking-controllers: + type: git + url: https://github.com/robotology/walking-controllers.git + version: v0.3.3 + icub-gazebo-wholebody: + type: git + url: https://github.com/robotology/icub-gazebo-wholebody.git + version: v0.1.0 + whole-body-controllers: + type: git + url: https://github.com/robotology/whole-body-controllers.git + version: v2.5 + whole-body-estimators: + type: git + url: https://github.com/robotology/whole-body-estimators.git + version: v0.3.0 + walking-teleoperation: + type: git + url: https://github.com/robotology/walking-teleoperation.git + version: v0.2.0 + forcetorque-yarp-devices: + type: git + url: https://github.com/robotology/forcetorque-yarp-devices.git + version: v0.2.0 + wearables: + type: git + url: https://github.com/robotology/wearables.git + version: v1.1.1 + human-dynamics-estimation: + type: git + url: https://github.com/robotology/human-dynamics-estimation.git + version: v2.1.0 + human-gazebo: + type: git + url: https://github.com/robotology/human-gazebo.git + version: v1.0 + icub_firmware_shared: + type: git + url: https://github.com/robotology/icub-firmware-shared.git + version: v1.18.0 + icub-firmware: + type: git + url: https://github.com/robotology/icub-firmware.git + version: v1.18.0 + icub-firmware-build: + type: git + url: https://github.com/robotology/icub-firmware-build.git + version: v1.18.0 + yarp-device-xsensmt: + type: git + url: https://github.com/robotology/yarp-device-xsensmt.git + version: v0.1.0 + speech: + type: git + url: https://github.com/robotology/speech.git + version: v1.0.0 + icub-basic-demos: + type: git + url: https://github.com/robotology/icub-basic-demos.git + version: v1.18.0 + funny-things: + type: git + url: https://github.com/robotology/funny-things.git + version: v1.0.0 + bipedal-locomotion-framework: + type: git + url: https://github.com/dic-iit/bipedal-locomotion-framework.git + version: 46d8f68c51fad71ae06db8686fa88218f7404923 + LieGroupControllers: + type: git + url: https://github.com/dic-iit/lie-group-controllers.git + version: v0.0.1 + event-driven: + type: git + url: https://github.com/robotology/event-driven.git + version: v1.4 diff --git a/scripts/robotologyUpdateLatestReleases.sh b/scripts/robotologyUpdateLatestReleases.sh new file mode 100644 index 000000000..ee79609db --- /dev/null +++ b/scripts/robotologyUpdateLatestReleases.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Script to automatically update the latest.releases.yaml file +# With the latest tag done on the stable branches of a given repo + +# To run it locally, download yq binary from https://github.com/mikefarah/yq +# rename it to yq, and put it in a directory that you add to the path. +# Then, navigate to robotology-superbuild dir and execute in Bash: +# ./scripts/robotologyUpdateLatestReleases.sh +# Warning: if you run, it will update the tags only of the repos +# that are enabled and downloaded, so it is intendend to run after all the +# components and options have been enabled, and after the update-all target +# has been run + +# This array contains the project that for a given reason should not be updated by this script +# ICUBcontrib is here as it has several coincident tags, and the script currently is not able +# to get the last one +# External repos are in the list as we updated them manually +projects_to_skip=("ICUBcontrib" "qhull" "CppAD" "casadi" "manif" "osqp") + +getParentDir () { + SOURCE="${1}" + while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located + done + DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" +} + +getParentDir "${BASH_SOURCE[0]}" + +script_dir="$DIR" + +getParentDir $script_dir + +superbuild_root="$DIR" + +subdirs="$superbuild_root/src" + +latest_releases_yaml_file="$superbuild_root/releases/latest.releases.yaml" + +# Check if element is in array +# https://stackoverflow.com/a/8574392 +containsElement () { + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 0; done + return 1 +} + +updateLatestRelease () { + cd $1 + # Extract package name + package_name=`basename $1` + # Check if package is in skip list + containsElement "$package_name" "${projects_to_skip[@]}" + is_contained=$? + if [ $is_contained == 1 ] + then + # Extract latest tag + latest_tag=`git describe --abbrev=0 --tags` + # Update latest tag in latest-release.yaml file, + # using https://github.com/mikefarah/yq + # Only update non-empty tag + if [ ! -z "$latest_tag" ] + then + echo "Setting ${package_name} tag to ${latest_tag} ." + yq -i eval ".repositories.${package_name}.version=\"${latest_tag}\"" ${latest_releases_yaml_file} + fi + else + echo "Skipping update of ${package_name} as it is in the list of projects to skip." + fi +} + +for subdir in ${subdirs}; do \ + if [ -d "${subdir}" ] ; then + for i in ${subdir}/*/; do \ + updateLatestRelease $i + done + fi +done + From bd619b4fa64a27dc82b34d40a3d991d20642ebfd Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Tue, 19 Jan 2021 22:03:54 +0100 Subject: [PATCH 2/3] Update latest.releases.yaml --- releases/latest.releases.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releases/latest.releases.yaml b/releases/latest.releases.yaml index bfeaf33e5..9f7a24365 100644 --- a/releases/latest.releases.yaml +++ b/releases/latest.releases.yaml @@ -170,4 +170,4 @@ repositories: event-driven: type: git url: https://github.com/robotology/event-driven.git - version: v1.4 + version: 62523c05032f7b56c174a882e594cefd036f7590 From 379640c3e2f61f1fc71f232ad8f16d536a6e7663 Mon Sep 17 00:00:00 2001 From: Silvio Traversaro Date: Tue, 19 Jan 2021 22:05:13 +0100 Subject: [PATCH 3/3] Update robotologyUpdateLatestReleases.sh --- scripts/robotologyUpdateLatestReleases.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/robotologyUpdateLatestReleases.sh b/scripts/robotologyUpdateLatestReleases.sh index ee79609db..b390aabba 100644 --- a/scripts/robotologyUpdateLatestReleases.sh +++ b/scripts/robotologyUpdateLatestReleases.sh @@ -16,7 +16,8 @@ # ICUBcontrib is here as it has several coincident tags, and the script currently is not able # to get the last one # External repos are in the list as we updated them manually -projects_to_skip=("ICUBcontrib" "qhull" "CppAD" "casadi" "manif" "osqp") +# event-driven does not have a recent release +projects_to_skip=("ICUBcontrib" "qhull" "CppAD" "casadi" "manif" "osqp" "event-driven") getParentDir () { SOURCE="${1}"