Automatize the process of bumping the released version of superbuild …

…subprojects (#596) * Change workflow to bump version releases Do it automatically via GitHub Actions
robotology · Jan 20, 2021 · 2da222b · 2da222b
1 parent c4ade97
commit 2da222b
Show file tree

Hide file tree

Showing 6 changed files with 346 additions and 2 deletions.
diff --git a/.ci/all-options-enabled-cache.cmake b/.ci/all-options-enabled-cache.cmake
@@ -0,0 +1,26 @@
+# Dependencies options
+set(ROBOTOLOGY_USES_GAZEBO ON CACHE BOOL "")
+set(ROBOTOLOGY_USES_OCTAVE ON CACHE BOOL "")
+set(ROBOTOLOGY_USES_PYTHON ON CACHE BOOL "")
+set(ROBOTOLOGY_USES_MATLAB ON CACHE BOOL "")
+set(ROBOTOLOGY_USES_OCULUS_SDK ON CACHE BOOL "")
+set(ROBOTOLOGY_USES_CYBERITH_SDK ON CACHE BOOL "")
+set(ROBOTOLOGY_USES_CFW2CAN ON CACHE BOOL "")
+set(ROBOTOLOGY_USES_ESDCAN ON CACHE BOOL "")
+set(ROBOTOLOGY_USES_XSENS_MVN_SDK ON CACHE BOOL "")
+
+# Profiles options
+set(ROBOTOLOGY_ENABLE_ROBOT_TESTING ON CACHE BOOL "")
+set(ROBOTOLOGY_ENABLE_DYNAMICS ON CACHE BOOL "")
+set(ROBOTOLOGY_ENABLE_DYNAMICS_FULL_DEPS ON CACHE BOOL "")
+set(ROBOTOLOGY_ENABLE_HUMAN_DYNAMICS ON CACHE BOOL "")
+set(ROBOTOLOGY_ENABLE_ICUB_HEAD ON CACHE BOOL "")
+set(ROBOTOLOGY_ENABLE_ICUB_BASIC_DEMOS ON CACHE BOOL "")
+set(ROBOTOLOGY_ENABLE_TELEOPERATION ON CACHE BOOL "")
+set(ROBOTOLOGY_ENABLE_EVENT_DRIVEN ON CACHE BOOL "")
+
+
+
+
+
+
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -167,7 +167,7 @@ jobs:
       matrix:
         build_type: [Release]
         os: [ubuntu-18.04, ubuntu-20.04, macos-10.15, macos-11.0, windows-2019]
-        project_tags: [Default, Unstable, Release202102]
+        project_tags: [Default, Unstable, LatestReleases, Release202102]
         include:
           - os: ubuntu-18.04
             build_type: Release
@@ -194,6 +194,8 @@ jobs:
             project_tags_cmake_options: ""
           - project_tags: Unstable
             project_tags_cmake_options: "-DROBOTOLOGY_PROJECT_TAGS=Unstable"
+          - project_tags: LatestReleases
+            project_tags_cmake_options: "-DROBOTOLOGY_PROJECT_TAGS=Custom -DROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE=${GITHUB_WORKSPACE}/releases/latest.releases.yaml"
           - project_tags: Release202102
             project_tags_cmake_options: "-DROBOTOLOGY_PROJECT_TAGS=Custom -DROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE=${GITHUB_WORKSPACE}/releases/2021.02.yaml"
     steps:

diff --git a/.github/workflows/update-latest-releases.yml b/.github/workflows/update-latest-releases.yml
@@ -0,0 +1,53 @@
+name: Update latest releases
+# This action checks if the releases/latest.releases.yaml file needs to be updated
+# Check doc/developer-faqs.md for more info
+
+on:
+  workflow_dispatch:
+  schedule:
+  # Run the job once a week
+  - cron: '0 0 * * 0'
+
+jobs:
+    update-latest-releases:
+        name: "Update latest releases"
+        runs-on: [ubuntu-20.04]
+        steps:
+        - uses: actions/checkout@v2
+
+        - name: Install dependencies
+          run: |
+            sudo wget https://github.com/mikefarah/yq/releases/download/v4.4.1/yq_linux_amd64 -O /usr/bin/yq
+            sudo chmod +x /usr/bin/yq
+
+        - name: Download repositories
+          run: |
+            mkdir -p build
+            cd build
+            cmake -C ${GITHUB_WORKSPACE}/.ci/all-options-enabled-cache.cmake -DYCM_BOOTSTRAP_VERBOSE=ON -DNON_INTERACTIVE_BUILD:BOOL=TRUE ..
+            make update-all
+
+        - name: Run script
+          run: |
+            bash ./scripts/robotologyUpdateLatestReleases.sh
+
+        - name: Create Pull Request
+          id: cpr
+          uses: peter-evans/create-pull-request@v3
+          with:
+            commit-message: Automatic update of latest releases versions
+            committer: GitHub <noreply@github.com>
+            branch: automatic-update-latest-releases-versions
+            delete-branch: true
+            title: 'Automatic update of latest releases versions'
+            body: |
+              This is the **periodic automatic update** of the latest releases versions.
+              
+              Before merging, wait for the continuous integration outcome as it is possible that the automatic update of a given project created problems in the compilation of the overall superbuild.
+              
+              For more info, check the [developer FAQs documentation of the robotology-superbuild](https://github.com/robotology/robotology-superbuild/blob/master/doc/developers-faqs.md). 
+
+        - name: Check outputs
+          run: |
+            echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+            echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
diff --git a/doc/developers-faqs.md b/doc/developers-faqs.md
@@ -45,8 +45,14 @@ endif()
 * Add the profile documentation in [`doc/profile.md#profile-specific-documentation`](profiles.md#profile-specific-documentation). Take inspiration from the documentation of existing profiles. If the profile need a specific enviroment variable to be set of a value to be appended (such as `YARP_DATA_DIRS`), document it in the documentation and add it in the templates in https://github.com/robotology/robotology-superbuild/blob/master/cmake/template and in [`doc/environment-variables-configuration.md`](environment-variables-configuration.md).
 * Add the profile option in the `.ci/initial-cache.ci.cmake`, so it will be tested in the Continuous Integration and binaries will be generated for it.
 
+## How to bump the version of a subproject
+* The superbuild contains a `releases/latest.releases.yaml` file that is meant to contain the latest release of each subproject of the superbuild.
+* This file is automatically updated by the `update-latest-releases.yml` GitHub action, that periodically checks the default branches of the repo (the one used if `ROBOTOLOGY_PROJECT_TAGS` is set to `Stable`) and extract the latest tag done on that branch. The action can also be run manually using the [`workflow_dispatch` event](https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/).
+* If you have any project for which you want to manually manage the release used in the `releases/latest.releases.yaml` file,
+you can disable the automatical update of the tags by adding its CMake name in the `projects_to_skip` array in the `scripts/robotologyUpdateLatestReleases.sh` script.
+
 ## How to do a new release
-* Sometime before the release, add a `yyyy.mm.yaml` file in https://github.com/robotology/robotology-superbuild/tree/master/releases, containing the version of package contained in the new release.
+* Sometime before the release, copy the `releases/latest.releases.yaml` file in `releases/yyyy.mm.yaml` file, containing the version of package contained in the new release.
 * Modify the CI scripts to start testing the `yyyy.mm.yaml`
 * Once the release is ready to be made, create a `releases/yyyy.mm` branch from `master`
 * On the branch `releases/yyyy.mm` modify the default value of the `ROBOTOLOGY_PROJECT_TAGS` CMake option to be `Custom` and of the `ROBOTOLOGY_PROJECT_TAGS_CUSTOM_FILE` to the point to the `yyyy.mm.yaml` file.

diff --git a/releases/latest.releases.yaml b/releases/latest.releases.yaml
@@ -0,0 +1,173 @@
+repositories:
+  qpOASES:
+    type: git
+    url: https://github.com/robotology-dependencies/qpOASES.git
+    version: v3.2.0.1
+  osqp:
+    type: git
+    url: https://github.com/oxfordcontrol/osqp.git
+    version: v0.6.0
+  manif:
+    type: git
+    url: https://github.com/artivis/manif.git
+    version: 44bdfebff0fbc56cb189f680212257dc7f20ea58
+  qhull:
+    type: git
+    url: https://github.com/qhull/qhull.git
+    version: v8.0.2
+  CppAD:
+    type: git
+    url: https://github.com/coin-or/CppAD.git
+    version: 4aac52664de911af02cfade06131a9a6f6b48d7e
+  casadi:
+    type: git
+    url: https://github.com/GiulioRomualdi/casadi.git
+    version: a26cd8ffba99052b74553eec1daeff640eea7e79
+  YCM:
+    type: git
+    url: https://github.com/robotology/ycm.git
+    version: v0.12.1
+  YARP:
+    type: git
+    url: https://github.com/robotology/yarp.git
+    version: 513cf7a32c83c9c96abb51a9535cf8714e027e6b
+  ICUB:
+    type: git
+    url: https://github.com/robotology/icub-main.git
+    version: v1.18.0
+  ICUBcontrib:
+    type: git
+    url: https://github.com/robotology/icub-contrib-common.git
+    version: v1.18.0
+  robots-configuration:
+    type: git
+    url: https://github.com/robotology/robots-configuration.git
+    version: v1.18.0
+  GazeboYARPPlugins:
+    type: git
+    url: https://github.com/robotology/gazebo-yarp-plugins.git
+    version: v3.5.1
+  icub-gazebo:
+    type: git
+    url: https://github.com/robotology/icub-gazebo.git
+    version: v1.18.0
+  icub-models:
+    type: git
+    url: https://github.com/robotology/icub-models.git
+    version: v1.18.1
+  yarp-matlab-bindings:
+    type: git
+    url: https://github.com/robotology/yarp-matlab-bindings.git
+    version: v3.4.0
+  RobotTestingFramework:
+    type: git
+    url: https://github.com/robotology/robot-testing-framework.git
+    version: v2.0.1
+  icub-tests:
+    type: git
+    url: https://github.com/robotology/icub-tests.git
+    version: v1.18.0
+  blocktestcore:
+    type: git
+    url: https://github.com/robotology/blocktest.git
+    version: v2.3.0
+  blocktest-yarp-plugins:
+    type: git
+    url: https://github.com/robotology/blocktest-yarp-plugins.git
+    version: v1.1.0
+  iDynTree:
+    type: git
+    url: https://github.com/robotology/idyntree.git
+    version: v2.0.1
+  BlockFactory:
+    type: git
+    url: https://github.com/robotology/blockfactory.git
+    version: v0.8.1
+  WBToolbox:
+    type: git
+    url: https://github.com/robotology/wb-toolbox.git
+    version: v5.3
+  OsqpEigen:
+    type: git
+    url: https://github.com/robotology/osqp-eigen.git
+    version: v0.6.2
+  UnicyclePlanner:
+    type: git
+    url: https://github.com/robotology/unicycle-footstep-planner.git
+    version: v0.3.0
+  walking-controllers:
+    type: git
+    url: https://github.com/robotology/walking-controllers.git
+    version: v0.3.3
+  icub-gazebo-wholebody:
+    type: git
+    url: https://github.com/robotology/icub-gazebo-wholebody.git
+    version: v0.1.0
+  whole-body-controllers:
+    type: git
+    url: https://github.com/robotology/whole-body-controllers.git
+    version: v2.5
+  whole-body-estimators:
+    type: git
+    url: https://github.com/robotology/whole-body-estimators.git
+    version: v0.3.0
+  walking-teleoperation:
+    type: git
+    url: https://github.com/robotology/walking-teleoperation.git
+    version: v0.2.0
+  forcetorque-yarp-devices:
+    type: git
+    url: https://github.com/robotology/forcetorque-yarp-devices.git
+    version: v0.2.0
+  wearables:
+    type: git
+    url: https://github.com/robotology/wearables.git
+    version: v1.1.1
+  human-dynamics-estimation:
+    type: git
+    url: https://github.com/robotology/human-dynamics-estimation.git
+    version: v2.1.0
+  human-gazebo:
+    type: git
+    url: https://github.com/robotology/human-gazebo.git
+    version: v1.0
+  icub_firmware_shared:
+    type: git
+    url: https://github.com/robotology/icub-firmware-shared.git
+    version: v1.18.0
+  icub-firmware:
+    type: git
+    url: https://github.com/robotology/icub-firmware.git
+    version: v1.18.0
+  icub-firmware-build:
+    type: git
+    url: https://github.com/robotology/icub-firmware-build.git
+    version: v1.18.0
+  yarp-device-xsensmt:
+    type: git
+    url: https://github.com/robotology/yarp-device-xsensmt.git
+    version: v0.1.0
+  speech:
+    type: git
+    url: https://github.com/robotology/speech.git
+    version: v1.0.0
+  icub-basic-demos:
+    type: git
+    url: https://github.com/robotology/icub-basic-demos.git
+    version: v1.18.0
+  funny-things:
+    type: git
+    url: https://github.com/robotology/funny-things.git
+    version: v1.0.0
+  bipedal-locomotion-framework:
+    type: git
+    url: https://github.com/dic-iit/bipedal-locomotion-framework.git
+    version: 46d8f68c51fad71ae06db8686fa88218f7404923
+  LieGroupControllers:
+    type: git
+    url: https://github.com/dic-iit/lie-group-controllers.git
+    version: v0.0.1
+  event-driven:
+    type: git
+    url: https://github.com/robotology/event-driven.git
+    version: 62523c05032f7b56c174a882e594cefd036f7590
diff --git a/scripts/robotologyUpdateLatestReleases.sh b/scripts/robotologyUpdateLatestReleases.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+# Script to automatically update the latest.releases.yaml file
+# With the latest tag done on the stable branches of a given repo
+
+# To run it locally, download yq binary from https://github.com/mikefarah/yq
+# rename it to yq, and put it in a directory that you add to the path.
+# Then, navigate to robotology-superbuild dir and execute in Bash:
+# ./scripts/robotologyUpdateLatestReleases.sh
+# Warning: if you run, it will update the tags only of the repos
+# that are enabled and downloaded, so it is intendend to run after all the
+# components and options have been enabled, and after the update-all target
+# has been run
+
+# This array contains the project that for a given reason should not be updated by this script
+# ICUBcontrib is here as it has several coincident tags, and the script currently is not able
+# to get the last one
+# External repos are in the list as we updated them manually
+# event-driven does not have a recent release
+projects_to_skip=("ICUBcontrib" "qhull" "CppAD" "casadi" "manif" "osqp" "event-driven")
+
+getParentDir () {
+    SOURCE="${1}"
+    while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+      DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
+      SOURCE="$(readlink "$SOURCE")"
+      [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+    done
+    DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
+}
+
+getParentDir "${BASH_SOURCE[0]}"
+
+script_dir="$DIR"
+
+getParentDir $script_dir
+
+superbuild_root="$DIR"
+
+subdirs="$superbuild_root/src"
+
+latest_releases_yaml_file="$superbuild_root/releases/latest.releases.yaml"
+
+# Check if element is in array
+# https://stackoverflow.com/a/8574392
+containsElement () {
+  local e match="$1"
+  shift
+  for e; do [[ "$e" == "$match" ]] && return 0; done
+  return 1
+}
+
+updateLatestRelease () {
+    cd $1
+    # Extract package name
+    package_name=`basename $1`
+    # Check if package is in skip list
+    containsElement "$package_name" "${projects_to_skip[@]}"
+    is_contained=$?
+    if [ $is_contained == 1 ]
+    then
+        # Extract latest tag
+        latest_tag=`git describe --abbrev=0 --tags`
+        # Update latest tag in latest-release.yaml file,
+        # using https://github.com/mikefarah/yq
+        # Only update non-empty tag
+        if [ ! -z "$latest_tag" ]
+        then
+            echo "Setting ${package_name} tag to ${latest_tag} ."
+            yq -i eval ".repositories.${package_name}.version=\"${latest_tag}\"" ${latest_releases_yaml_file}
+        fi
+    else
+        echo "Skipping update of ${package_name} as it is in the list of projects to skip."
+    fi
+}
+
+for subdir in ${subdirs}; do \
+    if [ -d "${subdir}" ] ; then
+        for i in ${subdir}/*/; do \
+            updateLatestRelease $i
+        done
+    fi
+done
+