Download libs script update (adds #7907 wget2) - params and fixes dep…

…loy logs (#8002) * downloader in parallel (cherry picked from commit 6e8819c) * Update download_libs.sh - parallel downloads (cherry picked from commit 2391c4f) * wget2 changes downloader fixes * Updated downloader script to support changes for parellel downloads, no ssl, wget2, curl, wget Cleaned up download_libs with better logs and overwrite settings to the target download type * downloader fix up URLs parellel * Downloader fix parallel curl - add commands * Apothecary Fix up commands * Download script osx - no need to send -a causing double download * Downloader updates - fixes command determination artefact now tracks installed of curl/wget2/wget in variable - added wget2 params for override or continue last download (-nc) - t20 retries - console formatting - added error message if no commands installed * cURL Test Ci * wget og Ci test * wget2 Ci Test / SSL off * download libs ci just BITS target * ci focused VS bits off for the moment * ci test downloader --silent * Download libs fix arch * Download script v3.0.1 - fixes for cURL in silent mode cURL now has progress bar Version in echo Wget2/wget Verbose turned off Download_libs silent now passed as bool * Download_libs fixed Overwrite - Overwrite / remove old libs was not working at all - not looping correctly for core libs - rewrote loop - Overwrite now removes old libs for that target type for the download... will remove only osx if download osx etc rather than all for that library - Overwrite Addons now will work if folders are deleted * downloader 3.1.0 wget/wget2 - now checks remote file vs local if found and optimises - won't download if same * download_libs 2.1 - - files downloaded (zip/tar.bz2) are now stored in libs/download/ - creates libs/download for downloading zips/tars for optimisation - clean up of logs - not removing dl zip now - code to remove zips on start also done commented out * download_libs - set PLATFORM to target if set - include overwrite off * downloader 3.1.1 - fixes for cURL - no ssl fixed * github actions workflow fix for concurrency in PRs being cancelled by other PRs of not the same hash. fallback to sha for non-prs * cURL optimisation with remote and local file check. Using HEAD flags so it just queries the content size and time rather than getting whole file * github actions cleanup cache * Linux Cache off * apt-get broken for ssl-dev * no cache on libssl * cache remove ssl/curl * chmod +x for msys2 * cache disable for linux * linux cache? * ubuntu-22.04 * Downloader 3.2.1 - wget2 re-enabled default if available --------- Co-authored-by: Dimitre <dimitre@users.noreply.github.com>
openframeworks · May 30, 2024 · 676ae97 · 676ae97
1 parent 12f4a84
commit 676ae97
Show file tree

Hide file tree

Showing 17 changed files with 317 additions and 63 deletions.
diff --git a/.github/workflows/actions/build-vs2022.yml b/.github/workflows/actions/build-vs2022.yml
@@ -38,6 +38,7 @@ jobs:
             git
             unzip
             rsync
+            wget2
             wget
       - name: Install dependencies
         shell: msys2 {0}

diff --git a/.github/workflows/clean-up-cache.yml b/.github/workflows/clean-up-cache.yml
@@ -0,0 +1,33 @@
+name: cleanup caches by a branch
+on:
+  pull_request:
+    types:
+      - closed
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Cleanup
+        run: |
+          gh extension install actions/gh-actions-cache
+          REPO=${{ github.repository }}
+          BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
+          echo "Fetching list of cache key"
+          cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH -L 100 | cut -f 1 )
+          ## Setting this to not fail the workflow while deleting cache keys.
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeysForPR
+          do
+              gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/of.yml b/.github/workflows/of.yml
@@ -13,7 +13,7 @@ on:
       - 'examples/**'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: true
 
 env:
@@ -201,7 +201,7 @@ jobs:
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 
 
   build-linux64:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
     strategy:
         matrix:
           cfg:
@@ -216,7 +216,7 @@ jobs:
     - name: Cache Packages
       uses: awalsh128/cache-apt-pkgs-action@latest
       with:
-        packages: aptitude aptitude-common libboost-iostreams1.83.0 libcwidget4 libsigc++-2.0-0v5 libxapian30 fonts-wine{a} libasound2-plugins{a} libcapi20-3t64{a} libosmesa6{a} libpcsclite1{a} libspeexdsp1{a} libwine{a} libxkbregistry0{a} libz-mingw-w64{a} wine{a} wine64 wget2 make curl libjack-jackd2-0 libjack-jackd2-dev freeglut3-dev libasound2-dev libxmu-dev libxxf86vm-dev g++ libgl1-mesa-dev libglu1-mesa-dev libraw1394-dev libudev-dev libdrm-dev libglew-dev libopenal-dev libsndfile1-dev libfreeimage-dev libcairo2-dev libfreetype6-dev libssl-dev libpulse-dev libusb-1.0-0-dev libgtk2.0-dev libopencv-dev libassimp-dev librtaudio-dev gdb libglfw3-dev liburiparser-dev libcurl4-openssl-dev libpugixml-dev libgconf-2-4 libgtk2.0-0 libpoco-dev libxcursor-dev libxi-dev libxinerama-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev gstreamer1.0-libav gstreamer1.0-pulseaudio gstreamer1.0-x gstreamer1.0-plugins-bad gstreamer1.0-alsa gstreamer1.0-plugins-base gstreamer1.0-plugins-good 
+        packages: aptitude aptitude-common libboost-iostreams1.83.0 libcwidget4 libsigc++-2.0-0v5 libxapian30 fonts-wine{a} libasound2-plugins{a} libcapi20-3t64{a} libosmesa6{a} libpcsclite1{a} libspeexdsp1{a} libwine{a} libxkbregistry0{a} libz-mingw-w64{a} wine{a} wine64 wget2 make libjack-jackd2-0 libjack-jackd2-dev freeglut3-dev libasound2-dev libxmu-dev libxxf86vm-dev g++ libgl1-mesa-dev libglu1-mesa-dev libraw1394-dev libudev-dev libdrm-dev libglew-dev libopenal-dev libsndfile1-dev libfreeimage-dev libcairo2-dev libfreetype6-dev libpulse-dev libusb-1.0-0-dev libgtk2.0-dev libopencv-dev libassimp-dev librtaudio-dev gdb libglfw3-dev liburiparser-dev libpugixml-dev libgconf-2-4 libgtk2.0-0 libpoco-dev libxcursor-dev libxi-dev libxinerama-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev gstreamer1.0-libav gstreamer1.0-pulseaudio gstreamer1.0-x gstreamer1.0-plugins-bad gstreamer1.0-alsa gstreamer1.0-plugins-base gstreamer1.0-plugins-good 
         version: 1.0
 
     - uses: actions/checkout@v4

diff --git a/scripts/ci/msys2/build.sh b/scripts/ci/msys2/build.sh
diff --git a/scripts/ci/msys2/install.sh b/scripts/ci/msys2/install.sh
diff --git a/scripts/ci/msys2/run_tests.sh b/scripts/ci/msys2/run_tests.sh
diff --git a/scripts/ci/vs/install.sh b/scripts/ci/vs/install.sh
diff --git a/scripts/ci/vs/run_tests.bat b/scripts/ci/vs/run_tests.bat
diff --git a/scripts/dev/download_libs.sh b/scripts/dev/download_libs.sh
@@ -5,7 +5,9 @@ PLATFORM=""
 ARCH=""
 OVERWRITE=1
 SILENT_ARGS=""
+NO_SSL=""
 BLEEDING_EDGE=0
+DL_VERSION=2.2
 
 printHelp(){
 cat << EOF
@@ -25,6 +27,7 @@ cat << EOF
                                 If not set deletes any existing libraries
     -s, --silent                Silent download progress
     -h, --help                  Shows this message
+    -k, --no-ssl                Allow no SSL validation
 EOF
 }
 
@@ -33,15 +36,21 @@ if [[ ! -d "$SCRIPT_DIR" ]]; then SCRIPT_DIR="$PWD"; fi
 . "$SCRIPT_DIR/downloader.sh"
 
 download(){
-    echo "Downloading $1"
+    echo ' -----'
+    #echo " Downloading $1"
     # downloader ci.openframeworks.cc/libs/$1 $SILENT_ARGS
+
+    COMMAND=" "
+    REPO="nightly"
     if [[ $BLEEDING_EDGE = 1 ]] ; then
-        echo downloader https://github.com/openframeworks/apothecary/releases/download/bleeding/$1 $SILENT_ARGS
-        downloader https://github.com/openframeworks/apothecary/releases/download/bleeding/$1 $SILENT_ARGS
-    else
-        echo downloader https://github.com/openframeworks/apothecary/releases/download/nightly/$1 $SILENT_ARGS
-        downloader https://github.com/openframeworks/apothecary/releases/download/nightly/$1 $SILENT_ARGS
+        REPO="bleeding"
     fi
+
+    for PKG in $1; do
+        COMMAND+="https://github.com/openframeworks/apothecary/releases/download/$REPO/$PKG "
+    done
+    # echo $COMMAND;
+    downloader $COMMAND $SILENT_ARGS $NO_SSL
 }
 
 # trap any script errors and exit
@@ -89,7 +98,10 @@ while [[ $# -gt 0 ]]; do
         BLEEDING_EDGE=1
         ;;
         -s|--silent)
-        SILENT_ARGS=-nv
+        SILENT_ARGS=1
+        ;;
+        -k|--no-ssl)
+        NO_SSL=1
         ;;
         -h|--help)
         printHelp
@@ -104,6 +116,10 @@ while [[ $# -gt 0 ]]; do
     shift # past argument or value
 done
 
+if [[ "$TARGET" != "" ]] && [[ "$PLATFORM" == "" ]]; then
+    PLATFORM=$TARGET
+fi
+
 if [ "$PLATFORM" == "" ]; then
     OS=$(uname)
     if [ "$OS" == "Linux" ]; then
@@ -176,6 +192,8 @@ if [ "$PLATFORM" == "linux" ] && [ "$ARCH" == "64" ]; then
     fi
 fi
 
+echo " openFrameworks download_libs.sh v$DL_VERSION"
+
 if [ "$PLATFORM" == "msys2" ]; then
     PKGS="openFrameworksLibs_${VER}_${PLATFORM}_${ARCH}.zip"
 elif [ "$ARCH" == "" ] && [ "$PLATFORM" == "vs" ]; then
@@ -207,9 +225,9 @@ elif [[ "$PLATFORM" =~ ^(osx|ios|tvos|xros|catos|watchos)$ ]]; then
 elif [ "$ARCH" == "" ] && [ "$PLATFORM" == "android" ]; then
     if [[ $BLEEDING_EDGE = 1 ]] ; then
         PKGS="openFrameworksLibs_${VER}_${PLATFORM}_armv7.tar.bz2 \
-          openFrameworksLibs_${VER}_${PLATFORM}_arm64.tar.bz2 \
-           openFrameworksLibs_${VER}_${PLATFORM}_x86_64.tar.bz2
-          openFrameworksLibs_${VER}_${PLATFORM}_x86.tar.bz2"
+              openFrameworksLibs_${VER}_${PLATFORM}_arm64.tar.bz2 \
+              openFrameworksLibs_${VER}_${PLATFORM}_x86_64.tar.bz2
+              openFrameworksLibs_${VER}_${PLATFORM}_x86.tar.bz2"
     else
         PKGS="openFrameworksLibs_${VER}_${PLATFORM}armv7.tar.bz2 \
           openFrameworksLibs_${VER}_${PLATFORM}arm64.tar.bz2 \
@@ -223,33 +241,63 @@ else # Linux
     fi
 fi
 
-for PKG in $PKGS; do
-    download $PKG
-done
-
 cd ../../
 mkdir -p libs
 cd libs
 
+mkdir -p download
+cd download
+
+# IFS=' ' read -r -a PKGS_DATA <<< "$PKGS"
+# if [ $OVERWRITE -eq 1 ]; then
+#     for ((i = 0; i < ${#PKGS_DATA[@]}; i++)); do
+#         FILE_CHECK="${PKGS_DATA[$i]}"
+#         # Check if the file exists
+#         if [ -e "${FILE_CHECK}" ]; then
+#             echo " Removing Prior Download:[${FILE_CHECK}]"
+#             # Remove the file or directory
+#             rm -rf "${FILE_CHECK}"
+#         fi
+#     done
+# fi
+
+download "${PKGS[@]}"
+
+cd ../ # back to libs
+
 if [ $OVERWRITE -eq 1 ]; then
-    echo "Removing old libraries"
-    libs=("boost" "cairo" "curl" "FreeImage" "freetype" "glew" "glfw" "json" "libpng" "openssl" "pixman" "poco" "rtAudio" "tess2" "uriparser" "utf8" "videoInput" "zlib" "opencv" "ippicv" "assimp" "libxml2" "svgtiny" "README.md")
-    for lib in $libs; do
-        if [ -e $lib ]; then
-            rm -rf $lib
+    echo " "
+    echo " Overwrite - Removing prior libraries for [$PLATFORM]"
+    libs=("boost" "cairo" "curl" "FreeImage" "brotli" "fmod" "freetype" "glew" "glfw" "json" "libpng" "openssl" "pixman" "poco" "rtAudio" "tess2" "uriparser" "utf8" "videoInput" "zlib" "opencv" "ippicv" "assimp" "libxml2" "svgtiny" "fmt")
+    for ((i=0;i<${#libs[@]};++i)); do
+        if [ -e "${libs[i]}/lib/$PLATFORM" ]; then
+            echo "  Removing: [${libs[i]}/lib/$PLATFORM]"
+            rm -rf "${libs[i]}/lib/$PLATFORM"
         fi
+        if [ "$PLATFORM" == "msys2" ] || [ "$PLATFORM" == "vs" ]; then
+            if [ -e "${libs[i]}/bin" ]; then
+                echo "  Removing: [${libs[i]}/bin]"
+                rm -rf "${libs[i]}/bin"
+            fi
+        fi
+        # if [ -e "${libs[i]}/include" ]; then
+        #     echo "  Removing: [${libs[i]}/include]"
+        #     rm -rf "${libs[i]}/include"
+        # fi
     done
 fi
 
+echo " ------ "
 for PKG in $PKGS; do
-    echo "Uncompressing libraries ${PLATFORM}${ARCH} from $PKG"
+    echo " Uncompressing libraries [${PLATFORM}] from [$PKG]"
     if [ "$PLATFORM" == "msys2" ] || [ "$PLATFORM" == "vs" ]; then
-        unzip -qo ../scripts/dev/$PKG
-        rm ../scripts/dev/$PKG
+        unzip -qo download/$PKG
+        # rm -r download/$PKG
     else
-        tar xjf ../scripts/dev/$PKG
-        rm ../scripts/dev/$PKG
+        tar xjf download/$PKG
+        # rm -r download/$PKG
     fi
+    echo " Deployed libraries from [download/$PKG]to [/libs]"
 done
 
 if [ "$PLATFORM" == "osx" ]; then
@@ -266,13 +314,31 @@ else
     addons=("ofxOpenCv" "ofxOpenCv" "ofxAssimpModelLoader" "ofxSvg" "ofxSvg" "ofxPoco")
 fi
 
-for ((i=0;i<${#addonslibs[@]};++i)); do
-    if [ -e ${addonslibs[i]} ]; then
-        echo "Copying ${addonslibs[i]} to ${addons[i]}"
-        if [ $OVERWRITE -eq 1 ] && [ -e ../addons/${addons[i]}/libs/${addonslibs[i]} ]; then
-            echo "Removing old opencv libraries"
-            rm -rf ../addons/${addons[i]}/libs/${addonslibs[i]}
+echo "   ------ "
+if [ $OVERWRITE -eq 1 ]; then 
+    for ((i=0;i<${#addonslibs[@]};++i)); do
+        if [ -e ${addonslibs[i]} ] ; then
+            echo " Overwrite - addon: [${addons[i]} - ${addonslibs[i]}]"
+            if [ -e ../addons/${addons[i]}/libs/${addonslibs[i]}/lib/$PLATFORM ]; then
+                echo "   Remove binaries: [${addons[i]}/libs/${addonslibs[i]}/lib/$PLATFORM]"
+                rm -rf ../addons/${addons[i]}/libs/${addonslibs[i]}/lib/$PLATFORM
+            fi
+            if [ -e ../addons/${addons[i]}/libs/${addonslibs[i]}/bin ]; then
+                echo "   Remove binaries: [${addons[i]}/libs/${addonslibs[i]}/bin]"
+                rm -rf ../addons/${addons[i]}/libs/${addonslibs[i]}/bin
+            fi
+            # if [ -e ../addons/${addons[i]}/libs/${addonslibs[i]}/include ]; then
+            #     echo "   Remove include: [${addons[i]}/libs/include]"
+            #     rm -rf ../addons/${addons[i]}/libs/${addonslibs[i]}/include
+            # fi
         fi
+    done
+    echo "   ------ "
+fi
+
+for ((i=0;i<${#addonslibs[@]};++i)); do
+    if [ -e "${addonslibs[i]}" ]; then
+        echo "   Deploying [${addonslibs[i]}] to [../addons/${addons[i]}/libs]"
         mkdir -p ../addons/${addons[i]}/libs/${addonslibs[i]}
         if ! command -v rsync &> /dev/null
         then      
@@ -284,3 +350,5 @@ for ((i=0;i<${#addonslibs[@]};++i)); do
     fi
 done
 
+echo " ------ "
+echo " openFrameworks download_libs and install complete!"