ci: retry all setup commands that need network access, till they succ…

…eed (so the CI jobs have less false positives) (#21103)

.github/workflows/build_vinix_locally.sh

@@ -21,11 +21,11 @@ mkdir -p $BUILD
 
 cd $BUILD
 echo "Clone current Vinix"
-git clone https://github.com/vlang/vinix.git --depth=1
+.github/workflows/retry.sh git clone https://github.com/vlang/vinix.git --depth=1
 
 cd $BUILD
 echo "Clone current mlibc"
-git clone https://github.com/managarm/mlibc.git --depth=1
+.github/workflows/retry.sh git clone https://github.com/managarm/mlibc.git --depth=1
 
 cd $BUILD
 echo "Patch mlibc for Vinix"

.github/workflows/c2v_ci.yml

@@ -37,19 +37,19 @@ jobs:
         run: make && ./v symlink -githubci
       - name: Install C2V
         run: |
-          v install --git https://github.com/vlang/c2v
+          .github/workflows/retry.sh v install --git https://github.com/vlang/c2v
           v -g ~/.vmodules/c2v/ || true
       - name: Install dependencies
         run: |
           if [ "${{ runner.os }}" == "Linux" ]; then
-            sudo apt update -y -qq
-            sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
+            .github/workflows/retry.sh sudo apt update -y -qq
+            .github/workflows/retry.sh sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
           else
-            brew install sdl2 sdl2_mixer sdl2_net libpng libsamplerate
+            .github/workflows/retry.sh brew install sdl2 sdl2_mixer sdl2_net libpng libsamplerate
           fi
       - name: Build original Chocolate Doom
         run: |
-          git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
+          .github/workflows/retry.sh git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
           cd ~/code/doom/chocolate-doom
           cmake -DCMAKE_BUILD_TYPE=Debug .
           make chocolate-doom
@@ -73,32 +73,32 @@ jobs:
         run: make && ./v symlink -githubci
       - name: Install C2V
         run: |
-          v install --git https://github.com/vlang/c2v
+          .github/workflows/retry.sh v install --git https://github.com/vlang/c2v
           v -g ~/.vmodules/c2v/ || true
       - name: Install dependencies
         run: |
-          sudo apt update -y -qq
-          sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
+          .github/workflows/retry.sh sudo apt update -y -qq
+          .github/workflows/retry.sh sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
           # c2v / DOOM dependencies
           # vgret dependencies
           # imagemagick              : convert, mogrify, import
           # xvfb                     : For starting X11 Virtual FrameBuffers
           # openimageio-tools        : idiff
           # libgl1-mesa-dri          : For headless rendering / software DRI driver (LIBGL_ALWAYS_SOFTWARE=true)
           # freeglut3-dev            : Fixes graphic apps compilation with tcc
-          sudo apt install imagemagick openimageio-tools freeglut3-dev libgl1-mesa-dri xvfb xsel xclip
+          .github/workflows/retry.sh sudo apt install imagemagick openimageio-tools freeglut3-dev libgl1-mesa-dri xvfb xsel xclip
       - name: Setup test tools
         run: |
           # Fetch the free ~4MB DOOM1.WAD from the link at https://doomwiki.org/wiki/DOOM1.WAD
-          wget https://distro.ibiblio.org/slitaz/sources/packages/d/doom1.wad -O ~/doom1.wad
+          .github/workflows/retry.sh wget https://distro.ibiblio.org/slitaz/sources/packages/d/doom1.wad -O ~/doom1.wad
           # Get imgur upload script
-          wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
+          .github/workflows/retry.sh wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
           chmod +x ./imgur.sh
           # Get regression images to test against
-          git clone https://github.com/Larpon/doom-regression-images
+          .github/workflows/retry.sh git clone https://github.com/Larpon/doom-regression-images
       - name: Build original Chocolate Doom
         run: |
-          git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
+          .github/workflows/retry.sh git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
           cd ~/code/doom/chocolate-doom
           cmake -DCMAKE_BUILD_TYPE=Debug .
           make chocolate-doom

.github/workflows/compile_v_with_vtcc.sh

@@ -10,7 +10,7 @@ show "Prepare"
 rm -rf vtcc/
 
 show "Clone vtcc"
-git clone https://github.com/felipensp/vtcc --branch stable --quiet vtcc/
+.github/workflows/retry.sh git clone https://github.com/felipensp/vtcc --branch stable --quiet vtcc/
 du -s vtcc/
 ## TODO: just `./v vtcc`, later will cause V, to detect the compiler as tcc (which it is), and add `-fwrapv`, which causes the vtcc compiler to panic currently
 show "Compile vtcc"

.github/workflows/cross_ci.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          brew install mingw-w64
+          .github/workflows/retry.sh brew install mingw-w64
           export LIBRARY_PATH="$LIBRARY_PATH:/usr/local/opt/openssl/lib/"
 
       - name: Build V
@@ -55,10 +55,10 @@ jobs:
       - name: Install dependencies
         run: |
           ## sudo dpkg --add-architecture i386
-          sudo apt update
-          sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev
-          sudo apt-get install --quiet -y mingw-w64 wine-stable winetricks
-          ## sudo apt-get install --quiet -y wine32
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y mingw-w64 wine-stable winetricks
+          ## .github/workflows/retry.sh sudo apt-get install --quiet -y wine32
 
       - name: Turn off the wine crash dialog
         run: winetricks nocrashdialog

.github/workflows/freebsd_build_tcc.sh

@@ -7,7 +7,7 @@ rm -rf thirdparty/tcc/
 
 pushd .
 
-git clone git://repo.or.cz/tinycc.git
+.github/workflows/retry.sh git clone git://repo.or.cz/tinycc.git
 cd tinycc
 
 export CC=clang

.github/workflows/gen_vc_ci.yml

@@ -26,7 +26,7 @@ jobs:
           COMMIT_MSG=$(git log -1 --oneline --pretty='%s' HEAD)
 
           rm -rf vc
-          git clone --depth=1 \
+          .github/workflows/retry.sh git clone --depth=1 \
             https://vlang-bot:${{ secrets.VLANG_BOT_SECRET }}@$VREPO
 
           rm -rf vc/v.c vc/v_win.c
@@ -45,7 +45,7 @@ jobs:
           git -C vc commit -m "[v:master] $COMMIT_HASH - $COMMIT_MSG"
 
           # in case there are recent commits:
-          git -C vc pull --rebase origin master
+          .github/workflows/retry.sh git -C vc pull --rebase origin master
           # Note that failure below may happen, due to vlang/vc rejecting the push
           # from forked repos; that is not usually a problem.
           git -C vc push || true 

.github/workflows/gg_regressions_ci.yml

@@ -33,10 +33,10 @@ jobs:
           # libxcursor-dev libxi-dev : V gfx deps
           # libgl1-mesa-dri          : For headless rendering / software DRI driver (LIBGL_ALWAYS_SOFTWARE=true)
           # freeglut3-dev            : Fixes graphic apps compilation with tcc
-          sudo apt-get update
-          sudo apt-get install imagemagick openimageio-tools libgl1-mesa-dri xvfb libxcursor-dev libxi-dev freeglut3-dev xsel xclip
-          wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
-          git clone https://github.com/Larpon/gg-regression-images gg-regression-images
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install imagemagick openimageio-tools libgl1-mesa-dri xvfb libxcursor-dev libxi-dev freeglut3-dev xsel xclip
+          .github/workflows/retry.sh wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
+          .github/workflows/retry.sh git clone https://github.com/Larpon/gg-regression-images gg-regression-images
           chmod +x ./imgur.sh
 
       - name: Sample and compare

.github/workflows/linux_ci.yml

@@ -14,14 +14,14 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
           # The following is needed for examples/wkhtmltopdf.v
-          wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
-          sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
-          sudo apt-get install --quiet -y expect
-          sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo apt-get install --quiet -y expect
+          .github/workflows/retry.sh sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
       - name: Build v
         run: |
           echo $VFLAGS
@@ -94,14 +94,14 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install --quiet -y libgc-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libgc-dev
           # The following is needed for examples/wkhtmltopdf.v
-          wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
-          sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
-          sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
       - name: Build v
         run: |
           echo $VFLAGS
@@ -149,9 +149,9 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
       - name: Build V
         run: make -j4 && ./v -cc gcc -cg -cstrict -o v cmd/v
       - name: Valgrind v.c
@@ -169,9 +169,9 @@ jobs:
       # - name: Test v->js
       #   run: ./v -o hi.js examples/hello_v_js.v && node hi.js
       # - name: Build Vorum
-      #   run: git clone --depth 1 https://github.com/vlang/vorum && cd vorum && ../v . && cd ..
+      #   run: .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/vorum && cd vorum && ../v . && cd ..
       - name: Build vpm
-        run: ./v install markdown && git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
+        run: .github/workflows/retry.sh ./v install markdown && .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
       - name: Freestanding
         run: ./v -freestanding run vlib/os/bare/bare_example_linux.v
       - name: V self compilation
@@ -259,10 +259,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install --quiet -y clang
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y clang
       - name: Build V
         run: make -j4 && ./v -cc clang -cg -cstrict -o v cmd/v
       - name: Valgrind
@@ -363,7 +363,7 @@ jobs:
   #   - uses: actions/checkout@v4
   #   - name: Install dependencies
   #     run: |
-  #        sudo apt-get install --quiet -y musl musl-tools libssl-dev sqlite3 libsqlite3-dev valgrind
+  #        .github/workflows/retry.sh sudo apt-get install --quiet -y musl musl-tools libssl-dev sqlite3 libsqlite3-dev valgrind
   #   - name: Build v
   #     run: echo $VFLAGS && make -j4 && ./v -cg -o v cmd/v
   #   # - name: Test v binaries

.github/workflows/macos_ci.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Install dependencies
         run: |
           echo "PKG_CONFIG_PATH is '$PKG_CONFIG_PATH'"
-          brew install libpq openssl mercurial
+          .github/workflows/retry.sh brew install libpq openssl mercurial
           export LIBRARY_PATH="$LIBRARY_PATH:/usr/local/opt/openssl/lib/"
           echo "LIBRARY_PATH is '$LIBRARY_PATH'"
       - name: Build V
@@ -30,7 +30,7 @@ jobs:
       - name: Test symlink
         run: ./v symlink
       - name: Build vpm
-        run: ./v install markdown && git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
+        run: .github/workflows/retry.sh ./v install markdown && .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
       # - name: Set up pg database
       #   run: |
       #     pg_ctl -D /usr/local/var/postgres -l /usr/local/var/postgres/server.log start
@@ -68,14 +68,14 @@ jobs:
           ./v doctor
       - name: Test ved
         run: |
-          git clone --depth 1 https://github.com/vlang/ved
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/ved
           cd ved && ../v -o ved .
           ../v -autofree .
           ../v -prod .
           cd ..
       - name: Build V UI examples
         run: |
-          git clone --depth 1 https://github.com/vlang/ui
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/ui
           cd ui
           mkdir -p ~/.vmodules
           ln -s $(pwd) ~/.vmodules/ui

.github/workflows/native_backend_tests_ci.yml

@@ -63,7 +63,7 @@ jobs:
       - name: Install linker
         if: ${{ startsWith(matrix.os, 'ubuntu')}}
         run: |
-          sudo apt-get install --quiet -y binutils
+          .github/workflows/retry.sh sudo apt-get install --quiet -y binutils
 
       - name: Build V with make.bat
         if: ${{ startsWith(matrix.os, 'windows') }}

.github/workflows/other_ci.yml

@@ -54,7 +54,7 @@ jobs:
         run: make -j4
       - name: Clone & Build previous vmaster/v
         run: |
-          git clone --depth=1 https://github.com/vlang/v vmaster/
+          .github/workflows/retry.sh git clone --depth=1 https://github.com/vlang/v vmaster/
           (cd vmaster; make -j4)
       - name: V versions
         run: ./v version && ./vmaster/v version
@@ -78,10 +78,10 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libsodium-dev libssl-dev sqlite3 libsqlite3-dev postgresql libpq-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
-          sudo apt-get install --quiet -y g++-9 g++-11
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libsodium-dev libssl-dev sqlite3 libsqlite3-dev postgresql libpq-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo apt-get install --quiet -y g++-9 g++-11
 
       - name: Build v
         run: make
@@ -112,7 +112,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install zzuf
-        run: sudo apt install -qq zzuf
+        run: .github/workflows/retry.sh sudo apt install -qq zzuf
       - name: Build local v
         run: |
           make -j4

.github/workflows/periodic_ci.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install dependencies 1
-        run: sudo apt-get install --quiet -y sqlite3 libsqlite3-dev
+        run: .github/workflows/retry.sh sudo apt-get install --quiet -y sqlite3 libsqlite3-dev
       - name: Build v
         run: make
       - name: Symlink V

.github/workflows/retry.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+while ! $@; do "command failed, retrying ..."; sleep 1; done
+