From 38d73d303b4212d79ad1da66a78ecd03f1139691 Mon Sep 17 00:00:00 2001
From: Delyan Angelov <delian66@gmail.com>
Date: Tue, 26 Mar 2024 16:34:47 +0200
Subject: [PATCH] ci: retry all setup commands that need network access, till
 they succeed (so the CI jobs have less false positives) (#21103)

---
 .github/workflows/build_vinix_locally.sh      |  4 +-
 .github/workflows/c2v_ci.yml                  | 26 ++++-----
 .github/workflows/compile_v_with_vtcc.sh      |  2 +-
 .github/workflows/cross_ci.yml                | 10 ++--
 .github/workflows/freebsd_build_tcc.sh        |  2 +-
 .github/workflows/gen_vc_ci.yml               |  4 +-
 .github/workflows/gg_regressions_ci.yml       |  8 +--
 .github/workflows/linux_ci.yml                | 48 ++++++++---------
 .github/workflows/macos_ci.yml                |  8 +--
 .github/workflows/native_backend_tests_ci.yml |  2 +-
 .github/workflows/other_ci.yml                | 12 ++---
 .github/workflows/periodic_ci.yml             |  2 +-
 .github/workflows/retry.sh                    |  4 ++
 .github/workflows/sanitized_ci.yml            | 38 ++++++-------
 .github/workflows/sdl_ci.yml                  |  8 +--
 .github/workflows/toml_ci.yml                 | 14 ++---
 .../v_apps_and_modules_compile_ci.yml         | 54 +++++++++----------
 .github/workflows/vab_ci.yml                  |  4 +-
 .github/workflows/vinix_ci.yml                |  6 +--
 .github/workflows/websockets_ci.yml           |  2 +-
 20 files changed, 131 insertions(+), 127 deletions(-)
 create mode 100755 .github/workflows/retry.sh

diff --git a/.github/workflows/build_vinix_locally.sh b/.github/workflows/build_vinix_locally.sh
index 1c569956508819..09613ceac410a8 100755
--- a/.github/workflows/build_vinix_locally.sh
+++ b/.github/workflows/build_vinix_locally.sh
@@ -21,11 +21,11 @@ mkdir -p $BUILD
 
 cd $BUILD
 echo "Clone current Vinix"
-git clone https://github.com/vlang/vinix.git --depth=1
+.github/workflows/retry.sh git clone https://github.com/vlang/vinix.git --depth=1
 
 cd $BUILD
 echo "Clone current mlibc"
-git clone https://github.com/managarm/mlibc.git --depth=1
+.github/workflows/retry.sh git clone https://github.com/managarm/mlibc.git --depth=1
 
 cd $BUILD
 echo "Patch mlibc for Vinix"
diff --git a/.github/workflows/c2v_ci.yml b/.github/workflows/c2v_ci.yml
index 69ac2192d741d4..c153b6b4820625 100644
--- a/.github/workflows/c2v_ci.yml
+++ b/.github/workflows/c2v_ci.yml
@@ -37,19 +37,19 @@ jobs:
         run: make && ./v symlink -githubci
       - name: Install C2V
         run: |
-          v install --git https://github.com/vlang/c2v
+          .github/workflows/retry.sh v install --git https://github.com/vlang/c2v
           v -g ~/.vmodules/c2v/ || true
       - name: Install dependencies
         run: |
           if [ "${{ runner.os }}" == "Linux" ]; then
-            sudo apt update -y -qq
-            sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
+            .github/workflows/retry.sh sudo apt update -y -qq
+            .github/workflows/retry.sh sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
           else
-            brew install sdl2 sdl2_mixer sdl2_net libpng libsamplerate
+            .github/workflows/retry.sh brew install sdl2 sdl2_mixer sdl2_net libpng libsamplerate
           fi
       - name: Build original Chocolate Doom
         run: |
-          git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
+          .github/workflows/retry.sh git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
           cd ~/code/doom/chocolate-doom
           cmake -DCMAKE_BUILD_TYPE=Debug .
           make chocolate-doom
@@ -73,12 +73,12 @@ jobs:
         run: make && ./v symlink -githubci
       - name: Install C2V
         run: |
-          v install --git https://github.com/vlang/c2v
+          .github/workflows/retry.sh v install --git https://github.com/vlang/c2v
           v -g ~/.vmodules/c2v/ || true
       - name: Install dependencies
         run: |
-          sudo apt update -y -qq
-          sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
+          .github/workflows/retry.sh sudo apt update -y -qq
+          .github/workflows/retry.sh sudo apt install libsdl2-dev libsdl2-mixer-dev libsdl2-net-dev libpng-dev libsamplerate0-dev
           # c2v / DOOM dependencies
           # vgret dependencies
           # imagemagick              : convert, mogrify, import
@@ -86,19 +86,19 @@ jobs:
           # openimageio-tools        : idiff
           # libgl1-mesa-dri          : For headless rendering / software DRI driver (LIBGL_ALWAYS_SOFTWARE=true)
           # freeglut3-dev            : Fixes graphic apps compilation with tcc
-          sudo apt install imagemagick openimageio-tools freeglut3-dev libgl1-mesa-dri xvfb xsel xclip
+          .github/workflows/retry.sh sudo apt install imagemagick openimageio-tools freeglut3-dev libgl1-mesa-dri xvfb xsel xclip
       - name: Setup test tools
         run: |
           # Fetch the free ~4MB DOOM1.WAD from the link at https://doomwiki.org/wiki/DOOM1.WAD
-          wget https://distro.ibiblio.org/slitaz/sources/packages/d/doom1.wad -O ~/doom1.wad
+          .github/workflows/retry.sh wget https://distro.ibiblio.org/slitaz/sources/packages/d/doom1.wad -O ~/doom1.wad
           # Get imgur upload script
-          wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
+          .github/workflows/retry.sh wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
           chmod +x ./imgur.sh
           # Get regression images to test against
-          git clone https://github.com/Larpon/doom-regression-images
+          .github/workflows/retry.sh git clone https://github.com/Larpon/doom-regression-images
       - name: Build original Chocolate Doom
         run: |
-          git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
+          .github/workflows/retry.sh git clone --quiet --depth 1 https://github.com/vlang/doom ~/code/doom
           cd ~/code/doom/chocolate-doom
           cmake -DCMAKE_BUILD_TYPE=Debug .
           make chocolate-doom
diff --git a/.github/workflows/compile_v_with_vtcc.sh b/.github/workflows/compile_v_with_vtcc.sh
index d3476d79c6a435..c75724d2c05aa3 100755
--- a/.github/workflows/compile_v_with_vtcc.sh
+++ b/.github/workflows/compile_v_with_vtcc.sh
@@ -10,7 +10,7 @@ show "Prepare"
 rm -rf vtcc/
 
 show "Clone vtcc"
-git clone https://github.com/felipensp/vtcc --branch stable --quiet vtcc/
+.github/workflows/retry.sh git clone https://github.com/felipensp/vtcc --branch stable --quiet vtcc/
 du -s vtcc/
 ## TODO: just `./v vtcc`, later will cause V, to detect the compiler as tcc (which it is), and add `-fwrapv`, which causes the vtcc compiler to panic currently
 show "Compile vtcc"
diff --git a/.github/workflows/cross_ci.yml b/.github/workflows/cross_ci.yml
index 7e2b8d72570762..be6d7dde02b169 100644
--- a/.github/workflows/cross_ci.yml
+++ b/.github/workflows/cross_ci.yml
@@ -23,7 +23,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          brew install mingw-w64
+          .github/workflows/retry.sh brew install mingw-w64
           export LIBRARY_PATH="$LIBRARY_PATH:/usr/local/opt/openssl/lib/"
 
       - name: Build V
@@ -55,10 +55,10 @@ jobs:
       - name: Install dependencies
         run: |
           ## sudo dpkg --add-architecture i386
-          sudo apt update
-          sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev
-          sudo apt-get install --quiet -y mingw-w64 wine-stable winetricks
-          ## sudo apt-get install --quiet -y wine32
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y mingw-w64 wine-stable winetricks
+          ## .github/workflows/retry.sh sudo apt-get install --quiet -y wine32
 
       - name: Turn off the wine crash dialog
         run: winetricks nocrashdialog
diff --git a/.github/workflows/freebsd_build_tcc.sh b/.github/workflows/freebsd_build_tcc.sh
index 9bedc16e113214..e87cfc66908294 100755
--- a/.github/workflows/freebsd_build_tcc.sh
+++ b/.github/workflows/freebsd_build_tcc.sh
@@ -7,7 +7,7 @@ rm -rf thirdparty/tcc/
 
 pushd .
 
-git clone git://repo.or.cz/tinycc.git
+.github/workflows/retry.sh git clone git://repo.or.cz/tinycc.git
 cd tinycc
 
 export CC=clang
diff --git a/.github/workflows/gen_vc_ci.yml b/.github/workflows/gen_vc_ci.yml
index 7bf97f937ad010..d17ca2c30cb671 100644
--- a/.github/workflows/gen_vc_ci.yml
+++ b/.github/workflows/gen_vc_ci.yml
@@ -26,7 +26,7 @@ jobs:
           COMMIT_MSG=$(git log -1 --oneline --pretty='%s' HEAD)
 
           rm -rf vc
-          git clone --depth=1 \
+          .github/workflows/retry.sh git clone --depth=1 \
             https://vlang-bot:${{ secrets.VLANG_BOT_SECRET }}@$VREPO
 
           rm -rf vc/v.c vc/v_win.c
@@ -45,7 +45,7 @@ jobs:
           git -C vc commit -m "[v:master] $COMMIT_HASH - $COMMIT_MSG"
 
           # in case there are recent commits:
-          git -C vc pull --rebase origin master
+          .github/workflows/retry.sh git -C vc pull --rebase origin master
           # Note that failure below may happen, due to vlang/vc rejecting the push
           # from forked repos; that is not usually a problem.
           git -C vc push || true 
diff --git a/.github/workflows/gg_regressions_ci.yml b/.github/workflows/gg_regressions_ci.yml
index fe054d6f97809b..17e88d30ca3e2a 100644
--- a/.github/workflows/gg_regressions_ci.yml
+++ b/.github/workflows/gg_regressions_ci.yml
@@ -33,10 +33,10 @@ jobs:
           # libxcursor-dev libxi-dev : V gfx deps
           # libgl1-mesa-dri          : For headless rendering / software DRI driver (LIBGL_ALWAYS_SOFTWARE=true)
           # freeglut3-dev            : Fixes graphic apps compilation with tcc
-          sudo apt-get update
-          sudo apt-get install imagemagick openimageio-tools libgl1-mesa-dri xvfb libxcursor-dev libxi-dev freeglut3-dev xsel xclip
-          wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
-          git clone https://github.com/Larpon/gg-regression-images gg-regression-images
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install imagemagick openimageio-tools libgl1-mesa-dri xvfb libxcursor-dev libxi-dev freeglut3-dev xsel xclip
+          .github/workflows/retry.sh wget https://raw.githubusercontent.com/tremby/imgur.sh/c98345d/imgur.sh
+          .github/workflows/retry.sh git clone https://github.com/Larpon/gg-regression-images gg-regression-images
           chmod +x ./imgur.sh
 
       - name: Sample and compare
diff --git a/.github/workflows/linux_ci.yml b/.github/workflows/linux_ci.yml
index ba0dfb93af4447..a4f0b3c9d9d9ae 100644
--- a/.github/workflows/linux_ci.yml
+++ b/.github/workflows/linux_ci.yml
@@ -14,14 +14,14 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
           # The following is needed for examples/wkhtmltopdf.v
-          wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
-          sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
-          sudo apt-get install --quiet -y expect
-          sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo apt-get install --quiet -y expect
+          .github/workflows/retry.sh sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
       - name: Build v
         run: |
           echo $VFLAGS
@@ -94,14 +94,14 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install --quiet -y libgc-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libgc-dev
           # The following is needed for examples/wkhtmltopdf.v
-          wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
-          sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
-          sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb
+          .github/workflows/retry.sh sudo apt-get install --quiet -y xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo dpkg -i wkhtmltox_0.12.6-1.focal_amd64.deb
       - name: Build v
         run: |
           echo $VFLAGS
@@ -149,9 +149,9 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
       - name: Build V
         run: make -j4 && ./v -cc gcc -cg -cstrict -o v cmd/v
       - name: Valgrind v.c
@@ -169,9 +169,9 @@ jobs:
       # - name: Test v->js
       #   run: ./v -o hi.js examples/hello_v_js.v && node hi.js
       # - name: Build Vorum
-      #   run: git clone --depth 1 https://github.com/vlang/vorum && cd vorum && ../v . && cd ..
+      #   run: .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/vorum && cd vorum && ../v . && cd ..
       - name: Build vpm
-        run: ./v install markdown && git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
+        run: .github/workflows/retry.sh ./v install markdown && .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
       - name: Freestanding
         run: ./v -freestanding run vlib/os/bare/bare_example_linux.v
       - name: V self compilation
@@ -259,10 +259,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install --quiet -y clang
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y clang
       - name: Build V
         run: make -j4 && ./v -cc clang -cg -cstrict -o v cmd/v
       - name: Valgrind
@@ -363,7 +363,7 @@ jobs:
   #   - uses: actions/checkout@v4
   #   - name: Install dependencies
   #     run: |
-  #        sudo apt-get install --quiet -y musl musl-tools libssl-dev sqlite3 libsqlite3-dev valgrind
+  #        .github/workflows/retry.sh sudo apt-get install --quiet -y musl musl-tools libssl-dev sqlite3 libsqlite3-dev valgrind
   #   - name: Build v
   #     run: echo $VFLAGS && make -j4 && ./v -cg -o v cmd/v
   #   # - name: Test v binaries
diff --git a/.github/workflows/macos_ci.yml b/.github/workflows/macos_ci.yml
index 13bf9475e2d4df..298a7cb96b2214 100644
--- a/.github/workflows/macos_ci.yml
+++ b/.github/workflows/macos_ci.yml
@@ -16,7 +16,7 @@ jobs:
       - name: Install dependencies
         run: |
           echo "PKG_CONFIG_PATH is '$PKG_CONFIG_PATH'"
-          brew install libpq openssl mercurial
+          .github/workflows/retry.sh brew install libpq openssl mercurial
           export LIBRARY_PATH="$LIBRARY_PATH:/usr/local/opt/openssl/lib/"
           echo "LIBRARY_PATH is '$LIBRARY_PATH'"
       - name: Build V
@@ -30,7 +30,7 @@ jobs:
       - name: Test symlink
         run: ./v symlink
       - name: Build vpm
-        run: ./v install markdown && git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
+        run: .github/workflows/retry.sh ./v install markdown && .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/vpm && cd vpm && ../v . || ../v cmd/vpm && cd ..
       # - name: Set up pg database
       #   run: |
       #     pg_ctl -D /usr/local/var/postgres -l /usr/local/var/postgres/server.log start
@@ -68,14 +68,14 @@ jobs:
           ./v doctor
       - name: Test ved
         run: |
-          git clone --depth 1 https://github.com/vlang/ved
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/ved
           cd ved && ../v -o ved .
           ../v -autofree .
           ../v -prod .
           cd ..
       - name: Build V UI examples
         run: |
-          git clone --depth 1 https://github.com/vlang/ui
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/ui
           cd ui
           mkdir -p ~/.vmodules
           ln -s $(pwd) ~/.vmodules/ui
diff --git a/.github/workflows/native_backend_tests_ci.yml b/.github/workflows/native_backend_tests_ci.yml
index c886b9bea61fb4..3bb4a9a8174ac3 100644
--- a/.github/workflows/native_backend_tests_ci.yml
+++ b/.github/workflows/native_backend_tests_ci.yml
@@ -63,7 +63,7 @@ jobs:
       - name: Install linker
         if: ${{ startsWith(matrix.os, 'ubuntu')}}
         run: |
-          sudo apt-get install --quiet -y binutils
+          .github/workflows/retry.sh sudo apt-get install --quiet -y binutils
 
       - name: Build V with make.bat
         if: ${{ startsWith(matrix.os, 'windows') }}
diff --git a/.github/workflows/other_ci.yml b/.github/workflows/other_ci.yml
index 43cda690a9b19d..1ccfba10e750ba 100644
--- a/.github/workflows/other_ci.yml
+++ b/.github/workflows/other_ci.yml
@@ -54,7 +54,7 @@ jobs:
         run: make -j4
       - name: Clone & Build previous vmaster/v
         run: |
-          git clone --depth=1 https://github.com/vlang/v vmaster/
+          .github/workflows/retry.sh git clone --depth=1 https://github.com/vlang/v vmaster/
           (cd vmaster; make -j4)
       - name: V versions
         run: ./v version && ./vmaster/v version
@@ -78,10 +78,10 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libsodium-dev libssl-dev sqlite3 libsqlite3-dev postgresql libpq-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
-          sudo apt-get install --quiet -y g++-9 g++-11
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libsodium-dev libssl-dev sqlite3 libsqlite3-dev postgresql libpq-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo apt-get install --quiet -y g++-9 g++-11
 
       - name: Build v
         run: make
@@ -112,7 +112,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install zzuf
-        run: sudo apt install -qq zzuf
+        run: .github/workflows/retry.sh sudo apt install -qq zzuf
       - name: Build local v
         run: |
           make -j4
diff --git a/.github/workflows/periodic_ci.yml b/.github/workflows/periodic_ci.yml
index 84152fa9d0021e..140d17701ccf46 100644
--- a/.github/workflows/periodic_ci.yml
+++ b/.github/workflows/periodic_ci.yml
@@ -14,7 +14,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install dependencies 1
-        run: sudo apt-get install --quiet -y sqlite3 libsqlite3-dev
+        run: .github/workflows/retry.sh sudo apt-get install --quiet -y sqlite3 libsqlite3-dev
       - name: Build v
         run: make
       - name: Symlink V
diff --git a/.github/workflows/retry.sh b/.github/workflows/retry.sh
new file mode 100755
index 00000000000000..e478376caac34f
--- /dev/null
+++ b/.github/workflows/retry.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+while ! $@; do "command failed, retrying ..."; sleep 1; done
+
diff --git a/.github/workflows/sanitized_ci.yml b/.github/workflows/sanitized_ci.yml
index 1ed2b340a8949f..0511033058baed 100644
--- a/.github/workflows/sanitized_ci.yml
+++ b/.github/workflows/sanitized_ci.yml
@@ -86,10 +86,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install clang
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install clang
       - name: Build V
         run: make && ./v -cg -cstrict -o v cmd/v
       - name: Ensure code is well formatted
@@ -111,9 +111,9 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
       - name: Build V
         run: make && ./v -cg -cstrict -o v cmd/v
       - name: Ensure code is well formatted
@@ -135,10 +135,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install clang
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install clang
       - name: Build V
         run: make && ./v -cg -cstrict -o v cmd/v
       - name: Ensure code is well formatted
@@ -193,10 +193,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install clang
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install clang
       - name: Build V
         run: make && ./v -cg -cstrict -o v cmd/v
       - name: Ensure code is well formatted
@@ -222,10 +222,10 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt update
-          sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
-          sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
-          sudo apt-get install clang
+          .github/workflows/retry.sh sudo apt update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y postgresql libpq-dev libssl-dev sqlite3 libsqlite3-dev valgrind
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libfreetype6-dev  libxi-dev libxcursor-dev libgl-dev
+          .github/workflows/retry.sh sudo apt-get install clang
       - name: Build V
         run: make && ./v -cc clang -cg -cstrict -o v cmd/v
       - name: Ensure code is well formatted
diff --git a/.github/workflows/sdl_ci.yml b/.github/workflows/sdl_ci.yml
index 26bd365cb2f098..522061d22f7f28 100644
--- a/.github/workflows/sdl_ci.yml
+++ b/.github/workflows/sdl_ci.yml
@@ -28,16 +28,16 @@ jobs:
 
       - name: Clone sdl into .vmodules
         run: |
-          git clone --depth 1 https://github.com/vlang/sdl
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/sdl
           cd sdl
           mkdir -p ~/.vmodules
           ln -s $(pwd) ~/.vmodules/sdl
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libsdl2-dev libsdl2-ttf-dev
-          sudo apt-get install --quiet -y libsdl2-mixer-dev libsdl2-image-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libsdl2-dev libsdl2-ttf-dev
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libsdl2-mixer-dev libsdl2-image-dev
 
       - name: Run tests
         run: ./v test sdl
diff --git a/.github/workflows/toml_ci.yml b/.github/workflows/toml_ci.yml
index 506754eabc6cff..3b928f3ab07778 100644
--- a/.github/workflows/toml_ci.yml
+++ b/.github/workflows/toml_ci.yml
@@ -27,8 +27,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y jq libgc-dev
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y jq libgc-dev
       - name: Build V
         run: make -j2 && ./v -cc gcc -o v cmd/v
 
@@ -41,15 +41,15 @@ jobs:
         # Tests found at https://github.com/BurntSushi/toml-test
       - name: Clone BurntSushi/toml-test
         run: |
-          git clone -n https://github.com/BurntSushi/toml-test.git $TOML_BS_TESTS_PATH
-          git -C $TOML_BS_TESTS_PATH checkout $TOML_BS_TESTS_PINNED_COMMIT
+          .github/workflows/retry.sh git clone -n https://github.com/BurntSushi/toml-test.git $TOML_BS_TESTS_PATH
+          .github/workflows/retry.sh git -C $TOML_BS_TESTS_PATH checkout $TOML_BS_TESTS_PINNED_COMMIT
 
       - name: Run BurntSushi TOML tests
         run: ./v vlib/toml/tests/burntsushi_toml_test.v
 
         # Tests found at gist
       - name: Get large_toml_file_test.toml
-        run: wget https://gist.githubusercontent.com/Larpon/89b0e3d94c6903851ff15559e5df7a05/raw/62a1f87a4e37bf157f2e0bfb32d85d840c98e422/large_toml_file_test.toml -O vlib/toml/tests/testdata/large_toml_file_test.toml
+        run: .github/workflows/retry.sh wget https://gist.githubusercontent.com/Larpon/89b0e3d94c6903851ff15559e5df7a05/raw/62a1f87a4e37bf157f2e0bfb32d85d840c98e422/large_toml_file_test.toml -O vlib/toml/tests/testdata/large_toml_file_test.toml
 
       - name: Run large TOML file tests
         run: ./v vlib/toml/tests/large_toml_file_test.v
@@ -57,7 +57,7 @@ jobs:
         # Tests found at https://github.com/iarna/toml-spec-tests
       - name: Clone iarna/toml-spec-tests
         run: |
-          git clone -n https://github.com/iarna/toml-spec-tests.git $TOML_IARNA_TESTS_PATH
+          .github/workflows/retry.sh git clone -n https://github.com/iarna/toml-spec-tests.git $TOML_IARNA_TESTS_PATH
           git -C $TOML_IARNA_TESTS_PATH checkout $TOML_IARNA_TESTS_PINNED_COMMIT
 
       - name: Run iarna TOML tests
@@ -66,7 +66,7 @@ jobs:
         # Tests found at https://github.com/alexcrichton/toml-rs
       - name: Clone alexcrichton/toml-rs
         run: |
-          git clone -n https://github.com/alexcrichton/toml-rs.git $TOML_AC_TESTS_PATH
+          .github/workflows/retry.sh git clone -n https://github.com/alexcrichton/toml-rs.git $TOML_AC_TESTS_PATH
           git -C $TOML_AC_TESTS_PATH checkout $TOML_AC_TESTS_PINNED_COMMIT
 
       - name: Run alexcrichton TOML tests
diff --git a/.github/workflows/v_apps_and_modules_compile_ci.yml b/.github/workflows/v_apps_and_modules_compile_ci.yml
index 911de8595c9b5d..234fb344994851 100644
--- a/.github/workflows/v_apps_and_modules_compile_ci.yml
+++ b/.github/workflows/v_apps_and_modules_compile_ci.yml
@@ -23,9 +23,9 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libgc-dev libsodium-dev libssl-dev sqlite3 libsqlite3-dev libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
-          sudo apt-get install --quiet -y --no-install-recommends sassc libgit2-dev ## needed by gitly
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libgc-dev libsodium-dev libssl-dev sqlite3 libsqlite3-dev libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo apt-get install --quiet -y --no-install-recommends sassc libgit2-dev ## needed by gitly
 
       - name: Test vtcc
         run: .github/workflows/compile_v_with_vtcc.sh
@@ -33,7 +33,7 @@ jobs:
       - name: Test vsql compilation and examples
         run: |
           echo "Install vsql"
-          v install elliotchance.vsql ; cd ~/.vmodules/elliotchance/vsql
+          .github/workflows/retry.sh v install elliotchance.vsql ; cd ~/.vmodules/elliotchance/vsql
           echo "Generate vsql/grammar.v"
           make vsql/grammar.v
           echo "Compile vsql"
@@ -46,7 +46,7 @@ jobs:
       - name: Test discord.v
         run: |
           echo "Clone https://github.com/DarpHome/discord.v/"
-          git clone https://github.com/DarpHome/discord.v/ discord && cd discord
+          .github/workflows/retry.sh git clone https://github.com/DarpHome/discord.v/ discord && cd discord
           echo "Checkout last known good commit"
           git checkout 65448b687ae759e385c127c8739b97b6ac62d3e3
           echo "Execute Tests"
@@ -55,7 +55,7 @@ jobs:
       - name: Build vlang/vab
         run: |
           echo "Install VAB"
-          v install vab
+          .github/workflows/retry.sh v install vab
           echo "Build vab"
           v ~/.vmodules/vab
           echo "Build vab with -gc boehm -skip-unused"
@@ -63,7 +63,7 @@ jobs:
 
       - name: Build vlang/ved
         run: |
-          git clone --depth 1 https://github.com/vlang/ved
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/ved
           cd ved && ../v -o ved .
           ../v -autofree .
           ../v -prod .
@@ -71,32 +71,32 @@ jobs:
 
       - name: Build vlang/pdf
         run: |
-          v install pdf
+          .github/workflows/retry.sh v install pdf
           echo "PDF examples should compile"
           v should-compile-all ~/.vmodules/pdf/examples
 
       - name: Build vlang/libsodium
         run: |
           echo "Install the libsodium wrapper"
-          v install libsodium
+          .github/workflows/retry.sh v install libsodium
           echo "Test libsodium"
           VJOBS=1 v test ~/.vmodules/libsodium
 
       - name: Build vlang/coreutils
         run: |
           echo "Clone Coreutils"
-          git clone --depth 1 https://github.com/vlang/coreutils /tmp/coreutils
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/coreutils /tmp/coreutils
           echo "Build Coreutils"
           cd /tmp/coreutils; make
 
       - name: Build vlang/gitly
         run: |
           echo "Install markdown"
-          v install markdown
+          .github/workflows/retry.sh v install markdown
           echo "Install pcre"
-          v install pcre
+          .github/workflows/retry.sh v install pcre
           echo "Clone Gitly"
-          git clone https://github.com/vlang/gitly /tmp/gitly
+          .github/workflows/retry.sh git clone https://github.com/vlang/gitly /tmp/gitly
           echo "Build Gitly"
           v -cc gcc /tmp/gitly
           ## echo "Build Gitly with -autofree"
@@ -110,7 +110,7 @@ jobs:
       - name: Build V Language Server (v-analyzer) vlang/v-analyzer
         run: |
           echo "Clone v-analyzer"
-          git clone --depth=1 --filter=blob:none --recursive --shallow-submodules https://github.com/vlang/v-analyzer /tmp/v-analyzer
+          .github/workflows/retry.sh git clone --depth=1 --filter=blob:none --recursive --shallow-submodules https://github.com/vlang/v-analyzer /tmp/v-analyzer
           cd /tmp/v-analyzer
           echo "Installing dependencies"
           v install
@@ -122,7 +122,7 @@ jobs:
       - name: Build vlang/go2v
         run: |
           echo "Clone Go2V"
-          git clone --depth=1 https://github.com/vlang/go2v /tmp/go2v/
+          .github/workflows/retry.sh git clone --depth=1 https://github.com/vlang/go2v /tmp/go2v/
           echo "Build Go2V"
           v /tmp/go2v/
           echo "Run Go2V tests"
@@ -131,23 +131,23 @@ jobs:
       - name: Install UI through VPM and make sure its examples compile
         run: |
           echo "Official VPM modules should be installable"
-          v install ui
+          .github/workflows/retry.sh v install ui
           echo "Examples of UI should compile"
           v ~/.vmodules/ui/examples/build_examples.vsh
 
       - name: Build vlang/adventofcode
         run: |
           echo "Clone the AdventOfCode repo"
-          git clone --depth 1 https://github.com/vlang/adventofcode  /tmp/adventofcode
+          .github/workflows/retry.sh git clone --depth 1 https://github.com/vlang/adventofcode  /tmp/adventofcode
           echo "Install dependencies"
-          v install pcre
+          .github/workflows/retry.sh v install pcre
           echo "Execute Tests"
           cd /tmp/adventofcode && v run verify.v
 
       - name: Build vlang/msgpack
         run: |
           echo "Install msgpack"
-          v install msgpack
+          .github/workflows/retry.sh v install msgpack
           echo "Build msgpack"
           v -shared ~/.vmodules/msgpack/
           echo "Run msgpack tests"
@@ -159,7 +159,7 @@ jobs:
 #      - name: Build VEX
 #        run: |
 #          echo "Install Vex"
-#          v install nedpals.vex
+#          .github/workflows/retry.sh v install nedpals.vex
 #          echo "Compile all of the Vex examples"
 #          v should-compile-all ~/.vmodules/nedpals/vex/examples
 #          echo "Compile the simple Vex example with -skip-unused"
@@ -184,15 +184,15 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install --quiet -y libgc-dev   libsodium-dev libssl-dev sqlite3 libsqlite3-dev libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
-          sudo apt-get install --quiet -y --no-install-recommends gfortran liblapacke-dev libopenblas-dev ## for vsl/vtl
-          sudo apt-get install --quiet -y --no-install-recommends libhdf5-cpp-103 libhdf5-dev libhdf5-mpi-dev hdf5-tools libopenmpi-dev opencl-headers liblapacke-dev libopenblas-dev ## needed by VSL
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install --quiet -y libgc-dev   libsodium-dev libssl-dev sqlite3 libsqlite3-dev libfreetype6-dev libxi-dev libxcursor-dev libgl-dev xfonts-75dpi xfonts-base
+          .github/workflows/retry.sh sudo apt-get install --quiet -y --no-install-recommends gfortran liblapacke-dev libopenblas-dev ## for vsl/vtl
+          .github/workflows/retry.sh sudo apt-get install --quiet -y --no-install-recommends libhdf5-cpp-103 libhdf5-dev libhdf5-mpi-dev hdf5-tools libopenmpi-dev opencl-headers liblapacke-dev libopenblas-dev ## needed by VSL
 
       - name: Build vlang/vsl
         run: |
           echo "Installing dependencies"
-          sudo apt-get install --quiet -y --no-install-recommends \
+          .github/workflows/retry.sh sudo apt-get install --quiet -y --no-install-recommends \
             gfortran \
             libxi-dev \
             libxcursor-dev \
@@ -204,7 +204,7 @@ jobs:
             libopenmpi-dev \
             opencl-headers
           echo "Install VSL"
-          v install vsl
+          .github/workflows/retry.sh v install vsl
           echo "Execute Tests using Pure V Backend"
           ~/.vmodules/vsl/bin/test
           echo "Execute Tests using Pure V Backend with Pure V Math"
@@ -217,7 +217,7 @@ jobs:
       - name: Build vlang/vtl
         run: |
           echo "Install VTL"
-          v install vtl
+          .github/workflows/retry.sh v install vtl
           echo "Install dependencies"
           echo "Execute Tests using Pure V Backend"
           ~/.vmodules/vtl/bin/test
diff --git a/.github/workflows/vab_ci.yml b/.github/workflows/vab_ci.yml
index a18bce7947f5a5..60d1bf972dfd95 100644
--- a/.github/workflows/vab_ci.yml
+++ b/.github/workflows/vab_ci.yml
@@ -28,7 +28,7 @@ jobs:
 
       - name: Install vab
         run: |
-          v install vab
+          .github/workflows/retry.sh v install vab
           v -g ~/.vmodules/vab
           sudo ln -s ~/.vmodules/vab/vab /usr/local/bin/vab
 
@@ -64,7 +64,7 @@ jobs:
 
       - name: Install vab
         run: |
-          v install vab
+          .github/workflows/retry.sh v install vab
           v -g ~/.vmodules/vab
           sudo ln -s ~/.vmodules/vab/vab /usr/local/bin/vab
 
diff --git a/.github/workflows/vinix_ci.yml b/.github/workflows/vinix_ci.yml
index 2b2ee756fd6c74..68950f97fbc103 100644
--- a/.github/workflows/vinix_ci.yml
+++ b/.github/workflows/vinix_ci.yml
@@ -25,14 +25,14 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install build-essential meson -y
+          .github/workflows/retry.sh sudo apt-get update
+          .github/workflows/retry.sh sudo apt-get install build-essential meson -y
 
       - name: Build V
         run: make
 
       - name: Clone current Vinix
-        run: git clone https://github.com/vlang/vinix.git
+        run: .github/workflows/retry.sh git clone https://github.com/vlang/vinix.git
 
       - name: Download Vinix kernel dependencies
         run: cd vinix/kernel && ./get-deps
diff --git a/.github/workflows/websockets_ci.yml b/.github/workflows/websockets_ci.yml
index c39f7b6a4b77a8..010a07b8648a35 100644
--- a/.github/workflows/websockets_ci.yml
+++ b/.github/workflows/websockets_ci.yml
@@ -18,7 +18,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install dependencies
-        run: sudo apt-get install --quiet -y libssl-dev
+        run: .github/workflows/retry.sh sudo apt-get install --quiet -y libssl-dev
       - name: Build v
         run: |
           echo $VFLAGS