master cherry-picks to 6.2

.github/workflows/ARM_BUILD_AND_PACKAGE_REUSABLE.yml

@@ -206,8 +206,12 @@ jobs:
      - name: install qemu
        uses: docker/setup-qemu-action@v2
 
+     # https://github.com/docker/buildx/issues/1509
      - name: set up buildx
        uses: docker/setup-buildx-action@v2
+       with:
+         install: true
+         version: v0.9.1
 
      - run: gem install fpm -v ${{env.fpmversion}}
      - name: display structure

.github/workflows/DOCKER_REUSABLE.yml

@@ -83,7 +83,12 @@ jobs:
           echo "VERSION=$realversion" >> $GITHUB_OUTPUT
 
       - uses: docker/setup-qemu-action@v2
+
+      # https://github.com/docker/buildx/issues/1509
       - uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+          version: v0.9.1
 
       - name: generate docker file
         run: |
@@ -92,13 +97,11 @@ jobs:
 
       - name: Log in to DockerHub
         uses: docker/login-action@v2
-        if: steps.iamafork.outputs.IAMAFORK == 'false'
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: build and publish docker
-        if: steps.iamafork.outputs.IAMAFORK == 'false'
         run: |
           source .venv/bin/activate
           invoke dockerbuild -a ${{inputs.arch}} -d envs/dockers/Dockerfile.${{ matrix.package }} -t redisfab/${{matrix.package}}:${{steps.get_version.outputs.VERSION}}-${{inputs.arch}} -r . -b
@@ -110,6 +113,7 @@ jobs:
           invoke dockerbuild -a ${{inputs.arch}} -d envs/dockers/Dockerfile.${{ matrix.package }} -t redisfab/${{matrix.package}}:${{steps.get_version.outputs.VERSION}}-${{inputs.arch}} -r .
 
       - name: test docker build
+        if: ${{ inputs.arch != 'arm64' }}
         run: |
           source .venv/bin/activate
           invoke test-ci-dockers -d ${{matrix.package}} -a ${{inputs.arch}} -v ${{steps.get_version.outputs.VERSION}}-${{inputs.arch}}

.github/workflows/nightly-edge-docker.yml

@@ -104,7 +104,7 @@ jobs:
 
  docker-push:
    needs: [x86_64-docker, arm64-docker]
-   name: Push ${{matrix.package}} manifest
+   name: Push ${{matrix.package}}
    strategy:
      matrix:
        package: ['redis-stack-server', 'redis-stack']
@@ -127,16 +127,13 @@ jobs:
 
      - name: pull, retag, push arm64
        run: |
-         docker pull redisfab/${{matrix.package}}:${{env.version}}-arm64
+         docker pull redisfab/${{matrix.package}}:${{env.version}}-arm64 --platform arm64
          docker tag redisfab/${{matrix.package}}:${{env.version}}-arm64 redis/${{matrix.package}}:${{env.version}}-arm64
          docker push redis/${{matrix.package}}:${{env.version}}-arm64
 
-     - name: create the manifest
+     - name: create and push the manifest
        run: |
          docker manifest create redis/${{ matrix.package }}:${{ env.version }} \
            redis/${{ matrix.package }}:${{ env.version }}-x86_64 \
            redis/${{ matrix.package }}:${{ env.version }}-arm64
-
-     - name: push the manifest
-       run: |
          docker manifest push redis/${{matrix.package}}:${{ env.version }}

stack/components/get.py

@@ -0,0 +1,17 @@
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+
+def get_stream_and_store(url: str, destfile: str):
+    """Fetch a URL from a known location, and right to file,
+    relying on retries.
+    """
+    session = requests.Session()
+    adapter = HTTPAdapter(max_retries=Retry(total=4, backoff_factor=1))
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    r = requests.get(url, stream=True)
+    if r.status_code > 204:
+        logger.error(f"{url} could not be retrieved")
+        raise requests.HTTPError
+    open(destfile, "wb").write(r.content)

stack/components/modules.py

@@ -9,11 +9,11 @@
 import zipfile
 from typing import Union
 
-import requests
 from loguru import logger
 
 from ..config import Config
 from ..paths import Paths
+from .get import get_stream_and_store
 
 
 class Modules(object):
@@ -141,11 +141,7 @@ def _fetch_and_unzip(self, url: str, destfile: str, custom_dest: str = None):
         if os.path.isfile(destfile):
             return
 
-        r = requests.get(url, stream=True)
-        if r.status_code > 204:
-            logger.error(f"{url} could not be retrieved")
-            raise requests.HTTPError
-        open(destfile, "wb").write(r.content)
+        get_stream_and_store(url, destfile)
 
         if custom_dest is None:
             dest = self.__PATHS__.DESTDIR

stack/components/nodejs.py

@@ -8,11 +8,11 @@
 import tarfile
 from typing import Union
 
-import requests
 from loguru import logger
 
 from ..config import Config
 from ..paths import Paths
+from .get import get_stream_and_store
 
 
 class NodeJS(object):
@@ -51,11 +51,7 @@ def _fetch_and_unzip(self, url: str, destfile: str):
         if os.path.isfile(destfile):
             return
 
-        r = requests.get(url, stream=True)
-        if r.status_code > 204:
-            logger.error(f"{url} could not be retrieved")
-            raise requests.HTTPError
-        open(destfile, "wb").write(r.content)
+        get_stream_and_store(url, destfile)
 
         logger.debug(f"Unzipping {destfile} and storing in {self.__PATHS__.DESTDIR}")
         with tarfile.open(destfile) as tar:

stack/components/redis.py

@@ -8,11 +8,11 @@
 import tarfile
 from typing import Union
 
-import requests
 from loguru import logger
 
 from ..config import Config
 from ..paths import Paths
+from .get import get_stream_and_store
 
 
 class Redis(object):
@@ -38,11 +38,7 @@ def _fetch_and_unzip(self, url: str, destfile: str):
         if os.path.isfile(destfile):
             return
 
-        r = requests.get(url, stream=True)
-        if r.status_code > 204:
-            logger.error(f"{url} could not be retrieved")
-            raise requests.HTTPError
-        open(destfile, "wb").write(r.content)
+        get_stream_and_store(url, destfile)
 
         logger.debug(f"Unzipping {destfile} and storing in {self.__PATHS__.DESTDIR}")
         with tarfile.open(destfile) as tar:

stack/components/redisinsight.py

@@ -10,11 +10,11 @@
 import urllib
 from typing import Union
 
-import requests
 from loguru import logger
 
 from ..config import Config
 from ..paths import Paths
+from .get import get_stream_and_store
 
 
 class RedisInsightBase(object):
@@ -51,11 +51,7 @@ def _fetch_and_unzip(self, url: str, destfile: str):
         logger.debug(f"Package URL: {url}")
 
         if not os.path.isfile(destfile):
-            r = requests.get(url, stream=True)
-            if r.status_code > 204:
-                logger.error(f"{url} could not be retrieved")
-                raise requests.HTTPError
-            open(destfile, "wb").write(r.content)
+            get_stream_and_store(url, destfile)
 
         # logger.debug(f"Unzipping {destfile} and storing in {self.__PATHS__.DESTDIR}")
         # with zipfile.ZipFile(destfile, "r") as zp: