Merge remote-tracking branch 'upstream/master'

ShareX · Jan 11, 2024 · 61b6856 · 61b6856
2 parents 2b2d712 + bf74c79
commit 61b6856
Show file tree

Hide file tree

Showing 115 changed files with 253 additions and 404 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -63,11 +63,8 @@ jobs:
         id: imagename
         run: |
           IMG="${GITHUB_REPOSITORY,,}/base"
-          DLIMG="${GITHUB_REPOSITORY,,}/dl_cache"
           echo "name=ghcr.io/${IMG/ /-}" >> $GITHUB_OUTPUT
           echo "rawname=${IMG/ /-}" >> $GITHUB_OUTPUT
-          echo "dlname=ghcr.io/${DLIMG/ /-}" >> $GITHUB_OUTPUT
-          echo "dlrawname=${DLIMG/ /-}" >> $GITHUB_OUTPUT
       - name: Stop Commands
         run: T="$(echo -n ${{ github.token }} | sha256sum | head -c 64)" && echo -e "::add-mask::${T}\n::stop-commands::${T}"
       - name: Build target base image
@@ -80,23 +77,22 @@ jobs:
           tags: ${{ steps.imagename.outputs.name }}:latest
           cache-to: type=registry,mode=max,ref=${{ steps.imagename.outputs.name }}:cache
           cache-from: type=registry,ref=${{ steps.imagename.outputs.name }}:cache
-      - name: Generate download cache Dockerfile
+      - name: Get download cache key
         id: dl_cache
         run: |
-          ./generate.sh dl only
           echo "dltagname=$(./util/get_dl_cache_tag.sh)" >> $GITHUB_OUTPUT
-          cat Dockerfile.dl
-      - name: Build download cache image
-        uses: docker/build-push-action@v5
+      - name: Cache
+        uses: actions/cache@v3
         with:
-          context: .
-          file: Dockerfile.dl
-          pull: true
-          push: true
-          provenance: false
-          tags: ${{ steps.imagename.outputs.dlname }}:${{ steps.dl_cache.outputs.dltagname }}
-          cache-to: type=registry,mode=max,ref=${{ steps.imagename.outputs.dlname }}:cache
-          cache-from: type=registry,ref=${{ steps.imagename.outputs.dlname }}:cache
+          path: .cache/downloads
+          key: download-cache-${{ steps.dl_cache.outputs.dltagname }}
+          restore-keys: |
+            download-cache-
+      - name: Update Cache
+        run: |
+          set -e
+          ./download.sh
+          ./util/clean_cache.sh
       - name: Cleanup
         if: ${{ env.HAVE_CLEANUP_PAT == 'true' }}
         continue-on-error: true
@@ -108,18 +104,6 @@ jobs:
           repository: ${{ github.repository }}
           owner_type: user
           untagged_only: true
-      - name: Cleanup Download Cache
-        if: ${{ env.HAVE_CLEANUP_PAT == 'true' }}
-        continue-on-error: true
-        uses: BtbN/delete-untagged-ghcr-action@main
-        with:
-          token: ${{ secrets.CLEANUP_PAT }}
-          package_name: ${{ steps.imagename.outputs.dlrawname }}
-          repository_owner: ${{ github.repository_owner }}
-          repository: ${{ github.repository }}
-          owner_type: user
-          untagged_only: false
-          keep_latest: 20
   build_target_bases:
     name: Build target base image
     if: ${{ github.event.inputs.buildOnly != 'true' }}
@@ -185,7 +169,7 @@ jobs:
       fail-fast: false
       matrix:
         target: [win64,win32]
-        variant: [gpl,gpl 6.0]
+        variant: [gpl,gpl 6.1]
     steps:
       - name: Free Disk-Space
         run: df -h && sudo apt-get clean && docker system prune -a -f && sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc && df -h
@@ -201,16 +185,26 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ github.token }}
-      - name: Generate Dockerfile
-        run: ./generate.sh ${{ matrix.target }} ${{ matrix.variant }}
       - name: Image Name
         id: imagename
         run: |
           IMG="${GITHUB_REPOSITORY,,}/${{ matrix.target }}-${{ matrix.variant }}"
           echo "name=ghcr.io/${IMG/ /-}" >> $GITHUB_OUTPUT
           echo "rawname=${IMG/ /-}" >> $GITHUB_OUTPUT
+      - name: Get download cache key
+        id: dl_cache
+        run: |
+          echo "dltagname=$(./util/get_dl_cache_tag.sh)" >> $GITHUB_OUTPUT
       - name: Stop Commands
         run: T="$(echo -n ${{ github.token }} | sha256sum | head -c 64)" && echo -e "::add-mask::${T}\n::stop-commands::${T}"
+      - name: Generate Dockerfile
+        run: ./generate.sh ${{ matrix.target }} ${{ matrix.variant }}
+      - name: Cache
+        uses: actions/cache/restore@v3
+        with:
+          path: .cache/downloads
+          key: download-cache-${{ steps.dl_cache.outputs.dltagname }}
+          fail-on-cache-miss: true
       - name: Build target base image
         uses: docker/build-push-action@v5
         with:
@@ -241,7 +235,7 @@ jobs:
       fail-fast: false
       matrix:
         target: [win64,win32]
-        variant: [gpl,gpl 6.0]
+        variant: [gpl,gpl 6.1]
     steps:
       - name: Free Disk-Space
         run: df -h && sudo apt-get clean && docker system prune -a -f && sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc && df -h

diff --git a/build.sh b/build.sh
@@ -46,6 +46,7 @@ docker run --rm -v "$PWD:/uidtestdir" "$IMAGE" touch "/uidtestdir/$TESTFILE"
 DOCKERUID="$(stat -c "%u" "$TESTFILE")"
 rm -f "$TESTFILE"
 [[ "$DOCKERUID" != "$(id -u)" ]] && UIDARGS=( -u "$(id -u):$(id -g)" ) || UIDARGS=()
+unset TESTFILE
 
 rm -rf ffbuild
 mkdir ffbuild
@@ -79,7 +80,7 @@ EOF
 
 [[ -t 1 ]] && TTY_ARG="-t" || TTY_ARG=""
 
-docker run --rm -i $TTY_ARG "${UIDARGS[@]}" -v $PWD/ffbuild:/ffbuild -v "$BUILD_SCRIPT":/build.sh "$IMAGE" bash /build.sh
+docker run --rm -i $TTY_ARG "${UIDARGS[@]}" -v "$PWD/ffbuild":/ffbuild -v "$BUILD_SCRIPT":/build.sh "$IMAGE" bash /build.sh
 
 mkdir -p artifacts
 ARTIFACTS_PATH="$PWD/artifacts"

diff --git a/download.sh b/download.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+set -xe
+cd "$(dirname "$0")"
+source util/vars.sh dl only
+
+TESTFILE="uidtestfile"
+rm -f "$TESTFILE"
+docker run --rm -v "$PWD:/uidtestdir" "${REGISTRY}/${REPO}/base:latest" touch "/uidtestdir/$TESTFILE"
+DOCKERUID="$(stat -c "%u" "$TESTFILE")"
+rm -f "$TESTFILE"
+[[ "$DOCKERUID" != "$(id -u)" ]] && UIDARGS=( -u "$(id -u):$(id -g)" ) || UIDARGS=()
+unset TESTFILE
+
+[[ -t 1 ]] && TTY_ARG="-t" || TTY_ARG=""
+
+DL_SCRIPT_DIR="$(mktemp -d)"
+trap "rm -rf -- '$DL_SCRIPT_DIR'" EXIT
+
+mkdir -p "${PWD}"/.cache/downloads
+
+for STAGE in scripts.d/*.sh scripts.d/*/*.sh; do
+    STAGENAME="$(basename "$STAGE" | sed 's/.sh$//')"
+
+	cat <<-EOF >"${DL_SCRIPT_DIR}/${STAGENAME}.sh"
+		set -xe -o pipefail
+		shopt -s dotglob
+
+        source /dl_functions.sh
+		source "/$STAGE"
+		STG="\$(ffbuild_dockerdl)"
+
+		if [[ -z "\$STG" ]]; then
+			exit 0
+		fi
+
+		DLHASH="\$(sha256sum <<<"\$STG" | cut -d" " -f1)"
+		DLNAME="$STAGENAME"
+
+		if [[ "$1" == "hashonly" ]]; then
+			echo "\$DLHASH"
+			exit 0
+		fi
+
+		TGT="/dldir/\${DLNAME}_\${DLHASH}.tar.xz"
+		if [[ -f "\$TGT" ]]; then
+			rm -f "/dldir/\${DLNAME}.tar.xz"
+			ln -s "\${DLNAME}_\${DLHASH}.tar.xz" "/dldir/\${DLNAME}.tar.xz"
+			exit 0
+		fi
+
+		WORKDIR="\$(mktemp -d)"
+        trap "rm -rf -- '\$WORKDIR'" EXIT
+		cd "\$WORKDIR"
+
+		eval "\$STG"
+
+		tar cpJf "\$TGT.tmp" .
+		mv "\$TGT.tmp" "\$TGT"
+		rm -f "/dldir/\${DLNAME}.tar.xz"
+		ln -s "\${DLNAME}_\${DLHASH}.tar.xz" "/dldir/\${DLNAME}.tar.xz"
+	EOF
+done
+
+docker run -i $TTY_ARG --rm "${UIDARGS[@]}" -v "${DL_SCRIPT_DIR}":/stages -v "${PWD}/.cache/downloads":/dldir -v "${PWD}/scripts.d":/scripts.d -v "${PWD}/util/dl_functions.sh":/dl_functions.sh "${REGISTRY}/${REPO}/base:latest" \
+    bash -c 'set -xe && for STAGE in /stages/*.sh; do bash $STAGE; done'
diff --git a/generate.sh b/generate.sh
@@ -18,49 +18,6 @@ to_df() {
     echo >> "$_of"
 }
 
-default_dl() {
-    to_df "RUN git-mini-clone \"$SCRIPT_REPO\" \"$SCRIPT_COMMIT\" \"$1\""
-}
-
-###
-### Generate download Dockerfile
-###
-
-exec_dockerstage_dl() {
-    SCRIPT="$1"
-    (
-        SELF="$SCRIPT"
-        SELFLAYER="$(layername "$STAGE")"
-        source "$SCRIPT"
-        ffbuild_dockerdl || exit $?
-        TODF="Dockerfile.dl.final" ffbuild_dockerlayer_dl || exit $?
-    )
-}
-
-export TODF="Dockerfile.dl"
-
-to_df "FROM ${REGISTRY}/${REPO}/base:latest AS base"
-to_df "ENV TARGET=$TARGET VARIANT=$VARIANT REPO=$REPO ADDINS_STR=$ADDINS_STR"
-to_df "WORKDIR \$FFBUILD_DLDIR"
-
-for STAGE in scripts.d/*.sh scripts.d/*/*.sh; do
-    to_df "FROM base AS $(layername "$STAGE")"
-    exec_dockerstage_dl "$STAGE"
-done
-
-to_df "FROM base AS intermediate"
-cat Dockerfile.dl.final >> "$TODF"
-rm Dockerfile.dl.final
-
-to_df "FROM scratch"
-to_df "COPY --from=intermediate /opt/ffdl/. /"
-
-if [[ "$TARGET" == "dl" && "$VARIANT" == "only" ]]; then
-    exit 0
-fi
-
-DL_IMAGE="${DL_IMAGE_RAW}:$(./util/get_dl_cache_tag.sh)"
-
 ###
 ### Generate main Dockerfile
 ###
@@ -69,11 +26,23 @@ exec_dockerstage() {
     SCRIPT="$1"
     (
         SELF="$SCRIPT"
+        STAGENAME="$(basename "$SCRIPT" | sed 's/.sh$//')"
+        source util/dl_functions.sh
         source "$SCRIPT"
 
         ffbuild_enabled || exit 0
 
-        to_df "ENV SELF=\"$SELF\""
+        to_df "ENV SELF=\"$SELF\" STAGENAME=\"$STAGENAME\""
+
+        set -x
+
+        STG="$(ffbuild_dockerdl)"
+        if [[ -n "$STG" ]]; then
+            HASH="$(sha256sum <<<"$STG" | cut -d" " -f1)"
+            to_df "ADD .cache/downloads/${STAGENAME}_${HASH}.tar.xz /${STAGENAME}"
+            to_df "WORKDIR /${STAGENAME}"
+        fi
+
         ffbuild_dockerstage || exit $?
     )
 }
@@ -82,6 +51,7 @@ export TODF="Dockerfile"
 
 to_df "FROM ${REGISTRY}/${REPO}/base-${TARGET}:latest AS base"
 to_df "ENV TARGET=$TARGET VARIANT=$VARIANT REPO=$REPO ADDINS_STR=$ADDINS_STR"
+to_df "COPY util/run_stage.sh /usr/bin/run_stage"
 
 for addin in "${ADDINS[@]}"; do
 (

diff --git a/images/base/Dockerfile b/images/base/Dockerfile
@@ -1,10 +1,7 @@
-FROM ubuntu:23.04
-
-ENV FFBUILD_DLDIR="/opt/ffdl"
+FROM ubuntu:23.10
 
 ENV DEBIAN_FRONTEND noninteractive
 RUN \
-    mkdir -p "$FFBUILD_DLDIR" && \
     apt-get -y update && \
     apt-get -y dist-upgrade && \
     apt-get -y install build-essential yasm nasm \

diff --git a/images/base/retry-tool.sh b/images/base/retry-tool.sh
@@ -1,11 +1,11 @@
 #!/bin/bash
-set -xe
+set -xe -o pipefail
 
 RETRY_COUNTER=0
 MAX_RETRY=10
 CUR_TIMEOUT=120
 while [[ $RETRY_COUNTER -lt $MAX_RETRY ]]; do
-    timeout $CUR_TIMEOUT "$@" && break || sleep $(shuf -i 5-90 -n 1)
+    (timeout $CUR_TIMEOUT "$@") && break || sleep $(shuf -i 5-90 -n 1)
     RETRY_COUNTER=$(( $RETRY_COUNTER + 1 ))
     CUR_TIMEOUT=$(( $CUR_TIMEOUT + 60 ))
     echo "Retry $RETRY_COUNTER..."

diff --git a/makeimage.sh b/makeimage.sh
@@ -23,8 +23,6 @@ docker container inspect ffbuildreg &>/dev/null || \
 LOCAL_REG_PORT="$(docker container inspect --format='{{range $p, $conf := .NetworkSettings.Ports}}{{(index $conf 0).HostPort}}{{end}}' ffbuildreg)"
 LOCAL_ROOT="127.0.0.1:${LOCAL_REG_PORT}/local"
 
-export REGISTRY_OVERRIDE_DL="127.0.0.1:${LOCAL_REG_PORT}" GITHUB_REPOSITORY_DL="local"
-
 if [[ -z "$QUICKBUILD" ]]; then
     if grep "FROM.*base.*" "images/base-${TARGET}/Dockerfile" >/dev/null 2>&1; then
         docker buildx --builder ffbuilder build \
@@ -39,24 +37,11 @@ if [[ -z "$QUICKBUILD" ]]; then
         --push --tag "${LOCAL_ROOT}/base-${TARGET}:latest" \
         --build-arg GH_REPO="$LOCAL_ROOT" "images/base-${TARGET}"
 
-    export REGISTRY_OVERRIDE="$REGISTRY_OVERRIDE_DL" GITHUB_REPOSITORY="$GITHUB_REPOSITORY_DL"
+    export REGISTRY_OVERRIDE="127.0.0.1:${LOCAL_REG_PORT}" GITHUB_REPOSITORY="local"
 fi
 
+./download.sh
 ./generate.sh "$TARGET" "$VARIANT" "${ADDINS[@]}"
-DL_CACHE_TAG="$(./util/get_dl_cache_tag.sh)"
-DL_IMAGE="${DL_IMAGE_RAW}:${DL_CACHE_TAG}"
-
-if docker pull "${DL_IMAGE}"; then
-    export REGISTRY_OVERRIDE_DL="$REGISTRY" GITHUB_REPOSITORY_DL="$REPO"
-    ./generate.sh "$TARGET" "$VARIANT" "${ADDINS[@]}"
-else
-    DL_IMAGE="${LOCAL_ROOT}/dl_cache:${DL_CACHE_TAG}"
-    docker manifest inspect --insecure "${DL_IMAGE}" >/dev/null ||
-        docker buildx --builder ffbuilder build -f Dockerfile.dl \
-            --cache-from=type=local,src=.cache/dl_image_cache \
-            --cache-to=type=local,mode=max,dest=.cache/dl_image_cache \
-            --push --tag "${DL_IMAGE}" .
-fi
 
 docker buildx --builder ffbuilder build \
     --cache-from=type=local,src=.cache/"${IMAGE/:/_}" \

diff --git a/scripts.d/10-mingw-std-threads.sh b/scripts.d/10-mingw-std-threads.sh
@@ -9,8 +9,6 @@ ffbuild_enabled() {
 }
 
 ffbuild_dockerbuild() {
-    cd "$FFBUILD_DLDIR/$SELF"
-
     mkdir -p "$FFBUILD_PREFIX"/include
     cp *.h "$FFBUILD_PREFIX"/include
 }
diff --git a/scripts.d/10-mingw.sh b/scripts.d/10-mingw.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 SCRIPT_REPO="https://git.code.sf.net/p/mingw-w64/mingw-w64.git"
-SCRIPT_COMMIT="7ea930ba381ace6f19f7cfa59596f6fe49949aa6"
+SCRIPT_COMMIT="f2653dd9150006ded8026631bb7b85695edf6127"
 
 ffbuild_enabled() {
     [[ $TARGET == win* ]] || return -1
@@ -18,12 +18,10 @@ ffbuild_dockerfinal() {
 }
 
 ffbuild_dockerdl() {
-    to_df "RUN retry-tool sh -c \"rm -rf mingw && git clone '$SCRIPT_REPO' mingw\" && cd mingw && git checkout \"$SCRIPT_COMMIT\""
+    echo "retry-tool sh -c \"rm -rf mingw && git clone '$SCRIPT_REPO' mingw\" && cd mingw && git checkout \"$SCRIPT_COMMIT\""
 }
 
 ffbuild_dockerbuild() {
-    cd "$FFBUILD_DLDIR/mingw"
-
     cd mingw-w64-headers
 
     unset CFLAGS

diff --git a/scripts.d/10-xorg-macros.sh b/scripts.d/10-xorg-macros.sh
@@ -14,8 +14,6 @@ ffbuild_dockerlayer() {
 }
 
 ffbuild_dockerbuild() {
-    cd "$FFBUILD_DLDIR/$SELF"
-
     autoreconf -i
     ./configure --prefix="$FFBUILD_PREFIX"
     make -j"$(nproc)"

diff --git a/scripts.d/20-libiconv.sh b/scripts.d/20-libiconv.sh
@@ -8,13 +8,11 @@ ffbuild_enabled() {
 }
 
 ffbuild_dockerdl() {
-    to_df "RUN retry-tool sh -c \"rm -rf $SELF && git clone '$SCRIPT_REPO' $SELF\" && git -C $SELF checkout \"$SCRIPT_COMMIT\""
-    to_df "RUN cd $SELF && retry-tool ./autopull.sh --one-time"
+    echo "retry-tool sh -c \"rm -rf iconv && git clone '$SCRIPT_REPO' iconv\" && git -C iconv checkout \"$SCRIPT_COMMIT\""
+    echo "cd iconv && retry-tool ./autopull.sh --one-time"
 }
 
 ffbuild_dockerbuild() {
-    cd "$FFBUILD_DLDIR/$SELF"
-
     (unset CC CFLAGS GMAKE && ./autogen.sh)
 
     local myconf=(