From 487ef518f3de95c793d772e4bf8d10215626185c Mon Sep 17 00:00:00 2001
From: IceCodeNew <32576256+IceCodeNew@users.noreply.github.com>
Date: Wed, 9 Sep 2020 02:46:46 +0800
Subject: [PATCH] Propose a better way to fetch latest source url.

It is reasonable to pass headers like `User-Agent`, see https://github.com/v2ray/v2ray-core/issues/2373.
Also I don't think it's a good idea to fetch source code url from webpages like `https://www.boost.org/users/download/`, they are definitely not designed for machine-readable. And we should prefer to fetch url from REST API response from GitHub.
---
 qbittorrent-nox-static-glibc.sh | 40 +++++++++++++++++++--------------
 qbittorrent-nox-static-musl.sh  | 34 ++++++++++++++++------------
 qbittorrent-nox-staticish.sh    | 34 ++++++++++++++++------------
 3 files changed, 63 insertions(+), 45 deletions(-)

diff --git a/qbittorrent-nox-static-glibc.sh b/qbittorrent-nox-static-glibc.sh
index d58c5ceb5..c90472036 100644
--- a/qbittorrent-nox-static-glibc.sh
+++ b/qbittorrent-nox-static-glibc.sh
@@ -229,34 +229,40 @@ export local_openssl="--with-openssl=$install_dir"
 #
 ## Define some URLs to download our apps. They are dynamic and set the most recent version or release.
 #
-export bison_url="http://ftpmirror.gnu.org/gnu/bison/$(curl -sSNL http://ftpmirror.gnu.org/gnu/bison/ | grep -Eo 'bison-([0-9]{1,3}[.]?)([0-9]{1,3}[.]?)([0-9]{1,3}?)\.tar.gz' | sort -V | tail -1)"
+export bison_url="http://ftpmirror.gnu.org/gnu/bison/$(curl -sSNLR http://ftpmirror.gnu.org/gnu/bison/ | grep -Eo 'bison-([0-9]{1,3}[.]?)([0-9]{1,3}[.]?)([0-9]{1,3}?)\.tar.gz' | sort -V | tail -1)"
 #
-export gawk_url="http://ftpmirror.gnu.org/gnu/gawk/$(curl -sSNL http://ftpmirror.gnu.org/gnu/gawk/ | grep -Eo 'gawk-([0-9]{1,3}[.]?)([0-9]{1,3}[.]?)([0-9]{1,3}?)\.tar.gz' | sort -V | tail -1)"
+export gawk_url="http://ftpmirror.gnu.org/gnu/gawk/$(curl -sSNLR http://ftpmirror.gnu.org/gnu/gawk/ | grep -Eo 'gawk-([0-9]{1,3}[.]?)([0-9]{1,3}[.]?)([0-9]{1,3}?)\.tar.gz' | sort -V | tail -1)"
 #
-# export glibc_url="http://ftpmirror.gnu.org/gnu/libc/$(curl -sSNL http://ftpmirror.gnu.org/gnu/libc/ | grep -Eo 'glibc-([0-9]{1,3}[.]?)([0-9]{1,3}[.]?)([0-9]{1,3}?)\.tar.gz' | sort -V | tail -1)"
+# export glibc_url="http://ftpmirror.gnu.org/gnu/libc/$(curl -sSNLR http://ftpmirror.gnu.org/gnu/libc/ | grep -Eo 'glibc-([0-9]{1,3}[.]?)([0-9]{1,3}[.]?)([0-9]{1,3}?)\.tar.gz' | sort -V | tail -1)"
 export glibc_url="http://ftpmirror.gnu.org/gnu/libc/glibc-2.31.tar.gz"
 #
-export zlib_github_tag="$(curl -sNL https://github.com/madler/zlib/releases | grep -Eom1 'v1.2.([0-9]{1,2})')"
-export zlib_url="https://github.com/madler/zlib/archive/$zlib_github_tag.tar.gz"
+export zlib_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                               https://api.github.com/repos/madler/zlib/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 '1\.2\.')"
+export zlib_url='https://github.com/madler/zlib/archive/'"$zlib_github_tag"'.tar.gz'
 #
-export icu_url="$(curl -sNL https://api.github.com/repos/unicode-org/icu/releases/latest | grep -Eom1 'ht(.*)icu4c(.*)-src.tgz')"
+export icu_url="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                       https://api.github.com/repos/unicode-org/icu/releases/latest | grep 'browser_download_url' | cut -d\" -f4 | grep -Em1 'icu4c(.+)-src.tgz')"
 #
-export openssl_github_tag="$(curl -sNL https://github.com/openssl/openssl/releases | grep -Eom1 'OpenSSL_1_1_([0-9][a-z])')"
-export openssl_url="https://github.com/openssl/openssl/archive/$openssl_github_tag.tar.gz"
+export openssl_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                                  https://api.github.com/repos/openssl/openssl/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 'OpenSSL_1_1_[0-9][a-z]')"
+export openssl_url='https://github.com/openssl/openssl/archive/'"$openssl_github_tag"'.tar.gz'
 #
-export boost_version="$(curl -sNL https://www.boost.org/users/download/ | sed -rn 's#(.*)e">Version (.*\.[0-9]{1,2})</s(.*)#\2#p')"
-export boost_github_tag="boost-$boost_version"
+export boost_prefix='boost-'
+export boost_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                             https://api.github.com/repos/boostorg/boost/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 ${boost_prefix}'1\.[0-9].+')"
+export boost_version="${boost_github_tag/${boost_prefix}/}"
 export boost_url="https://dl.bintray.com/boostorg/release/$boost_version/source/boost_${boost_version//./_}.tar.gz"
-export boost_url_status="$(curl -o /dev/null --silent --head --write-out '%{http_code}' https://dl.bintray.com/boostorg/release/$boost_version/source/boost_${boost_version//./_}.tar.gz)"
-export boost_build_url="https://github.com/boostorg/build/archive/$boost_github_tag.tar.gz"
+export boost_url_status="$(curl -o /dev/null -s --head --write-out '%{http_code}' https://dl.bintray.com/boostorg/release/$boost_version/source/boost_${boost_version//./_}.tar.gz)"
+export boost_build_url='https://github.com/boostorg/build/archive/'"$boost_github_tag"'.tar.gz'
 #
-export qt_version='5.15'
-export qt_github_tag="$(curl -sNL https://github.com/qt/qtbase/releases | grep -Eom1 "v$qt_version.([0-9]{1,2})")"
+export qt_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                             https://api.github.com/repos/qt/qtbase/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 '5\.15\.')"
 #
-export libtorrent_version='1.2'
-export libtorrent_github_tag="$(curl -sNL https://github.com/arvidn/libtorrent/releases | grep -Eom1 "v$libtorrent_version.([0-9]{1,2})")"
+export libtorrent_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                                     https://api.github.com/repos/arvidn/libtorrent/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 'libtorrent-1\.2\.')"
 #
-export qbittorrent_github_tag="$(curl -sNL https://github.com/qbittorrent/qBittorrent/releases | grep -Eom1 'release-([0-9]{1,4}\.?)+')"
+export qbittorrent_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                                      https://api.github.com/repos/qbittorrent/qBittorrent/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 'release-[0-9]\.')"
 #
 ## bison
 #
diff --git a/qbittorrent-nox-static-musl.sh b/qbittorrent-nox-static-musl.sh
index f5e2b86ca..526f8ac12 100644
--- a/qbittorrent-nox-static-musl.sh
+++ b/qbittorrent-nox-static-musl.sh
@@ -227,27 +227,33 @@ export local_openssl="--with-openssl=$install_dir"
 #
 ## Define some URLs to download our apps. They are dynamic and set the most recent version or release.
 #
-export zlib_github_tag="$(curl -sNL https://github.com/madler/zlib/releases | grep -Eom1 'v1.2.([0-9]{1,2})')"
-export zlib_url="https://github.com/madler/zlib/archive/$zlib_github_tag.tar.gz"
+export zlib_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                               https://api.github.com/repos/madler/zlib/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 '1\.2\.')"
+export zlib_url='https://github.com/madler/zlib/archive/'"$zlib_github_tag"'.tar.gz'
 #
-export icu_url="$(curl -sNL https://api.github.com/repos/unicode-org/icu/releases/latest | grep -Eom1 'ht(.*)icu4c(.*)-src.tgz')"
+export icu_url="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                       https://api.github.com/repos/unicode-org/icu/releases/latest | grep 'browser_download_url' | cut -d\" -f4 | grep -Em1 'icu4c(.+)-src.tgz')"
 #
-export openssl_github_tag="$(curl -sNL https://github.com/openssl/openssl/releases | grep -Eom1 'OpenSSL_1_1_([0-9][a-z])')"
-export openssl_url="https://github.com/openssl/openssl/archive/$openssl_github_tag.tar.gz"
+export openssl_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                                  https://api.github.com/repos/openssl/openssl/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 'OpenSSL_1_1_[0-9][a-z]')"
+export openssl_url='https://github.com/openssl/openssl/archive/'"$openssl_github_tag"'.tar.gz'
 #
-export boost_version="$(curl -sNL https://www.boost.org/users/download/ | sed -rn 's#(.*)e">Version (.*\.[0-9]{1,2})</s(.*)#\2#p')"
-export boost_github_tag="boost-$boost_version"
+export boost_prefix='boost-'
+export boost_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                             https://api.github.com/repos/boostorg/boost/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 ${boost_prefix}'1\.[0-9].+')"
+export boost_version="${boost_github_tag/${boost_prefix}/}"
 export boost_url="https://dl.bintray.com/boostorg/release/$boost_version/source/boost_${boost_version//./_}.tar.gz"
-export boost_url_status="$(curl -o /dev/null --silent --head --write-out '%{http_code}' https://dl.bintray.com/boostorg/release/$boost_version/source/boost_${boost_version//./_}.tar.gz)"
-export boost_build_url="https://github.com/boostorg/build/archive/$boost_github_tag.tar.gz"
+export boost_url_status="$(curl -o /dev/null -s --head --write-out '%{http_code}' https://dl.bintray.com/boostorg/release/$boost_version/source/boost_${boost_version//./_}.tar.gz)"
+export boost_build_url='https://github.com/boostorg/build/archive/'"$boost_github_tag"'.tar.gz'
 #
-export qt_version='5.15'
-export qt_github_tag="$(curl -sNL https://github.com/qt/qtbase/releases | grep -Eom1 "v$qt_version.([0-9]{1,2})")"
+export qt_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                             https://api.github.com/repos/qt/qtbase/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 '5\.15\.')"
 #
-export libtorrent_version='1.2'
-export libtorrent_github_tag="$(curl -sNL https://github.com/arvidn/libtorrent/releases | grep -Eom1 "v$libtorrent_version.([0-9]{1,2})")"
+export libtorrent_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                                     https://api.github.com/repos/arvidn/libtorrent/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 'libtorrent-1\.2\.')"
 #
-export qbittorrent_github_tag="$(curl -sNL https://github.com/qbittorrent/qBittorrent/releases | grep -Eom1 'release-([0-9]{1,4}\.?)+')"
+export qbittorrent_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                                      https://api.github.com/repos/qbittorrent/qBittorrent/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 'release-[0-9]\.')"
 #
 ## zlib installation
 #
diff --git a/qbittorrent-nox-staticish.sh b/qbittorrent-nox-staticish.sh
index ed9c728d4..9472044d4 100644
--- a/qbittorrent-nox-staticish.sh
+++ b/qbittorrent-nox-staticish.sh
@@ -220,27 +220,33 @@ export local_openssl="--with-openssl=$install_dir"
 #
 ## Define some URLs to download our apps. They are dynamic and set the most recent version or release.
 #
-export zlib_github_tag="$(curl -sNL https://github.com/madler/zlib/releases | grep -Eom1 'v1.2.([0-9]{1,2})')"
-export zlib_url="https://github.com/madler/zlib/archive/$zlib_github_tag.tar.gz"
+export zlib_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                               https://api.github.com/repos/madler/zlib/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 '1\.2\.')"
+export zlib_url='https://github.com/madler/zlib/archive/'"$zlib_github_tag"'.tar.gz'
 #
-export icu_url="$(curl -sNL https://api.github.com/repos/unicode-org/icu/releases/latest | grep -Eom1 'ht(.*)icu4c(.*)-src.tgz')"
+export icu_url="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                       https://api.github.com/repos/unicode-org/icu/releases/latest | grep 'browser_download_url' | cut -d\" -f4 | grep -Em1 'icu4c(.+)-src.tgz')"
 #
-export openssl_github_tag="$(curl -sNL https://github.com/openssl/openssl/releases | grep -Eom1 'OpenSSL_1_1_([0-9][a-z])')"
-export openssl_url="https://github.com/openssl/openssl/archive/$openssl_github_tag.tar.gz"
+export openssl_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                                  https://api.github.com/repos/openssl/openssl/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 'OpenSSL_1_1_[0-9][a-z]')"
+export openssl_url='https://github.com/openssl/openssl/archive/'"$openssl_github_tag"'.tar.gz'
 #
-export boost_version="$(curl -sNL https://www.boost.org/users/download/ | sed -rn 's#(.*)e">Version (.*\.[0-9]{1,2})</s(.*)#\2#p')"
-export boost_github_tag="boost-$boost_version"
+export boost_prefix='boost-'
+export boost_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                             https://api.github.com/repos/boostorg/boost/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 ${boost_prefix}'1\.[0-9].+')"
+export boost_version="${boost_github_tag/${boost_prefix}/}"
 export boost_url="https://dl.bintray.com/boostorg/release/$boost_version/source/boost_${boost_version//./_}.tar.gz"
-export boost_url_status="$(curl -o /dev/null --silent --head --write-out '%{http_code}' https://dl.bintray.com/boostorg/release/$boost_version/source/boost_${boost_version//./_}.tar.gz)"
-export boost_build_url="https://github.com/boostorg/build/archive/$boost_github_tag.tar.gz"
+export boost_url_status="$(curl -o /dev/null -s --head --write-out '%{http_code}' https://dl.bintray.com/boostorg/release/$boost_version/source/boost_${boost_version//./_}.tar.gz)"
+export boost_build_url='https://github.com/boostorg/build/archive/'"$boost_github_tag"'.tar.gz'
 #
-export qt_version='5.15'
-export qt_github_tag="$(curl -sNL https://github.com/qt/qtbase/releases | grep -Eom1 "v$qt_version.([0-9]{1,2})")"
+export qt_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                             https://api.github.com/repos/qt/qtbase/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 '5\.15\.')"
 #
-export libtorrent_version='1.2'
-export libtorrent_github_tag="$(curl -sNL https://github.com/arvidn/libtorrent/releases | grep -Eom1 "v$libtorrent_version.([0-9]{1,2})")"
+export libtorrent_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                                     https://api.github.com/repos/arvidn/libtorrent/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 'libtorrent-1\.2\.')"
 #
-export qbittorrent_github_tag="$(curl -sNL https://github.com/qbittorrent/qBittorrent/releases | grep -Eom1 'release-([0-9]{1,4}\.?)+')"
+export qbittorrent_github_tag="$(curl -H "Accept: application/json" -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" -sSNLR \
+                                      https://api.github.com/repos/qbittorrent/qBittorrent/tags?per_page=32 | grep 'name' | cut -d\" -f4 | grep -Em1 'release-[0-9]\.')"
 #
 ## zlib installation
 #