Update repo #40571
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update repo | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths: | |
| - '.github/workflows/repo-update.yml' | |
| schedule: | |
| - cron: '0,30 0-23 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-repo: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| target_repo: [main-aarch64, main-arm, main-i686, main-x86_64, x11-aarch64, x11-arm, x11-i686, x11-x86_64, root-aarch64, root-arm, root-i686, root-x86_64] | |
| fail-fast: false | |
| steps: | |
| - name: Clone check-tools | |
| run: git clone "https://termux-pacman-bot:${{ secrets.TOKEN }}@github.com/termux-pacman/check-tools" | |
| - name: Import key of Termux-Pacman | |
| run: gpg --keyserver keyserver.ubuntu.com --recv-keys 998de27318e867ea976ba877389ceed64573dfca | |
| - name: Clone termux-packages | |
| run: | | |
| cd check-tools | |
| git clone https://github.com/termux-pacman/termux-packages | |
| - name: Running a scan | |
| run: | | |
| target_repo='${{ matrix.target_repo }}' | |
| REPO="${target_repo%%-*}" | |
| ARCH="${target_repo#*-}" | |
| TOTAL="RESULT.${REPO}.${ARCH}" | |
| case "$REPO" in | |
| "main") DIR_REPO="termux-packages/packages";; | |
| "x11") DIR_REPO="termux-packages/x11-packages";; | |
| "root") DIR_REPO="termux-packages/root-packages";; | |
| *) | |
| echo "Error: repo '${REPO}' is unknown" | |
| exit 0;; | |
| esac | |
| cd check-tools | |
| if [ ! -d "$DIR_REPO" ]; then | |
| echo "Error: '$DIR_REPO' not found." | |
| exit 0 | |
| fi | |
| bash ./check_pkgs_by_repo.sh "$REPO" "$ARCH" "$DIR_REPO" | |
| # Packages should be removed as they do not support the architecture | |
| if [ -f "./output/NOTSUPPORTARCH.${REPO}.${ARCH}" ]; then | |
| # Install aws-cli | |
| git clone https://github.com/termux-pacman/aws-cli-action | |
| ./aws-cli-action/setup.sh '${{ secrets.AWS_ACCESS_KEY_ID }}' '${{ secrets.AWS_ACCESS_KEY }}' '${{ secrets.AWS_REGION }}' | |
| # Import gpg key | |
| mkdir gk | |
| echo "${{ secrets.SF_GPG_BOT }}" > gk/termux-pacman-bot.gpg | |
| gpg --pinentry-mode=loopback --passphrase '${{ secrets.PW_GPG_BOT }}' --import gk/termux-pacman-bot.gpg | |
| rm -fr gk | |
| # Upload file | |
| DELETED_FILE="./output/$(date +%s)-deleted_termux-${REPO}_packages.txt" | |
| mv "./output/NOTSUPPORTARCH.${REPO}.${ARCH}" "${DELETED_FILE}" | |
| gpg --batch --pinentry-mode=loopback --passphrase '${{ secrets.PW_GPG_BOT }}' --detach-sign --use-agent -u '${{ secrets.KEY_GPG_BOT }}' --no-armor "${DELETED_FILE}" | |
| for format_file in "" ".sig"; do | |
| aws s3 cp "${DELETED_FILE}${format_file}" s3://'${{ secrets.SFPU }}'/${REPO}/${ARCH}/"${DELETED_FILE##*/}${format_file}" | |
| done | |
| rm "${DELETED_FILE}.sig" | |
| fi | |
| mkdir total | |
| touch ./total/${TOTAL} | |
| # NOTFOUND.${REPO}.${ARCH} | |
| for RESULT in "./output/DOESNOTMATCH.${REPO}.${ARCH}"; do | |
| if [ -f ${RESULT} ]; then | |
| cat ${RESULT} | awk '{printf $1 "\n"}' >> ./total/${TOTAL} | |
| fi | |
| done | |
| tar cf total-${{ matrix.target_repo }}-${{ github.sha }}.tar.gz ./total | |
| - name: Store RESULT.* files | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: total-${{ matrix.target_repo }}-${{ github.sha }} | |
| path: ./check-tools/total-${{ matrix.target_repo }}-${{ github.sha }}.tar.gz | |
| if-no-files-found: error | |
| update-repo: | |
| needs: check-repo | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Clone repository | |
| run: git clone https://github.com/termux-pacman/termux-packages.git termux-packages-pacman | |
| - name: Get RESULT.* files | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ./ | |
| - name: Creating a REBUILD-LIST-BUILD | |
| run: | | |
| for i in ./total-*/total-*.tar.gz; do | |
| tar xf $i -C ./ | |
| done | |
| rm -fr ./total-*/ | |
| touch ./REBUILD-LIST-BUILD | |
| cd termux-packages-pacman | |
| for i in $(sort ../total/RESULT.* | uniq); do | |
| BUILD_SOURCE=$(ls */${i}/build.sh 2> /dev/null) || true | |
| if [ -z "${BUILD_SOURCE}" ]; then | |
| BUILD_SOURCE=$(ls */*/${i}.subpackage.sh 2> /dev/null) || true | |
| if [ -n "${BUILD_SOURCE}" ]; then | |
| BUILD_SOURCE="$(dirname ${BUILD_SOURCE})/build.sh" | |
| else | |
| echo "Source package '${i}' was not found, skip" | |
| continue | |
| fi | |
| fi | |
| if ! $(grep -q '^'${BUILD_SOURCE}'$' ../REBUILD-LIST-BUILD) && [ $(($(expr $(expr $(date +%s) - $(git log -n 1 --date=format:'%s' --format=%cd -- $(dirname ${BUILD_SOURCE}))) / 3600) > 8)) = 1 ]; then | |
| echo "${BUILD_SOURCE}" >> ../REBUILD-LIST-BUILD | |
| fi | |
| done | |
| cd .. | |
| - name: Clone root repository | |
| run: git clone https://github.com/termux/termux-packages.git | |
| - name: Update repo | |
| run: | | |
| info() { | |
| echo "==> $1" | |
| } | |
| commit() { | |
| echo "-> $1" | |
| } | |
| install_list_issues() { | |
| if [ -f ../LIST-ISSUES ]; then | |
| rm ../LIST-ISSUES | |
| fi | |
| curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -s -X GET -G \ | |
| "https://api.github.com/repos/${{ github.repository }}/issues" -o ../LIST-ISSUES | |
| } | |
| # Edit | |
| info "Edit" | |
| cd termux-packages-pacman | |
| rm -fr */ | |
| cd ../termux-packages | |
| cp -r * ../termux-packages-pacman | |
| # Get list pkg | |
| info "Get list pkg" | |
| cd ../termux-packages-pacman | |
| list_files=" " | |
| for i in $(git status -s packages root-packages x11-packages | awk '{print $2}'); do | |
| dir_sp=(${i//// }) | |
| if ! $(echo "$list_files" | grep -q " ${dir_sp[0]}/${dir_sp[1]} "); then | |
| list_files+="${dir_sp[0]}/${dir_sp[1]} " | |
| source_build="${dir_sp[0]}/${dir_sp[1]}/build.sh" | |
| if $(grep -q '^'${source_build}'$' ../REBUILD-LIST-BUILD); then | |
| sed -i "/^${source_build////'\/'}$/d" ../REBUILD-LIST-BUILD | |
| fi | |
| fi | |
| done | |
| list_files=($list_files) | |
| # Sort list pkg | |
| info "Sort list pkg" | |
| cd ../termux-packages | |
| declare -A list_values | |
| list_sort="" | |
| list_sha="" | |
| for i in ${list_files[@]}; do | |
| sha_file=$(git log -n 1 --pretty=format:%H -- $i) | |
| value="$(git rev-list --count $sha_file)" | |
| list_values[${value}]=$((${list_values[${value}]}+1)) | |
| list_sort+="${value}-${list_values[${value}]} " | |
| list_sha+="${sha_file} " | |
| done | |
| list_sort=($list_sort) | |
| list_sha=($list_sha) | |
| copy_list_sort=($(sort -n <(printf "%s\n" "${list_sort[@]}"))) | |
| search_index() { | |
| for i in "${!list_sort[@]}"; do | |
| if [[ "${list_sort[$i]}" = "${1}" ]]; then | |
| echo "${i}"; | |
| fi | |
| done | |
| } | |
| sort_list_files="" | |
| for i in ${copy_list_sort[@]}; do | |
| index=$(search_index "$i") | |
| if [[ "${i#*-}" = "1" ]]; then | |
| sort_list_files+=" ${list_sha[$index]}###" | |
| else | |
| sort_list_files+="&&" | |
| fi | |
| sort_list_files+="${list_files[$index]}" | |
| done | |
| # Start upload | |
| info "Start upload" | |
| cd ../termux-packages-pacman | |
| needbuild=false | |
| if $(git log -1 --pretty=%B | grep -q '%needbuild'); then | |
| needbuild=true | |
| fi | |
| git config --global user.name "termux-pacman-bot" | |
| git config --global user.email "termux-pacman-bot@outlook.com" | |
| git remote set-url origin "https://termux-pacman-bot:${{ secrets.TOKEN }}@github.com/termux-pacman/termux-packages.git" | |
| commit "Update system repo" | |
| git add . | |
| git reset packages root-packages x11-packages README.md SECURITY.md repo.json | |
| { | |
| git commit -m "Update repo" | |
| git push origin master | |
| } || true | |
| commit "Update repo packages" | |
| install_list_issues | |
| for i in $sort_list_files; do | |
| i_sp=(${i//###/ }) | |
| commit " ${i_sp[1]}" | |
| cd ../termux-packages | |
| commit_file="$(git show -s --format=%B ${i_sp[0]})" | |
| cd ../termux-packages-pacman | |
| git add $(echo ${i_sp[1]} | sed 's/&&/ /g') | |
| { | |
| if $needbuild; then | |
| git commit -m "$(echo $commit_file | sed 's/%ci:no-build//g; s/\[skip ci\]//g')" | |
| else | |
| git commit -m "$commit_file" | |
| fi | |
| git push origin master | |
| } || true | |
| for j in $(echo ${i_sp[1]} | sed 's/&&/ /g'); do | |
| while read LINE; do | |
| USER=$(echo "$LINE" | awk -F ' // ' '{printf $1}') | |
| STATE=$(echo "$LINE" | awk -F ' // ' '{printf $2}') | |
| TITLE=$(echo "$LINE" | awk -F ' // ' '{printf $4}') | |
| if [ $STATE != "open" ] || [ $USER != "termux-pacman-bot" ] || [[ $TITLE != "[AUTO-CHECK]: failed to recompile ${j}" ]]; then | |
| continue | |
| fi | |
| NUMBER=$(echo "$LINE" | awk -F ' // ' '{printf $3}') | |
| GH_TOKEN="${{ secrets.TOKEN }}" gh issue close ${NUMBER} \ | |
| -c "The package has been updated - https://github.com/termux-pacman/termux-packages/commit/$(git log -n 1 --format=%H -- ${j})." | |
| install_list_issues | |
| break | |
| done <<< $(jq -r '.[] | "\(.user.login) \("//") \(.state) \("//") \(.number) \("//") \(.title)"' ../LIST-ISSUES) | |
| done | |
| done | |
| # Recompile packages if necessary | |
| if [ -n "$(cat ../REBUILD-LIST-BUILD)" ]; then | |
| info "Rebuild packages" | |
| listchanged="" | |
| for i in $(cat ../REBUILD-LIST-BUILD); do | |
| # It is necessary to create an issue because the package has | |
| # a compilation error even after recompilation. | |
| if [[ $(git log -n 1 --format=%B -- "${i}") = "rebuild($(dirname ${i})): auto check by @termux-pacman-bot" ]] && \ | |
| [[ $(git log -n 1 --format=%an -- "${i}") = "termux-pacman-bot" ]] && \ | |
| [[ $(git log -n 1 --format=%ae -- "${i}") = "termux-pacman-bot@outlook.com" ]]; then | |
| if ! $(jq -r '.[] | "\(.user.login) \("/") \(.state) \("/") \(.title)"' ../LIST-ISSUES | grep -q "^termux-pacman-bot / open / \[AUTO-CHECK\]:.*$(dirname ${i})$"); then | |
| commit "Create an issue on $(dirname ${i})" | |
| GH_TOKEN="${{ secrets.TOKEN }}" gh issue create \ | |
| --title "[AUTO-CHECK]: failed to recompile $(dirname ${i})" \ | |
| --body "$(echo -e "This issue was automatically generated because the '$(dirname ${i})' package was not recompiled successfully.\n\nCommit by recompilation - https://github.com/termux-pacman/termux-packages/commit/$(git log -n 1 --format=%H -- ${i})")" \ | |
| --label "auto recompilation failed" \ | |
| --assignee Maxython | |
| fi | |
| continue | |
| fi | |
| echo "" >> ./${i} | |
| git add ./${i} | |
| listchanged+=" ${i}" | |
| done | |
| if [ -n "${listchanged}" ]; then | |
| git commit -m "$(echo -e "termux-pacman/auto-check-repo: preparing to recompile packages\n[skip ci]")" | |
| git push origin master | |
| for i in ${listchanged}; do | |
| sed -i '$d' ./${i} | |
| git add ./${i} | |
| git commit -m "rebuild($(dirname ${i})): auto check by @termux-pacman-bot" | |
| git push origin master | |
| done | |
| fi | |
| fi |