🛍️ (x86_64) Generate | Update ⚙️ Daily Package 📦 Metadata 🗄️ #5099
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🛍️ (x86_64) Generate | Update ⚙️ Daily Package 📦 Metadata 🗄️ | |
| #MAX_RUNTIME: | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 0-23/2 * * *" # Every Even hrs | |
| env: | |
| GITHUB_TOKEN: "${{ secrets.HYSP_TOKEN }}" | |
| jobs: | |
| #------------------------------------------------------------------------------------# | |
| #------------------------------------------------------------------------------------# | |
| fetch-meta: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| path: main | |
| filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
| - name: Install CoreUtils & Deps | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| sudo apt-get update -y | |
| sudo apt-get install coreutils ca-certificates curl git dos2unix lzip jq moreutils p7zip-full util-linux wget -y | |
| pip install ansi2txt --upgrade | |
| #Re | |
| sudo apt-get install coreutils ca-certificates curl git dos2unix lzip jq moreutils p7zip-full util-linux wget -y | |
| pip install ansi2txt --upgrade | |
| continue-on-error: true | |
| - name: Install Addons | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #action-lint | |
| sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/actionlint" -o "/usr/local/bin/actionlint" && sudo chmod +xwr "/usr/local/bin/actionlint" | |
| #delta | |
| sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/delta" -o "/usr/local/bin/delta" && sudo chmod +xwr "/usr/local/bin/delta" | |
| #eget | |
| sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/eget" -o "/usr/local/bin/eget" && sudo chmod +xwr "/usr/local/bin/eget" | |
| #validtoml | |
| sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/validtoml" -o "/usr/local/bin/validtoml" && sudo chmod +xwr "/usr/local/bin/validtoml" | |
| #Yq | |
| sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/yq" -o "/usr/local/bin/yq" && sudo chmod +xwr "/usr/local/bin/yq" | |
| #Yj | |
| sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/yj" -o "/usr/local/bin/yj" && sudo chmod +xwr "/usr/local/bin/yj" | |
| continue-on-error: true | |
| - name: Setup Env | |
| run: | | |
| # Create Output Dir | |
| mkdir -p "$GITHUB_WORKSPACE/main/data/x86_64" | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| #----------------------------# | |
| #For every Github Repo from https://bin.ajam.dev/x86_64_Linux/METADATA.json | |
| - name: Generate Meta ("bin.ajam.dev/x86_64_Linux/METADATA.json") | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #Get Json | |
| curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" -o "/tmp/METADATA.json" | |
| #Sanity Check | |
| if [ -s "/tmp/METADATA.json" ] && jq '.[].Repo // empty | if length == 0 then . / 0 else . end' "/tmp/METADATA.json" >/dev/null 2>&1; then | |
| # #This is unreliable | |
| # jq -c '.[] | select(.Name | test("\\.7z$|BLAKE3SUM|\\.txt$|\\.json$|\\.log$|\\.md$|SHA256SUM") | not) | select(has("Repo")) | select(.Repo != null) | select(.Repo | test("^https://github.com"))' "/tmp/METADATA.json" | while IFS= read -r obj; do | |
| #Instead use readarray (Bash Builtin) & a for loop | |
| readarray -t objs < <(jq -c '.[] | select(.Name | test("\\.7z$|BLAKE3SUM|\\.txt$|\\.json$|\\.log$|\\.md$|SHA256SUM") | not) | select(has("Repo")) | select(.Repo != null) | select(.Repo | test("^https://github.com"))' "/tmp/METADATA.json") | |
| for obj in "${objs[@]}"; do | |
| #Extract the Name and Repo from the JSON object | |
| bin="$(jq -r '.Name' <<< "$obj" | sed 's/\s//g' | tr -d '[:space:]')" && export bin="$bin" | |
| repo="$(jq -r '.Repo' <<< "$obj" | sed 's|^https://github.com/||' | sed 's/\s//g' | tr -d '[:space:]')" && export repo="$repo" | |
| # Set up the environment variables and execute the one-liner | |
| echo -e "\n\n[+] Generating Meta for $bin ($repo)\n" | |
| BIN="$bin" REPO="$repo" SOURCE_BIN="https://bin.ajam.dev/x86_64_Linux" bash <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/.github/scripts/pkg_metafetcher_x86_64_bin.ajam.dev.sh") | |
| #unset vars | |
| unset bin repo | |
| done | |
| fi | |
| continue-on-error: true | |
| #----------------------------# | |
| #For non github-repos | |
| #----------------------------# | |
| #7z : A file archiver with a high compression ratio | |
| - name: Generate Meta (7z) | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| #Generate Basic Meta | |
| export SOURCE_BIN="https://bin.ajam.dev/x86_64_Linux" | |
| export BIN="7z" ; export DESCRIPTION="7-Zip is a file archiver with a high compression ratio." ; export AUTHOR="Igor Pavlov" ; export REPO_URL="https://sourceforge.net/projects/sevenzip/" ; export STARS="" ; export SOURCE_URL="https://bin.ajam.dev/x86_64_Linux/7z" ; export LANGUAGE="C++" ; export LICENSE="LGPL-2.1-or-later" ; export TOPICS='["File archiver", "Compression", "tar gz 7z zip rar xz bzip2"]' | |
| PKG_METADATA="$(curl -qfsSL "$SOURCE_BIN/METADATA.json" 2>/dev/null | jq --arg bin "$BIN" '.[] | select(.Name == $bin)')" && export PKG_METADATA="$PKG_METADATA" | |
| #Generate Dynamic Meta | |
| BSUM="$(curl -qfsSL "$SOURCE_BIN/BLAKE3SUM.txt" 2>/dev/null | grep -i "$BIN$" | awk '{print $1}' | sort -u | head -n 1 | sed 's/"//g' | sed 's/^[ \t]*//;s/[ \t]*$//')" && export BSUM="$BSUM" | |
| SHA="$(curl -qfsSL "$SOURCE_BIN/SHA256SUM.txt" 2>/dev/null | grep -i "$BIN$" | awk '{print $1}' | sort -u | head -n 1 | sed 's/"//g' | sed 's/^[ \t]*//;s/[ \t]*$//')" && export SHA="$SHA" | |
| SIZE="$(echo $PKG_METADATA | jq -r '.Size' | sed 's/"//g' | sed 's/^[ \t]*//;s/[ \t]*$//')" && export SIZE="$SIZE" | |
| PKG_VERSION="$(curl -qfsSL "https://www.7-zip.org/download.html" | grep -oP '<B>Download 7-Zip \K[^<]*' | grep -iv 'windows' | sort -u | tail -n 1 | awk '{print $1}')" && export PKG_VERSION="$PKG_VERSION" | |
| PKG_RELEASED="$(curl -qfsSL "https://www.7-zip.org/download.html" | grep -oP '<B>Download 7-Zip \K[^<]*' | grep -iv 'windows' | sort -u | tail -n 1 | awk '{print $2}')" && export PKG_RELEASED="$PKG_RELEASED" | |
| #Update | |
| bash <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/.github/scripts/pkg_metaupdater_x86_64.sh") | |
| continue-on-error: true | |
| #------------------------------------------------------------------------------------# | |
| - name: Dos2Unix Everything | |
| run: | | |
| cd "$GITHUB_WORKSPACE/main/data/x86_64/" | |
| find . -type f -exec dos2unix {} \; | |
| continue-on-error: true | |
| - name: Git Pull | |
| run: | | |
| cd "$GITHUB_WORKSPACE/main" && git pull origin main | |
| continue-on-error: true | |
| - name: Validate TOML | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main/data" | |
| ## Just list | |
| #find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec yq --input-format "toml" --output-format "yaml" {} 1>/dev/null \; | |
| # List & Delete | |
| find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec sh -c 'yq --input-format "toml" --output-format "yaml" "$1" 1>/dev/null || rm "$1"' _ {} \; | |
| continue-on-error: true | |
| - name: Validate source | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main/data" | |
| ## Just list | |
| find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec yq --input-format "toml" --output-format "json" {} 2>/dev/null \; | jq -r 'select(.package.source == "") | .package.architecture + " ❯ " + .package.name + " ✗ " + .package.source' | sort -u | |
| echo -e "\n----------------------------\n" | |
| find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec sh -c 'yq --input-format "toml" --output-format "json" "$1" 2>/dev/null | jq -r --arg workspace "$GITHUB_WORKSPACE" '\''select(.package.source == "") | $workspace + "/main/data/" + .package.architecture + "/" + .package.name + ".toml"'\'' ' _ {} \; | |
| #Delete | |
| echo -e "\n----------------------------\n" | |
| find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec sh -c 'yq --input-format "toml" --output-format "json" "$1" 2>/dev/null | jq -r --arg workspace "$GITHUB_WORKSPACE" '\''select(.package.source == "") | $workspace + "/main/data/" + .package.architecture + "/" + .package.name + ".toml"'\'' ' _ {} \; | xargs rm 2>/dev/null | |
| continue-on-error: true | |
| - name: Update Available Index (JSON || TOML) | |
| run: | | |
| #Presets | |
| set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #Update $GITHUB_WORKSPACE/main/data/metadata.toml | |
| ##arm64_v8a_Android | |
| awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/arm64_v8a_Android/*.toml | tee "$GITHUB_WORKSPACE/main/data/metadata.toml" | |
| ##aarch64_arm64 | |
| awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/aarch64_arm64/*.toml | tee -a "$GITHUB_WORKSPACE/main/data/metadata.toml" | |
| ##x86_64 | |
| awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/x86_64/*.toml | tee -a "$GITHUB_WORKSPACE/main/data/metadata.toml" | |
| ##x64_Windows | |
| awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/x64_Windows/*.toml | tee -a "$GITHUB_WORKSPACE/main/data/metadata.toml" | |
| #Update $GITHUB_WORKSPACE/main/data/metadata.json | |
| cat "$GITHUB_WORKSPACE/main/data/metadata.toml" | yj -tj | jq . | tee "$GITHUB_WORKSPACE/main/data/metadata.json" | |
| continue-on-error: true | |
| - name: Generate Markdown (No Table --> ./data/x86_64.md) | |
| run: | | |
| #Presets | |
| set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #Generate $GITHUB_WORKSPACE/main/data/x86_64/README.md | |
| cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x86_64") | "### \(.name)\n- **Description:** \(.description)\n- **Author:** \(.author)\n- **Repository:** [\(.repo)](\(.repo))\n- **Stars:** \(.stars)\n- **Version:** \(.version)\n- **Updated:** \(.updated)\n- **Size:** \(.size)\n- **SHA256SUM:** \(.sha)\n- **B3SUM:** \(.bsum)\n- **Source:** \(.source)\n- **Language:** \(.language)\n- **License:** \(.license)\n\n---\n"' > "$GITHUB_WORKSPACE/main/data/x86_64.md" | |
| continue-on-error: true | |
| - name: Generate Markdown (Table --> ./data/x86_64/README.md) | |
| run: | | |
| #Presets | |
| set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #Generate $GITHUB_WORKSPACE/main/data/x86_64/README.md | |
| { echo "| Name | Description | Author | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat $GITHUB_WORKSPACE/main/data/metadata.json | jq -r '.packages[] | select(.architecture == "x86_64") | "| \(.name) | \(.description) | \(.author) | [\(.repo)](\(.repo)) | \(.stars) | \(.version) | \(.updated) | \(.size) | \(.sha) | \(.bsum) | \(.source) | \(.language) | \(.license) |"' ; } | tee "$GITHUB_WORKSPACE/main/data/x86_64/README.md" | |
| continue-on-error: true | |
| - name: Generate Markdown ( Main README.md) | |
| run: | | |
| #Presets | |
| set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #Generate $GITHUB_WORKSPACE/main/README.md | |
| cat "$GITHUB_WORKSPACE/main/INFO.md" > "$GITHUB_WORKSPACE/main/README.md" | |
| #aarch64_arm64 | |
| unset TOTAL_PACKAGES | |
| export TOTAL_PACKAGES="$(cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "aarch64_arm64")' | jq -r '.name' | sort -u | wc -l)" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
| sed "s/PLACEHOLDER_aarch64_arm64/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
| #x86_64 | |
| unset TOTAL_PACKAGES | |
| export TOTAL_PACKAGES="$(cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x86_64")' | jq -r '.name' | sort -u | wc -l)" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
| sed "s/PLACEHOLDER_X86_64/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
| #x64_Windows | |
| unset TOTAL_PACKAGES | |
| export TOTAL_PACKAGES="$(cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x64_Windows")' | jq -r '.name' | sort -u | wc -l)" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
| sed "s/PLACEHOLDER_x64_Windows/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
| continue-on-error: true | |
| - name: Generate Diff (Delta) | |
| run: | | |
| #Presets | |
| set +x ; set +e | |
| #--------------# | |
| cd "$GITHUB_WORKSPACE/main" | |
| #aarch64_arm64 | |
| echo -e "[+] aarch64_arm64\n" > "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | |
| delta -s --raw <(curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/METADATA.json" | jq -r '.[].Name' | grep -v "\.7z$\|\.bz2$\|\.json$\|\.gz$\|\.md$\|\.txt$\|\.tar$\|\.zip$" | sort -u) <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "aarch64_arm64")' | jq -r '.name' | sort -u) | ansi2txt | grep -iv "^/proc" | grep '\|' | grep -iv '@\|BLAKE3SUM\|README.md\|SHA256SUM' | grep -iv '/dev/fd' | tee -a "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | |
| #x86_64 | |
| echo -e "\n\n[+] x86_64\n" >> "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | |
| delta -s --raw <(curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" | jq -r '.[].Name' | grep -v "\.7z$\|\.bz2$\|\.json$\|\.gz$\|\.md$\|\.txt$\|\.tar$\|\.zip$" | sort -u) <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x86_64")' | jq -r '.name' | sort -u) | ansi2txt | grep -iv "^/proc" | grep '\|' | grep -iv '@\|BLAKE3SUM\|README.md\|SHA256SUM' | grep -iv '/dev/fd' | tee -a "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | |
| #Only Missing | |
| #aarch64_arm64 | |
| cat "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | grep -A 9999999999999 "aarch64_arm64" 2>/dev/null | awk '/x86_64/{exit} {print}' 2>/dev/null | grep -E "^\s*\│.*│\s*$" | awk -F'│' '{if($2 ~ /^ *$/ || $4 ~ /^ *$/) print}' | tee "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF_aarch64_arm64.txt" | |
| #x86_64 | |
| cat "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | grep -A 9999999999999 "x86_64" 2>/dev/null | grep -E "^\s*\│.*│\s*$" | awk -F'│' '{if($2 ~ /^ *$/ || $4 ~ /^ *$/) print}' | tee "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF_x86_64.txt" | |
| #x64_Windows | |
| cat "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | grep -A 9999999999999 "x64_Windows" 2>/dev/null | grep -E "^\s*\│.*│\s*$" | awk -F'│' '{if($2 ~ /^ *$/ || $4 ~ /^ *$/) print}' | tee "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF_x64_Windows.txt" | |
| continue-on-error: true | |
| - name: Get DateTime | |
| run: | | |
| # Date Time | |
| NEPALI_TIME=$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)') | |
| echo "NEPALI_TIME=$NEPALI_TIME" >> $GITHUB_ENV | |
| continue-on-error: true | |
| - uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| repository: ./main | |
| commit_user_name: Azathothas # defaults to "github-actions[bot]" | |
| commit_user_email: AjamX101@gmail.com # defaults to "41898282+github-actions[bot]@users.noreply.github.com" | |
| commit_message: "✅ 🛍️ (x86_64) Generated | Updated ⚙️ Daily Package 📦 Metadata 🗄️ <-- ${{ env.NEPALI_TIME }} ⌚" | |
| #push_options: '--force' | |
| #------------------------------------------------------------------------------------# |