pkg_metafetcher_metaupdater_x86_64.yaml

name: 🛍️ (x86_64) Generate | Update ⚙️ Daily Package 📦 Metadata 🗄️
#MAX_RUNTIME:  
on:
  workflow_dispatch:
  schedule:
    - cron: "0 0-23/2 * * *" #  Every Even hrs
  
env:
  GITHUB_TOKEN: "${{ secrets.HYSP_TOKEN }}"
  
jobs:
#------------------------------------------------------------------------------------#
#------------------------------------------------------------------------------------#
  fetch-meta:
    runs-on: ubuntu-latest
    
    permissions:
      contents: write
      
    steps:        

      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          path: main
          filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/

      - name: Install CoreUtils & Deps
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          sudo apt-get update -y
          sudo apt-get install coreutils ca-certificates curl git dos2unix lzip jq moreutils p7zip-full util-linux wget -y
          pip install ansi2txt --upgrade
          #Re
          sudo apt-get install coreutils ca-certificates curl git dos2unix lzip jq moreutils p7zip-full util-linux wget -y
          pip install ansi2txt --upgrade
        continue-on-error: true
        
      - name: Install Addons
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          #action-lint
          sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/actionlint" -o "/usr/local/bin/actionlint" && sudo chmod +xwr "/usr/local/bin/actionlint"
          #delta
          sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/delta" -o "/usr/local/bin/delta" && sudo chmod +xwr "/usr/local/bin/delta"
          #eget
          sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/eget" -o "/usr/local/bin/eget" && sudo chmod +xwr "/usr/local/bin/eget"
          #validtoml
          sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/validtoml" -o "/usr/local/bin/validtoml" && sudo chmod +xwr "/usr/local/bin/validtoml"
          #Yq
          sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/yq" -o "/usr/local/bin/yq" && sudo chmod +xwr "/usr/local/bin/yq"
          #Yj
          sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/yj" -o "/usr/local/bin/yj" && sudo chmod +xwr "/usr/local/bin/yj"
        continue-on-error: true

      - name: Setup Env
        run: |
          # Create Output Dir
          mkdir -p "$GITHUB_WORKSPACE/main/data/x86_64"
        continue-on-error: true
#------------------------------------------------------------------------------------#
     #----------------------------#
     #For every Github Repo from https://bin.ajam.dev/x86_64_Linux/METADATA.json
      - name: Generate Meta ("bin.ajam.dev/x86_64_Linux/METADATA.json")
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          #Get Json
          curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" -o "/tmp/METADATA.json"
          #Sanity Check
          if [ -s "/tmp/METADATA.json" ] && jq '.[].Repo // empty | if length == 0 then . / 0 else . end' "/tmp/METADATA.json" >/dev/null 2>&1; then
            # #This is unreliable
            # jq -c '.[] | select(.Name | test("\\.7z$|BLAKE3SUM|\\.txt$|\\.json$|\\.log$|\\.md$|SHA256SUM") | not) | select(has("Repo")) | select(.Repo != null) | select(.Repo | test("^https://github.com"))' "/tmp/METADATA.json" | while IFS= read -r obj; do
            #Instead use readarray (Bash Builtin) & a for loop
            readarray -t objs < <(jq -c '.[] | select(.Name | test("\\.7z$|BLAKE3SUM|\\.txt$|\\.json$|\\.log$|\\.md$|SHA256SUM") | not) | select(has("Repo")) | select(.Repo != null) | select(.Repo | test("^https://github.com"))' "/tmp/METADATA.json")
            for obj in "${objs[@]}"; do
                #Extract the Name and Repo from the JSON object
                bin="$(jq -r '.Name' <<< "$obj" | sed 's/\s//g' | tr -d '[:space:]')" && export bin="$bin"
                repo="$(jq -r '.Repo' <<< "$obj" | sed 's|^https://github.com/||' | sed 's/\s//g' | tr -d '[:space:]')" && export repo="$repo"
                # Set up the environment variables and execute the one-liner
                echo -e "\n\n[+] Generating Meta for $bin ($repo)\n"
                BIN="$bin" REPO="$repo" SOURCE_BIN="https://bin.ajam.dev/x86_64_Linux" bash <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/.github/scripts/pkg_metafetcher_x86_64_bin.ajam.dev.sh")
                #unset vars
                unset bin repo
            done
          fi
        continue-on-error: true
#----------------------------#
#For non github-repos
     #----------------------------#
     #7z : A file archiver with a high compression ratio
      - name: Generate Meta (7z)
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          #Generate Basic Meta
          export SOURCE_BIN="https://bin.ajam.dev/x86_64_Linux"
          export BIN="7z" ; export DESCRIPTION="7-Zip is a file archiver with a high compression ratio." ;  export AUTHOR="Igor Pavlov" ; export REPO_URL="https://sourceforge.net/projects/sevenzip/" ; export STARS="" ; export SOURCE_URL="https://bin.ajam.dev/x86_64_Linux/7z" ; export LANGUAGE="C++" ; export LICENSE="LGPL-2.1-or-later" ; export TOPICS='["File archiver", "Compression", "tar gz 7z zip rar xz bzip2"]'
          PKG_METADATA="$(curl -qfsSL "$SOURCE_BIN/METADATA.json" 2>/dev/null | jq --arg bin "$BIN" '.[] | select(.Name == $bin)')" && export PKG_METADATA="$PKG_METADATA" 
          #Generate Dynamic Meta
          BSUM="$(curl -qfsSL "$SOURCE_BIN/BLAKE3SUM.txt" 2>/dev/null | grep -i "$BIN$" | awk '{print $1}' | sort  -u | head -n 1 | sed 's/"//g' | sed 's/^[ \t]*//;s/[ \t]*$//')" && export BSUM="$BSUM"
          SHA="$(curl -qfsSL "$SOURCE_BIN/SHA256SUM.txt" 2>/dev/null | grep -i "$BIN$" | awk '{print $1}' | sort  -u | head -n 1 | sed 's/"//g' | sed 's/^[ \t]*//;s/[ \t]*$//')" && export SHA="$SHA"
          SIZE="$(echo $PKG_METADATA | jq -r '.Size' | sed 's/"//g' | sed 's/^[ \t]*//;s/[ \t]*$//')" && export SIZE="$SIZE"
          PKG_VERSION="$(curl -qfsSL "https://www.7-zip.org/download.html" | grep -oP '<B>Download 7-Zip \K[^<]*' | grep -iv 'windows' | sort -u | tail -n 1 |  awk '{print $1}')" && export PKG_VERSION="$PKG_VERSION" 
          PKG_RELEASED="$(curl -qfsSL "https://www.7-zip.org/download.html" | grep -oP '<B>Download 7-Zip \K[^<]*' | grep -iv 'windows' | sort -u | tail -n 1 |  awk '{print $2}')" && export PKG_RELEASED="$PKG_RELEASED" 
          #Update
          bash <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/.github/scripts/pkg_metaupdater_x86_64.sh")
        continue-on-error: true
#------------------------------------------------------------------------------------#        
      - name: Dos2Unix Everything
        run: |
          cd "$GITHUB_WORKSPACE/main/data/x86_64/"
          find . -type f -exec dos2unix {} \;
        continue-on-error: true

      - name: Git Pull
        run: |
           cd "$GITHUB_WORKSPACE/main" && git pull origin main
        continue-on-error: true

      - name: Validate TOML
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          cd "$GITHUB_WORKSPACE/main/data"
          ## Just list
          #find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec yq --input-format "toml" --output-format "yaml" {} 1>/dev/null \;
          # List & Delete
          find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec sh -c 'yq --input-format "toml" --output-format "yaml" "$1" 1>/dev/null || rm "$1"' _ {} \;
        continue-on-error: true

      - name: Validate source
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          cd "$GITHUB_WORKSPACE/main/data"
          ## Just list
          find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec yq --input-format "toml" --output-format "json" {} 2>/dev/null \; | jq -r 'select(.package.source == "") | .package.architecture + " ❯ " + .package.name + " ✗ " + .package.source' | sort -u
          echo -e "\n----------------------------\n"
          find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec sh -c 'yq --input-format "toml" --output-format "json" "$1" 2>/dev/null | jq -r --arg workspace "$GITHUB_WORKSPACE" '\''select(.package.source == "") | $workspace + "/main/data/" + .package.architecture + "/" + .package.name + ".toml"'\'' ' _ {} \;
          #Delete
          echo -e "\n----------------------------\n"
          find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec sh -c 'yq --input-format "toml" --output-format "json" "$1" 2>/dev/null | jq -r --arg workspace "$GITHUB_WORKSPACE" '\''select(.package.source == "") | $workspace + "/main/data/" + .package.architecture + "/" + .package.name + ".toml"'\'' ' _ {} \; | xargs rm 2>/dev/null
        continue-on-error: true

      - name: Update Available Index (JSON || TOML)
        run: |
           #Presets
           set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null
           #--------------#
           cd "$GITHUB_WORKSPACE/main"
           #Update $GITHUB_WORKSPACE/main/data/metadata.toml
           ##arm64_v8a_Android
           awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/arm64_v8a_Android/*.toml | tee "$GITHUB_WORKSPACE/main/data/metadata.toml"
           ##aarch64_arm64
           awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/aarch64_arm64/*.toml | tee -a "$GITHUB_WORKSPACE/main/data/metadata.toml"
           ##x86_64
           awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/x86_64/*.toml | tee -a "$GITHUB_WORKSPACE/main/data/metadata.toml"
           ##x64_Windows
           awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/x64_Windows/*.toml | tee -a "$GITHUB_WORKSPACE/main/data/metadata.toml"
           #Update $GITHUB_WORKSPACE/main/data/metadata.json
           cat "$GITHUB_WORKSPACE/main/data/metadata.toml" | yj -tj | jq . | tee "$GITHUB_WORKSPACE/main/data/metadata.json"
        continue-on-error: true
        
      - name: Generate Markdown (No Table --> ./data/x86_64.md)
        run: |
           #Presets
           set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null
           #--------------#
           cd "$GITHUB_WORKSPACE/main"
           #Generate $GITHUB_WORKSPACE/main/data/x86_64/README.md
           cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x86_64") | "### \(.name)\n- **Description:** \(.description)\n- **Author:** \(.author)\n- **Repository:** [\(.repo)](\(.repo))\n- **Stars:** \(.stars)\n- **Version:** \(.version)\n- **Updated:** \(.updated)\n- **Size:** \(.size)\n- **SHA256SUM:** \(.sha)\n- **B3SUM:** \(.bsum)\n- **Source:** \(.source)\n- **Language:** \(.language)\n- **License:** \(.license)\n\n---\n"' > "$GITHUB_WORKSPACE/main/data/x86_64.md"
        continue-on-error: true
        
      - name: Generate Markdown (Table --> ./data/x86_64/README.md)
        run: |
           #Presets
           set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null
           #--------------#
           cd "$GITHUB_WORKSPACE/main"
           #Generate $GITHUB_WORKSPACE/main/data/x86_64/README.md
           { echo "| Name | Description | Author | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat $GITHUB_WORKSPACE/main/data/metadata.json | jq -r '.packages[] | select(.architecture == "x86_64") | "| \(.name) | \(.description) | \(.author) | [\(.repo)](\(.repo)) | \(.stars) | \(.version) | \(.updated) | \(.size) | \(.sha) | \(.bsum) | \(.source) | \(.language) | \(.license) |"' ; } | tee "$GITHUB_WORKSPACE/main/data/x86_64/README.md"
        continue-on-error: true

      - name: Generate Markdown ( Main README.md)
        run: |
           #Presets
           set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null
           #--------------#
           cd "$GITHUB_WORKSPACE/main"
           #Generate $GITHUB_WORKSPACE/main/README.md
           cat "$GITHUB_WORKSPACE/main/INFO.md" > "$GITHUB_WORKSPACE/main/README.md"
           #aarch64_arm64
           unset TOTAL_PACKAGES
           export TOTAL_PACKAGES="$(cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "aarch64_arm64")' | jq -r '.name' | sort -u | wc -l)" && export TOTAL_PACKAGES="$TOTAL_PACKAGES"
           sed "s/PLACEHOLDER_aarch64_arm64/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md"
           #x86_64
           unset TOTAL_PACKAGES
           export TOTAL_PACKAGES="$(cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x86_64")' | jq -r '.name' | sort -u | wc -l)" && export TOTAL_PACKAGES="$TOTAL_PACKAGES"
           sed "s/PLACEHOLDER_X86_64/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md"
           #x64_Windows
           unset TOTAL_PACKAGES
           export TOTAL_PACKAGES="$(cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x64_Windows")' | jq -r '.name' | sort -u | wc -l)" && export TOTAL_PACKAGES="$TOTAL_PACKAGES"
           sed "s/PLACEHOLDER_x64_Windows/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md"
        continue-on-error: true

      - name: Generate Diff (Delta)
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          cd "$GITHUB_WORKSPACE/main"          
          #aarch64_arm64
          echo -e "[+] aarch64_arm64\n" > "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt"
          delta -s --raw <(curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/METADATA.json" | jq -r '.[].Name' | grep -v "\.7z$\|\.bz2$\|\.json$\|\.gz$\|\.md$\|\.txt$\|\.tar$\|\.zip$" | sort -u) <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "aarch64_arm64")' | jq -r '.name' | sort -u) | ansi2txt | grep -iv "^/proc" | grep '\|' | grep -iv '@\|BLAKE3SUM\|README.md\|SHA256SUM' | grep -iv '/dev/fd' | tee -a "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt"
          #x86_64
          echo -e "\n\n[+] x86_64\n" >> "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt"
          delta -s --raw <(curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" | jq -r '.[].Name' | grep -v "\.7z$\|\.bz2$\|\.json$\|\.gz$\|\.md$\|\.txt$\|\.tar$\|\.zip$" | sort -u) <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x86_64")' | jq -r '.name' | sort -u) | ansi2txt | grep -iv "^/proc" | grep '\|' | grep -iv '@\|BLAKE3SUM\|README.md\|SHA256SUM' | grep -iv '/dev/fd' | tee -a "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt"
          #Only Missing
          #aarch64_arm64
          cat "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | grep -A 9999999999999 "aarch64_arm64" 2>/dev/null | awk '/x86_64/{exit} {print}' 2>/dev/null | grep -E "^\s*\│.*│\s*$" | awk -F'│' '{if($2 ~ /^ *$/ || $4 ~ /^ *$/) print}' | tee "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF_aarch64_arm64.txt"
          #x86_64
          cat "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | grep -A 9999999999999 "x86_64" 2>/dev/null | grep -E "^\s*\│.*│\s*$" | awk -F'│' '{if($2 ~ /^ *$/ || $4 ~ /^ *$/) print}' | tee "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF_x86_64.txt"
          #x64_Windows
          cat "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | grep -A 9999999999999 "x64_Windows" 2>/dev/null | grep -E "^\s*\│.*│\s*$" | awk -F'│' '{if($2 ~ /^ *$/ || $4 ~ /^ *$/) print}' | tee "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF_x64_Windows.txt"
        continue-on-error: true

      - name: Get DateTime
        run: |
          # Date Time
          NEPALI_TIME=$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)')
          echo "NEPALI_TIME=$NEPALI_TIME" >> $GITHUB_ENV
        continue-on-error: true
                  
      - uses: stefanzweifel/git-auto-commit-action@v5
        with:
          repository: ./main        
          commit_user_name: Azathothas # defaults to "github-actions[bot]"
          commit_user_email: AjamX101@gmail.com # defaults to "41898282+github-actions[bot]@users.noreply.github.com"
          commit_message: "✅ 🛍️ (x86_64) Generated | Updated ⚙️ Daily Package 📦 Metadata 🗄️ <-- ${{ env.NEPALI_TIME }} ⌚"
          #push_options: '--force'
#------------------------------------------------------------------------------------#