-
-
Notifications
You must be signed in to change notification settings - Fork 2
250 lines (232 loc) · 17 KB
/
pkg_metafetcher_metaupdater_x86_64.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
name: 🛍️ (x86_64) Generate | Update ⚙️ Daily Package 📦 Metadata 🗄️
#MAX_RUNTIME:
on:
workflow_dispatch:
schedule:
- cron: "0 0-23/2 * * *" # Every Even hrs
env:
GITHUB_TOKEN: "${{ secrets.HYSP_TOKEN }}"
jobs:
#------------------------------------------------------------------------------------#
#------------------------------------------------------------------------------------#
fetch-meta:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
path: main
filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/
- name: Install CoreUtils & Deps
run: |
#Presets
set +x ; set +e
#--------------#
sudo apt-get update -y
sudo apt-get install coreutils ca-certificates curl git dos2unix lzip jq moreutils p7zip-full util-linux wget -y
pip install ansi2txt --upgrade
#Re
sudo apt-get install coreutils ca-certificates curl git dos2unix lzip jq moreutils p7zip-full util-linux wget -y
pip install ansi2txt --upgrade
continue-on-error: true
- name: Install Addons
run: |
#Presets
set +x ; set +e
#--------------#
#action-lint
sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/actionlint" -o "/usr/local/bin/actionlint" && sudo chmod +xwr "/usr/local/bin/actionlint"
#delta
sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/delta" -o "/usr/local/bin/delta" && sudo chmod +xwr "/usr/local/bin/delta"
#eget
sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/eget" -o "/usr/local/bin/eget" && sudo chmod +xwr "/usr/local/bin/eget"
#validtoml
sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/validtoml" -o "/usr/local/bin/validtoml" && sudo chmod +xwr "/usr/local/bin/validtoml"
#Yq
sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/yq" -o "/usr/local/bin/yq" && sudo chmod +xwr "/usr/local/bin/yq"
#Yj
sudo curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/yj" -o "/usr/local/bin/yj" && sudo chmod +xwr "/usr/local/bin/yj"
continue-on-error: true
- name: Setup Env
run: |
# Create Output Dir
mkdir -p "$GITHUB_WORKSPACE/main/data/x86_64"
continue-on-error: true
#------------------------------------------------------------------------------------#
#----------------------------#
#For every Github Repo from https://bin.ajam.dev/x86_64_Linux/METADATA.json
- name: Generate Meta ("bin.ajam.dev/x86_64_Linux/METADATA.json")
run: |
#Presets
set +x ; set +e
#--------------#
#Get Json
curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" -o "/tmp/METADATA.json"
#Sanity Check
if [ -s "/tmp/METADATA.json" ] && jq '.[].Repo // empty | if length == 0 then . / 0 else . end' "/tmp/METADATA.json" >/dev/null 2>&1; then
# #This is unreliable
# jq -c '.[] | select(.Name | test("\\.7z$|BLAKE3SUM|\\.txt$|\\.json$|\\.log$|\\.md$|SHA256SUM") | not) | select(has("Repo")) | select(.Repo != null) | select(.Repo | test("^https://github.com"))' "/tmp/METADATA.json" | while IFS= read -r obj; do
#Instead use readarray (Bash Builtin) & a for loop
readarray -t objs < <(jq -c '.[] | select(.Name | test("\\.7z$|BLAKE3SUM|\\.txt$|\\.json$|\\.log$|\\.md$|SHA256SUM") | not) | select(has("Repo")) | select(.Repo != null) | select(.Repo | test("^https://github.com"))' "/tmp/METADATA.json")
for obj in "${objs[@]}"; do
#Extract the Name and Repo from the JSON object
bin="$(jq -r '.Name' <<< "$obj" | sed 's/\s//g' | tr -d '[:space:]')" && export bin="$bin"
repo="$(jq -r '.Repo' <<< "$obj" | sed 's|^https://github.com/||' | sed 's/\s//g' | tr -d '[:space:]')" && export repo="$repo"
# Set up the environment variables and execute the one-liner
echo -e "\n\n[+] Generating Meta for $bin ($repo)\n"
BIN="$bin" REPO="$repo" SOURCE_BIN="https://bin.ajam.dev/x86_64_Linux" bash <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/.github/scripts/pkg_metafetcher_x86_64_bin.ajam.dev.sh")
#unset vars
unset bin repo
done
fi
continue-on-error: true
#----------------------------#
#For non github-repos
#----------------------------#
#7z : A file archiver with a high compression ratio
- name: Generate Meta (7z)
run: |
#Presets
set +x ; set +e
#--------------#
#Generate Basic Meta
export SOURCE_BIN="https://bin.ajam.dev/x86_64_Linux"
export BIN="7z" ; export DESCRIPTION="7-Zip is a file archiver with a high compression ratio." ; export AUTHOR="Igor Pavlov" ; export REPO_URL="https://sourceforge.net/projects/sevenzip/" ; export STARS="" ; export SOURCE_URL="https://bin.ajam.dev/x86_64_Linux/7z" ; export LANGUAGE="C++" ; export LICENSE="LGPL-2.1-or-later" ; export TOPICS='["File archiver", "Compression", "tar gz 7z zip rar xz bzip2"]'
PKG_METADATA="$(curl -qfsSL "$SOURCE_BIN/METADATA.json" 2>/dev/null | jq --arg bin "$BIN" '.[] | select(.Name == $bin)')" && export PKG_METADATA="$PKG_METADATA"
#Generate Dynamic Meta
BSUM="$(curl -qfsSL "$SOURCE_BIN/BLAKE3SUM.txt" 2>/dev/null | grep -i "$BIN$" | awk '{print $1}' | sort -u | head -n 1 | sed 's/"//g' | sed 's/^[ \t]*//;s/[ \t]*$//')" && export BSUM="$BSUM"
SHA="$(curl -qfsSL "$SOURCE_BIN/SHA256SUM.txt" 2>/dev/null | grep -i "$BIN$" | awk '{print $1}' | sort -u | head -n 1 | sed 's/"//g' | sed 's/^[ \t]*//;s/[ \t]*$//')" && export SHA="$SHA"
SIZE="$(echo $PKG_METADATA | jq -r '.Size' | sed 's/"//g' | sed 's/^[ \t]*//;s/[ \t]*$//')" && export SIZE="$SIZE"
PKG_VERSION="$(curl -qfsSL "https://www.7-zip.org/download.html" | grep -oP '<B>Download 7-Zip \K[^<]*' | grep -iv 'windows' | sort -u | tail -n 1 | awk '{print $1}')" && export PKG_VERSION="$PKG_VERSION"
PKG_RELEASED="$(curl -qfsSL "https://www.7-zip.org/download.html" | grep -oP '<B>Download 7-Zip \K[^<]*' | grep -iv 'windows' | sort -u | tail -n 1 | awk '{print $2}')" && export PKG_RELEASED="$PKG_RELEASED"
#Update
bash <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/.github/scripts/pkg_metaupdater_x86_64.sh")
continue-on-error: true
#------------------------------------------------------------------------------------#
- name: Dos2Unix Everything
run: |
cd "$GITHUB_WORKSPACE/main/data/x86_64/"
find . -type f -exec dos2unix {} \;
continue-on-error: true
- name: Git Pull
run: |
cd "$GITHUB_WORKSPACE/main" && git pull origin main
continue-on-error: true
- name: Validate TOML
run: |
#Presets
set +x ; set +e
#--------------#
cd "$GITHUB_WORKSPACE/main/data"
## Just list
#find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec yq --input-format "toml" --output-format "yaml" {} 1>/dev/null \;
# List & Delete
find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec sh -c 'yq --input-format "toml" --output-format "yaml" "$1" 1>/dev/null || rm "$1"' _ {} \;
continue-on-error: true
- name: Validate source
run: |
#Presets
set +x ; set +e
#--------------#
cd "$GITHUB_WORKSPACE/main/data"
## Just list
find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec yq --input-format "toml" --output-format "json" {} 2>/dev/null \; | jq -r 'select(.package.source == "") | .package.architecture + " ❯ " + .package.name + " ✗ " + .package.source' | sort -u
echo -e "\n----------------------------\n"
find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec sh -c 'yq --input-format "toml" --output-format "json" "$1" 2>/dev/null | jq -r --arg workspace "$GITHUB_WORKSPACE" '\''select(.package.source == "") | $workspace + "/main/data/" + .package.architecture + "/" + .package.name + ".toml"'\'' ' _ {} \;
#Delete
echo -e "\n----------------------------\n"
find "$GITHUB_WORKSPACE/main/data" -type f -name '*.toml' ! -name '*SAMPLE_SPEC*' -exec sh -c 'yq --input-format "toml" --output-format "json" "$1" 2>/dev/null | jq -r --arg workspace "$GITHUB_WORKSPACE" '\''select(.package.source == "") | $workspace + "/main/data/" + .package.architecture + "/" + .package.name + ".toml"'\'' ' _ {} \; | xargs rm 2>/dev/null
continue-on-error: true
- name: Update Available Index (JSON || TOML)
run: |
#Presets
set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null
#--------------#
cd "$GITHUB_WORKSPACE/main"
#Update $GITHUB_WORKSPACE/main/data/metadata.toml
##arm64_v8a_Android
awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/arm64_v8a_Android/*.toml | tee "$GITHUB_WORKSPACE/main/data/metadata.toml"
##aarch64_arm64
awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/aarch64_arm64/*.toml | tee -a "$GITHUB_WORKSPACE/main/data/metadata.toml"
##x86_64
awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/x86_64/*.toml | tee -a "$GITHUB_WORKSPACE/main/data/metadata.toml"
##x64_Windows
awk '/^\[package\]/ {print "[[packages]]"; in_package = 1; next} /^\[package\.conditions\]/ {in_package = 0; next} in_package {print $0}' $GITHUB_WORKSPACE/main/data/x64_Windows/*.toml | tee -a "$GITHUB_WORKSPACE/main/data/metadata.toml"
#Update $GITHUB_WORKSPACE/main/data/metadata.json
cat "$GITHUB_WORKSPACE/main/data/metadata.toml" | yj -tj | jq . | tee "$GITHUB_WORKSPACE/main/data/metadata.json"
continue-on-error: true
- name: Generate Markdown (No Table --> ./data/x86_64.md)
run: |
#Presets
set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null
#--------------#
cd "$GITHUB_WORKSPACE/main"
#Generate $GITHUB_WORKSPACE/main/data/x86_64/README.md
cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x86_64") | "### \(.name)\n- **Description:** \(.description)\n- **Author:** \(.author)\n- **Repository:** [\(.repo)](\(.repo))\n- **Stars:** \(.stars)\n- **Version:** \(.version)\n- **Updated:** \(.updated)\n- **Size:** \(.size)\n- **SHA256SUM:** \(.sha)\n- **B3SUM:** \(.bsum)\n- **Source:** \(.source)\n- **Language:** \(.language)\n- **License:** \(.license)\n\n---\n"' > "$GITHUB_WORKSPACE/main/data/x86_64.md"
continue-on-error: true
- name: Generate Markdown (Table --> ./data/x86_64/README.md)
run: |
#Presets
set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null
#--------------#
cd "$GITHUB_WORKSPACE/main"
#Generate $GITHUB_WORKSPACE/main/data/x86_64/README.md
{ echo "| Name | Description | Author | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat $GITHUB_WORKSPACE/main/data/metadata.json | jq -r '.packages[] | select(.architecture == "x86_64") | "| \(.name) | \(.description) | \(.author) | [\(.repo)](\(.repo)) | \(.stars) | \(.version) | \(.updated) | \(.size) | \(.sha) | \(.bsum) | \(.source) | \(.language) | \(.license) |"' ; } | tee "$GITHUB_WORKSPACE/main/data/x86_64/README.md"
continue-on-error: true
- name: Generate Markdown ( Main README.md)
run: |
#Presets
set -x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null
#--------------#
cd "$GITHUB_WORKSPACE/main"
#Generate $GITHUB_WORKSPACE/main/README.md
cat "$GITHUB_WORKSPACE/main/INFO.md" > "$GITHUB_WORKSPACE/main/README.md"
#aarch64_arm64
unset TOTAL_PACKAGES
export TOTAL_PACKAGES="$(cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "aarch64_arm64")' | jq -r '.name' | sort -u | wc -l)" && export TOTAL_PACKAGES="$TOTAL_PACKAGES"
sed "s/PLACEHOLDER_aarch64_arm64/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md"
#x86_64
unset TOTAL_PACKAGES
export TOTAL_PACKAGES="$(cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x86_64")' | jq -r '.name' | sort -u | wc -l)" && export TOTAL_PACKAGES="$TOTAL_PACKAGES"
sed "s/PLACEHOLDER_X86_64/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md"
#x64_Windows
unset TOTAL_PACKAGES
export TOTAL_PACKAGES="$(cat "$GITHUB_WORKSPACE/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x64_Windows")' | jq -r '.name' | sort -u | wc -l)" && export TOTAL_PACKAGES="$TOTAL_PACKAGES"
sed "s/PLACEHOLDER_x64_Windows/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md"
continue-on-error: true
- name: Generate Diff (Delta)
run: |
#Presets
set +x ; set +e
#--------------#
cd "$GITHUB_WORKSPACE/main"
#aarch64_arm64
echo -e "[+] aarch64_arm64\n" > "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt"
delta -s --raw <(curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/METADATA.json" | jq -r '.[].Name' | grep -v "\.7z$\|\.bz2$\|\.json$\|\.gz$\|\.md$\|\.txt$\|\.tar$\|\.zip$" | sort -u) <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "aarch64_arm64")' | jq -r '.name' | sort -u) | ansi2txt | grep -iv "^/proc" | grep '\|' | grep -iv '@\|BLAKE3SUM\|README.md\|SHA256SUM' | grep -iv '/dev/fd' | tee -a "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt"
#x86_64
echo -e "\n\n[+] x86_64\n" >> "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt"
delta -s --raw <(curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" | jq -r '.[].Name' | grep -v "\.7z$\|\.bz2$\|\.json$\|\.gz$\|\.md$\|\.txt$\|\.tar$\|\.zip$" | sort -u) <(curl -qfsSL "https://raw.githubusercontent.com/metis-os/hysp-pkgs/main/data/metadata.json" | jq -r '.packages[] | select(.architecture == "x86_64")' | jq -r '.name' | sort -u) | ansi2txt | grep -iv "^/proc" | grep '\|' | grep -iv '@\|BLAKE3SUM\|README.md\|SHA256SUM' | grep -iv '/dev/fd' | tee -a "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt"
#Only Missing
#aarch64_arm64
cat "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | grep -A 9999999999999 "aarch64_arm64" 2>/dev/null | awk '/x86_64/{exit} {print}' 2>/dev/null | grep -E "^\s*\│.*│\s*$" | awk -F'│' '{if($2 ~ /^ *$/ || $4 ~ /^ *$/) print}' | tee "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF_aarch64_arm64.txt"
#x86_64
cat "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | grep -A 9999999999999 "x86_64" 2>/dev/null | grep -E "^\s*\│.*│\s*$" | awk -F'│' '{if($2 ~ /^ *$/ || $4 ~ /^ *$/) print}' | tee "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF_x86_64.txt"
#x64_Windows
cat "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF.txt" | grep -A 9999999999999 "x64_Windows" 2>/dev/null | grep -E "^\s*\│.*│\s*$" | awk -F'│' '{if($2 ~ /^ *$/ || $4 ~ /^ *$/) print}' | tee "$GITHUB_WORKSPACE/main/.github/PKGS_DIFF_x64_Windows.txt"
continue-on-error: true
- name: Get DateTime
run: |
# Date Time
NEPALI_TIME=$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)')
echo "NEPALI_TIME=$NEPALI_TIME" >> $GITHUB_ENV
continue-on-error: true
- uses: stefanzweifel/git-auto-commit-action@v5
with:
repository: ./main
commit_user_name: Azathothas # defaults to "github-actions[bot]"
commit_user_email: AjamX101@gmail.com # defaults to "41898282+github-actions[bot]@users.noreply.github.com"
commit_message: "✅ 🛍️ (x86_64) Generated | Updated ⚙️ Daily Package 📦 Metadata 🗄️ <-- ${{ env.NEPALI_TIME }} ⌚"
#push_options: '--force'
#------------------------------------------------------------------------------------#