/
generate_fqdn_v2.sh
87 lines (66 loc) 路 2.01 KB
/
generate_fqdn_v2.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
#!/bin/bash
echo "Setup script"
# Detect package manager
if command -v apt-get &>/dev/null; then
PACKAGE_MANAGER="apt-get"
UPDATE_CMD="sudo apt-get update"
INSTALL_CMD="sudo apt-get install -y"
elif command -v apk &>/dev/null; then
PACKAGE_MANAGER="apk"
UPDATE_CMD="sudo apk update"
INSTALL_CMD="sudo apk add --no-cache"
else
echo "Unsupported package manager. Exiting."
exit 1
fi
# Update and install prerequisites
$UPDATE_CMD
$INSTALL_CMD python3
# Link python3 to python (for Ubuntu, since Alpine doesn't have python2 by default)
if [ "$PACKAGE_MANAGER" == "apt-get" ]; then
sudo ln -s /usr/bin/python3 /usr/bin/python
fi
python3 -m ensurepip --upgrade
pip3 install --no-cache-dir --upgrade pip setuptools tldextract tqdm
# Install pv and ncftp based on the detected package manager
for package in pv ncftp; do
if ! $INSTALL_CMD $package; then
echo "Failed to install '$package' using $PACKAGE_MANAGER."
exit 1
fi
done
LISTS="blacklists.fqdn.urls"
# Function to download a URL
download_url() {
local url="$1"
echo "Blacklist: $url"
random_filename=$(uuidgen | tr -dc '[:alnum:]')
if ! wget -q --progress=bar:force -O "$random_filename.fqdn.list" "$url"; then
echo "Failed to download: $url"
fi
}
echo "Download blacklists"
# Download URLs from the list
while IFS= read -r url; do
download_url "$url"
done < "$LISTS"
FILES=$(ls *.fqdn.list)
echo "Aggregate blacklists"
echo "">aggregated.fqdn.list
while IFS= read -r file; do
sudo cat "$file" >> aggregated.fqdn.list
done <<< "$FILES"
sudo cat aggregated.fqdn.list | sort -u > all.fqdn.blacklist
echo "Remove source files"
sudo rm ./*.fqdn.list
echo "Sanitize blacklists"
mv all.fqdn.blacklist input.txt
python sanitize.py
mv output.txt all.fqdn.blacklist
echo "Remove whitelisted domains"
mv all.fqdn.blacklist blacklist.txt
python whitelist.py
mv filtered_blacklist.txt all.fqdn.blacklist
rm blacklist.txt input.txt
total_lines_new=$(cat all.fqdn.blacklist | wc -l)
echo "Total domains: $total_lines_new."