From be934c77f029aa52c84ede976cf7ab91bf053c03 Mon Sep 17 00:00:00 2001 From: Michael Urspringer Date: Sun, 26 Jan 2020 11:46:07 +0100 Subject: [PATCH 001/101] Fixed wrong download URL for MalwarePatrol --- clamav-unofficial-sigs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index aa70db1d..c2f31d6f 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2979,7 +2979,7 @@ if [ "$malwarepatrol_enabled" == "yes" ] ; then xshok_pretty_echo_and_log "Checking for updated MalwarePatrol database file: ${malwarepatrol_db}" malwarepatrol_db_update="0" - xshok_file_download "${work_dir_malwarepatrol}/${malwarepatrol_db}" "${malwarepatrol_url}&receipt=${malwarepatrol_receipt_code}" + xshok_file_download "${work_dir_malwarepatrol}/${malwarepatrol_db}" "${malwarepatrol_url}" ret="$?" if [ "$ret" -eq 0 ] ; then From a3ec444b742ac5f805a3d8f09b10253a1f8970fa Mon Sep 17 00:00:00 2001 From: root Date: Tue, 28 Jan 2020 12:45:08 +0100 Subject: [PATCH 002/101] Add os.mailcleaner.conf --- config/os/os.mailcleaner.conf | 37 +++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 config/os/os.mailcleaner.conf diff --git a/config/os/os.mailcleaner.conf b/config/os/os.mailcleaner.conf new file mode 100644 index 00000000..5b6206ab --- /dev/null +++ b/config/os/os.mailcleaner.conf @@ -0,0 +1,37 @@ +# This file contains os configuration settings for clamav-unofficial-sigs.sh +################### +# This is property of eXtremeSHOK.com +# You are free to use, modify and distribute, however you may not remove this notice. +# Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com +# License: BSD (Berkeley Software Distribution) +################## +# +# Script updates can be found at: https://github.com/extremeshok/clamav-unofficial-sigs +# +################## +# +# NOT COMPATIBLE WITH VERSION 3.XX / 4.XX CONFIG +# +################################################################################ +# SEE MASTER.CONF FOR CONFIG EXPLANATIONS +################################################################################ +# Rename to os.conf to enable this file +################################################################################ + +# Mailcleaner - Debian 8 (Jessie) +PATH=$PATH:/opt/clamav/bin + +clam_user="clamav" +clam_group="clamav" + +logrotate_group="adm" + +clam_dbs="/var/mailcleaner/spool/clamspam" + +clamd_pid="/var/mailcleaner/run/clamav/clamd.pid" + +clamd_restart_opt="/etc/init.d/mailcleaner restart" + +#clamd_socket="/run/clamav/clamd.ctl" + +# https://eXtremeSHOK.com ###################################################### From eb8befbf622c46b9aa5fc644412f7821c1ed5362 Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Tue, 28 Jan 2020 09:48:37 -0800 Subject: [PATCH 003/101] Fixed URLhaus log message. --- clamav-unofficial-sigs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index aa70db1d..f9f6f3ec 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -3098,7 +3098,7 @@ if [ "$urlhaus_enabled" == "yes" ] ; then if [ "$time_interval" -ge "$((update_interval - 600))" ] ; then echo "$current_time" > "${work_dir_work_configs}/last-urlhaus-update.txt" - xshok_pretty_echo_and_log "Yara-Rules Database File Updates" "=" + xshok_pretty_echo_and_log "URLhaus Database File Updates" "=" xshok_pretty_echo_and_log "Checking for urlhaus updates..." urlhaus_updates="0" for db_file in "${urlhaus_dbs[@]}" ; do @@ -3185,7 +3185,7 @@ if [ "$urlhaus_enabled" == "yes" ] ; then fi else - xshok_pretty_echo_and_log "Yara-Rules Database File Updates" "=" + xshok_pretty_echo_and_log "URLhaus Database File Updates" "=" xshok_draw_time_remaining "$((update_interval - time_interval))" "$urlhaus_update_hours" "urlhaus" fi fi From 41e2ea35d7c3a9394729aeea77043ea561ae944a Mon Sep 17 00:00:00 2001 From: Michael Orlitzky Date: Tue, 4 Feb 2020 20:00:24 -0500 Subject: [PATCH 004/101] config/os/os.gentoo.conf: disable updates and upgrade checks. On Gentoo, these functions are handled by the package manager, and wouldn't work if you tried them besides. So let's just turn them off to improve the error message in that case. --- config/os/os.gentoo.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/os/os.gentoo.conf b/config/os/os.gentoo.conf index ad9e9590..8b555315 100644 --- a/config/os/os.gentoo.conf +++ b/config/os/os.gentoo.conf @@ -31,4 +31,7 @@ clamd_restart_opt="clamdscan --reload" clamd_socket="/var/run/clamav/clamd.sock" +allow_upgrades="no" +allow_update_checks="no" + # https://eXtremeSHOK.com ###################################################### From 664d053ce7010176fe04b3772f5cf01426a59509 Mon Sep 17 00:00:00 2001 From: Michel Weitbrecht Date: Wed, 26 Feb 2020 23:53:15 +0100 Subject: [PATCH 005/101] master.conf: Enable yararulesproject by default According to the README, the yara rules project should be enabled by default. --- config/master.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/master.conf b/config/master.conf index 32de3830..ac2c43e0 100644 --- a/config/master.conf +++ b/config/master.conf @@ -124,7 +124,7 @@ malwarepatrol_enabled="yes" # Malware Patrol sanesecurity_enabled="yes" # Sanesecurity securiteinfo_enabled="yes" # SecuriteInfo urlhaus_enabled="yes" # urlhaus -yararulesproject_enabled="no" # Yara-Rule Project, automatically disabled if clamav is older than 0.100 and enable_yararules is disabled +yararulesproject_enabled="yes" # Yara-Rule Project, automatically disabled if clamav is older than 0.100 and enable_yararules is disabled # Disabled by default ## Enabling this will also cause the yararulesproject to be enabled if they are det to enabled. From 9da5f5bbf14dd9734a4aa4bf4940484ec5bb1a7b Mon Sep 17 00:00:00 2001 From: Michel Weitbrecht Date: Fri, 28 Feb 2020 22:47:49 +0100 Subject: [PATCH 006/101] master.conf: Ignore yara files that include modules modules/includes are not supported by ClamAV and lead to parsing errors and crashes. --- config/master.conf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/config/master.conf b/config/master.conf index 32de3830..19aa50b0 100644 --- a/config/master.conf +++ b/config/master.conf @@ -397,9 +397,6 @@ email/email_Ukraine_BE_powerattack.yar|MEDIUM email/scam.yar|MEDIUM # Detect well-known software packers, that can be used by malware to hide itself. packers/JJencode.yar|MEDIUM -packers/packer_compiler_signatures.yar|MEDIUM -packers/packer.yar|MEDIUM -packers/peid.yar|MEDIUM # HIGH # Used with documents to find if they have been crafted to leverage malicious code. maldocs/Maldoc_APT_OLE_JSRat.yar|HIGH @@ -431,11 +428,15 @@ email/attachment.yar # detects all emails with attachments email/image.yar # detects all emails with images email/urls.yar # detects all emails with urls crypto/crypto_signatures.yar # detects all files which are encrypted +# These files use module includes not supported by ClamAV +packers/packer_compiler_signatures.yar +packers/packer.yar +packers/peid.yar +antidebug_antivm ) declare -a yararulesproject_dbs_catagories=( #LOW -antidebug_antivm|LOW cve_rules|LOW exploit_kits|LOW malware|LOW From 396793a25916a05536bd75dc38b4bb66165a681a Mon Sep 17 00:00:00 2001 From: jkellerer Date: Sun, 26 Apr 2020 11:40:28 +0200 Subject: [PATCH 007/101] Updated sanesecurity publickey.gpg url to use SSL --- config/master.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/master.conf b/config/master.conf index 32de3830..492e8787 100644 --- a/config/master.conf +++ b/config/master.conf @@ -674,7 +674,7 @@ user_configuration_complete="no" linuxmalwaredetect_sigpack_url="https://cdn.rfxn.com/downloads/maldet-sigpack.tgz" linuxmalwaredetect_version_url="https://cdn.rfxn.com/downloads/maldet.sigs.ver" malwarepatrol_url="https://lists.malwarepatrol.net/cgi/getfile" -sanesecurity_gpg_url="http://www.sanesecurity.net/publickey.gpg" +sanesecurity_gpg_url="https://www.sanesecurity.com/publickey.gpg" sanesecurity_url="rsync.sanesecurity.net" securiteinfo_url="https://www.securiteinfo.com/get/signatures" urlhaus_url="https://urlhaus.abuse.ch/downloads" From 9d3c9103b73e09161fb7fc2ad9c428d147d80985 Mon Sep 17 00:00:00 2001 From: jkellerer Date: Sun, 26 Apr 2020 11:50:51 +0200 Subject: [PATCH 008/101] Enforce HTTPS validation by default --- config/user.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/user.conf b/config/user.conf index 04bcb69d..eab8c8f4 100644 --- a/config/user.conf +++ b/config/user.conf @@ -56,6 +56,10 @@ # Uncomment the following line to enable the script user_configuration_complete="yes" +# Enforce HTTPS validation +# Uncomment the following line to ignore SSL errors leading to insecure transfers +downloader_ignore_ssl="no" + # Proxy Support # If necessary to proxy database downloads, define the rsync, curl, wget, dig, hosr proxy settings here. #curl_proxy="--proxy http://username:password@proxy_host:proxy_port" From 8e9496f78fc2c69e9e9d186337cafbf767467573 Mon Sep 17 00:00:00 2001 From: jkellerer Date: Sun, 26 Apr 2020 11:59:21 +0200 Subject: [PATCH 009/101] Improved comment to avoid ambiguity --- config/user.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/user.conf b/config/user.conf index eab8c8f4..7af3a144 100644 --- a/config/user.conf +++ b/config/user.conf @@ -57,7 +57,7 @@ user_configuration_complete="yes" # Enforce HTTPS validation -# Uncomment the following line to ignore SSL errors leading to insecure transfers +# Disable the following line to ignore SSL errors leading to insecure transfers downloader_ignore_ssl="no" # Proxy Support From 124ae01150c703f3f83122eacd65163e4362dab7 Mon Sep 17 00:00:00 2001 From: Dieter Bloms Date: Mon, 12 Oct 2020 21:05:41 +0200 Subject: [PATCH 010/101] o added socks proxy support for rsync with help of netcat (nc) command --- clamav-unofficial-sigs.sh | 6 ++++++ config/user.conf | 1 + 2 files changed, 7 insertions(+) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index aa70db1d..6c3c24c4 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2358,6 +2358,12 @@ if [ -n "$rsync_proxy" ] ; then export RSYNC_PROXY fi +# If rsync connect program is defined in the config file, then export it for use. (to use netcat for socks tunnel) +if [ -n "$rsync_connect_prog" ] ; then + RSYNC_CONNECT_PROG="$rsync_connect_prog" + export RSYNC_CONNECT_PROG +fi + # Create $current_dbsfiles containing lists of current and previously active 3rd-party databases # so that databases and/or backup files that are no longer being used can be removed. current_tmp="${work_dir_work_configs}/current-dbs.tmp" diff --git a/config/user.conf b/config/user.conf index 04bcb69d..288d1753 100644 --- a/config/user.conf +++ b/config/user.conf @@ -62,6 +62,7 @@ user_configuration_complete="yes" #dig_proxy="@proxy_host -p proxy_host:proxy_port" #host_proxy="@proxy_host" #does not support port #rsync_proxy="username:password@proxy_host:proxy_port" +#rsync_connect_prog="nc -X 5 -x socksproxy_host:socksproxy_port %H 873" #wget_proxy="-e http_proxy=http://username:password@proxy_host:proxy_port -e https_proxy=https://username:password@proxy_host:proxy_port" # https://eXtremeSHOK.com ###################################################### From cc9ef322338fc73ba23f38d15c1eb5625d87b641 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 14:30:59 +0200 Subject: [PATCH 011/101] Enforce HTTPS validation by default --- README.md | 18 +++++++++++------- config/master.conf | 5 +++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e961bbd6..202ab664 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Please use the sample package os.*.conf as a base for your os.conf, this will di * Default configs are loaded in the following order if they exist: * master.conf -> os.conf -> os.*.conf -> user.conf or your-specified.config * user.conf will always override os.conf and master.conf, os.conf will override master.conf -* please do not alter the master.conf, rather create a user.conf +* please do not alter the master.conf, rather create a user.conf * A minimum of 1 config is required. * Specifying a config on the command line (-c | --config) will override the loading of the default configs @@ -97,7 +97,7 @@ Usage of SecuriteInfo 2015 free clamav signatures : https://www.securiteinfo.com ### Linux Malware Detect support (as of May 2015, updated January 2020) Usage of free Linux Malware Detect clamav signatures: https://www.rfxn.com/projects/linux-malware-detect/ - Enabled by default, no configuration required - + ### Need a database added ? Missing a database or a database not working ? Please post on the issue tracker : https://github.com/extremeshok/clamav-unofficial-sigs/issues @@ -199,11 +199,15 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] its associated files and databases from the system ## Change Log -### Version 7.0.1 (Updated 25 January 2020) + +### Version 7.2 ( XX December 2020 : Under Development) + - Enforce HTTPS validation by default + +### Version 7.0.1 (25 January 2020) - Disable yara project rules duplicated in rxfn.yara (Thanks @dominicraf) - Incremented the config to version 91 - -### Version 7.0.0 (Updated 24 January 2020) + +### Version 7.0.0 (24 January 2020) - eXtremeSHOK.com Maintenance - Added urlhaus database - Added extra yararulesproject databases @@ -235,7 +239,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - Only check for and notify about script updates every 12hours - Incremented the config to version 90 -### Version 6.1.1 (Updated 02 September 2019) +### Version 6.1.1 (02 September 2019) - eXtremeSHOK.com Maintenance - Update os.archlinux.conf, thanks @amishmm - master.conf set default dbs rating to medium @@ -249,7 +253,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - Minor enhancement to travis-ci checks - Incremented the config to version 77 -### Version 6.1.0 (Updated 27 August 2019) +### Version 6.1.0 (27 August 2019) - eXtremeSHOK.com Maintenance - Thanks Reio Remma & Oliver Nissen - fail added to all curl commands diff --git a/config/master.conf b/config/master.conf index 75d81d32..610bab24 100644 --- a/config/master.conf +++ b/config/master.conf @@ -569,8 +569,9 @@ enable_gpg="yes" rsync_connect_timeout="60" rsync_max_time="180" -# Ignore ssl errors and warnings, ie. operate in insecure mode. -downloader_ignore_ssl="yes" # Default is "yes" ignore ssl errors and warnings +# Enforce HTTPS validation +# Disable the following line to ignore SSL errors leading to insecure transfers +downloader_ignore_ssl="no" # Default is "no" ignore ssl errors and warnings # Set downloader connection, data transfer timeout limits in seconds. # The defaults settings here are reasonable, only change if you are From 0a30d2d06f2d140c8f0b8cdbd3e6bedbed2bb382 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 14:54:47 +0200 Subject: [PATCH 012/101] Fix: set ownership of last-version-check.txt --- README.md | 11 ++++++++++- clamav-unofficial-sigs.sh | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 202ab664..3b0b2521 100644 --- a/README.md +++ b/README.md @@ -200,8 +200,17 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] ## Change Log -### Version 7.2 ( XX December 2020 : Under Development) +### Version 7.1 ( XX December 2020 : Under Development) - Enforce HTTPS validation by default + - Updated sanesecurity publickey.gpg url to use SSL + - Ignore yara files that include modules + - Enabled yararulesproject rules by default + - os.gentoo.conf: disable updates and upgrade checks + - Fix: URLhaus log message + - Fix wrong download URL for MalwarePatrol + - Fix: set ownership of last-version-check.txt + - Incremented the config to version 92 + - Thank you @ ### Version 7.0.1 (25 January 2020) - Disable yara project rules duplicated in rxfn.yara (Thanks @dominicraf) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 82ef50a6..e0bf4381 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -3673,6 +3673,9 @@ if [ "$allow_update_checks" != "no" ] ; then time_interval="$((current_time - last_version_check))" if [ "$time_interval" -ge $((update_check_interval - 600)) ] ; then echo "$current_time" > "${work_dir_work_configs}/last-version-check.txt" + if xshok_is_root ; then + perms chown -f "${clam_user}:${clam_group}" "${work_dir_work_configs}/last-version-check.txt" + fi check_new_version fi From 7385592a4a2b54b5c709c18f76cda1b1c0209520 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 15:21:22 +0200 Subject: [PATCH 013/101] Replace $PWD with readlink -f "$0" fixes #299 --- README.md | 1 + clamav-unofficial-sigs.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3b0b2521..5d253ef8 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - Ignore yara files that include modules - Enabled yararulesproject rules by default - os.gentoo.conf: disable updates and upgrade checks + - Replace $PWD with readlink -f "$0" - Fix: URLhaus log message - Fix wrong download URL for MalwarePatrol - Fix: set ownership of last-version-check.txt diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index e0bf4381..a477c9c1 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -346,7 +346,7 @@ function xshok_file_download() { #outputfile #url #notimestamp else if [ ! "${3}" ] ; then # the following is required because wget, cannot do --timestamping and --output-document together - this_dir="$PWD" + this_dir="$(readlink -f "$0")" output_file="$1" url="$2" output_dir="${output_file%/*}" @@ -1505,14 +1505,14 @@ minimum_yara_clamav_version="0.100" this_script_full_path="${BASH_SOURCE[0]}" # follow the symlinks while [ -h "$this_script_full_path" ]; do - this_script_path="$( cd -P "$( dirname "$this_script_full_path" )" >/dev/null 2>&1 && pwd )" + this_script_path="$( cd -P "$( dirname "$this_script_full_path" )" >/dev/null 2>&1 && readlink -f "$0" )" this_script_full_path="$(readlink "$this_script_full_path")" # if relative symlink, then resolve the path if [[ $this_script_full_path != /* ]] ; then this_script_full_path="$this_script_path/$this_script_full_path" fi done -this_script_path="$( cd -P "$( dirname "$this_script_full_path" )" >/dev/null 2>&1 && pwd )" +this_script_path="$( cd -P "$( dirname "$this_script_full_path" )" >/dev/null 2>&1 && readlink -f "$0" )" this_script_name="$(basename "$this_script_full_path")" if [ -z "$this_script_full_path" ] || [ -z "$this_script_path" ] || [ -z "$this_script_name" ] ; then From f1e0e71ee3ede346a5e1134da84e088c33c9d75a Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 15:40:23 +0200 Subject: [PATCH 014/101] Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) fixes: #300 --- README.md | 1 + clamav-unofficial-sigs.sh | 16 +++++++--------- config/master.conf | 3 ++- config/user.conf | 1 + 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5d253ef8..3252b685 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - Replace $PWD with readlink -f "$0" - Fix: URLhaus log message - Fix wrong download URL for MalwarePatrol + - Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) - Fix: set ownership of last-version-check.txt - Incremented the config to version 92 - Thank you @ diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index a477c9c1..68b2dc2a 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2234,12 +2234,10 @@ else malwarepatrol_product_code=8 fi fi - -if [ $malwarepatrol_list == "clamav_basic" ] ; then - malwarepatrol_db="malwarepatrol.db" -else - malwarepatrol_db="malwarepatrol.ndb" +if [ -z $malwarepatrol_db ] ; then + malwarepatrol_db="malwarepatrol.db" fi + malwarepatrol_url="${malwarepatrol_url}?receipt=${malwarepatrol_receipt_code}&product=${malwarepatrol_product_code}&list=${malwarepatrol_list}" # If "ham_dir" variable is set, then create initial whitelist files (skipped if first-time script run). @@ -2972,12 +2970,12 @@ if [ "$malwarepatrol_enabled" == "yes" ] ; then malwarepatrol_updates="0" # Cleanup any not required database files - if [ "$malwarepatrol_db" == "malwarepatrol.db" ] && [ -f "${clam_dbs}/malwarepatrol.ndb" ] ; then - rm -f "${clam_dbs}/malwarepatrol.ndb"; - fi - if [ "$malwarepatrol_db" == "malwarepatrol.ndb" ] && [ -f "${clam_dbs}/malwarepatrol.db" ] ; then + if [ "$malwarepatrol_db" != "malwarepatrol.db" ] && [ -f "${clam_dbs}/malwarepatrol.db" ] ; then rm -f "${clam_dbs}/malwarepatrol.db"; fi + if [ "$malwarepatrol_db" != "malwarepatrol.ndb" ] && [ -f "${clam_dbs}/malwarepatrol.ndb" ] ; then + rm -f "${clam_dbs}/malwarepatrol.ndb"; + fi if [ "$loop" == "1" ] ; then xshok_pretty_echo_and_log "---" diff --git a/config/master.conf b/config/master.conf index 610bab24..4bb17b82 100644 --- a/config/master.conf +++ b/config/master.conf @@ -83,6 +83,7 @@ malwarepatrol_list="clamav_basic" # clamav_basic or clamav_ext # the malwarepatrol_free is set to no (non-free) # set to no to enable the commercial subscription url, malwarepatrol_free="yes" +malwarepatrol_db="malwarepatrol.db" # ========================= # SecuriteInfo : https://www.SecuriteInfo.com @@ -684,7 +685,7 @@ yararulesproject_url="https://raw.githubusercontent.com/Yara-Rules/rules/master" # ======================== # DO NOT EDIT ! -config_version="91" +config_version="92" ################################################################################ # diff --git a/config/user.conf b/config/user.conf index 6877de3f..67d4f94c 100644 --- a/config/user.conf +++ b/config/user.conf @@ -27,6 +27,7 @@ # if the malwarepatrol_product_code is not 8 the malwarepatrol_free is set to no (non-free) #malwarepatrol_product_code="8" #malwarepatrol_receipt_code="YOUR-RECEIPT-NUMBER" +#malwarepatrol_db="malwarepatrol.db" #securiteinfo_authorisation_signature="YOUR-SIGNATURE-NUMBER" # Enable if you have a commercial/premium/non-free subscription From dd48c01854788f5d3279756fe726d080e70f9ed2 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 15:52:52 +0200 Subject: [PATCH 015/101] bsd fix for sed #303 --- README.md | 3 ++- clamav-unofficial-sigs.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3252b685..8fcacf10 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,8 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) - Fix: set ownership of last-version-check.txt - Incremented the config to version 92 - - Thank you @ + - BSD read config fix + - Thank you @Mrothyr ### Version 7.0.1 (25 January 2020) - Disable yara project rules duplicated in rxfn.yara (Thanks @dominicraf) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 68b2dc2a..fac43143 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -1767,7 +1767,8 @@ for config_file in "${config_files[@]}" ; do # Delete both trailing and leading whitespace # Delete all trailing whitespace # Delete all empty lines - clean_config="$(command sed -e '/^#.*/d' -e 's/[[:space:]]#.*//' -e 's/#[[:space:]].*//' -e 's/^[ \t]*//;s/[ \t]*$//' -e '/^\s*$/d' "$config_file")" + clean_config="$(command sed -e '/^#./d' -e 's/[[:space:]]#.//' -e 's/#[[:space:]].//' -e $'s/^[ \t]//;s/[ \t]$//' -e '/^\s$/d' "$config_file")" + fi #fix eval of | From 552a147a114151baa57ea07dda284a91c15d43ca Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 16:04:23 +0200 Subject: [PATCH 016/101] Added tar_executable to specify a non-default tar executable --- README.md | 3 +++ clamav-unofficial-sigs.sh | 8 ++++++-- config/master.conf | 3 +++ config/os/{os.macosx.conf => os.macos.conf} | 2 ++ config/os/os.openbsd.conf | 2 ++ guides/{macosx.md => macos.md} | 16 +++++++++++----- 6 files changed, 27 insertions(+), 7 deletions(-) rename config/os/{os.macosx.conf => os.macos.conf} (98%) rename guides/{macosx.md => macos.md} (96%) diff --git a/README.md b/README.md index 8fcacf10..fc49440d 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,9 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - Fix: URLhaus log message - Fix wrong download URL for MalwarePatrol - Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) + - Added tar_executable to specify a non-default tar executable + - Config os.macosx.conf renamed to os.macos.conf + - Added tar_executable=gtar to os.macos.conf and os.openbsd.conf - Fix: set ownership of last-version-check.txt - Incremented the config to version 92 - BSD read config fix diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index fac43143..97e7229c 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -1605,9 +1605,13 @@ if [ -x /usr/gnu/bin/grep ] ; then else grep_bin="$(command -v grep 2> /dev/null)" fi -# Detect support for tar +# Detect support for tar or custom tar_executable if [ -z "$tar_bin" ]; then - tar_bin="$(command -v tar 2> /dev/null)" + if [ -z "$tar_executable" ]; then + tar_bin="$(command -v tar 2> /dev/null)" + else + tar_bin="$(command -v "$tar_executable" 2> /dev/null)" + fi fi # Detect support for curl if [ -z "$curl_bin" ]; then diff --git a/config/master.conf b/config/master.conf index 4bb17b82..ace3e163 100644 --- a/config/master.conf +++ b/config/master.conf @@ -537,6 +537,9 @@ min_sleep_time="60" # Default minimum is 60 seconds (1 minute). # Command to do a full clamd service stop/start #clamd_restart_opt="service clamd restart" +#tar_executable, specofy a custom tar execurtable, ie mac/bsd = gtar +#tar_executable="tar" + # Custom Command Paths, these are detected with the which command when not set #clamscan_bin="/usr/bin/clamscan" #curl_bin="/usr/bin/curl" diff --git a/config/os/os.macosx.conf b/config/os/os.macos.conf similarity index 98% rename from config/os/os.macosx.conf rename to config/os/os.macos.conf index 2333618c..fb5e06f1 100644 --- a/config/os/os.macosx.conf +++ b/config/os/os.macos.conf @@ -43,6 +43,8 @@ clamd_restart_opt="/usr/local/bin/clamdscan --reload" #gpg_bin="/usr/local/bin/gpg" +tar_executable="gtar" + enable_gpg="no" # https://eXtremeSHOK.com ###################################################### diff --git a/config/os/os.openbsd.conf b/config/os/os.openbsd.conf index 9f311879..14196d36 100644 --- a/config/os/os.openbsd.conf +++ b/config/os/os.openbsd.conf @@ -35,6 +35,8 @@ log_file_path="/var/clamav/log" clamd_restart_opt="rcctl restart clamd" +tar_executable="gtar" + #clamd_socket="/var/run/clamav/clamd.sock" # https://eXtremeSHOK.com ###################################################### diff --git a/guides/macosx.md b/guides/macos.md similarity index 96% rename from guides/macosx.md rename to guides/macos.md index b22568ad..6dcd3a29 100644 --- a/guides/macosx.md +++ b/guides/macos.md @@ -25,12 +25,18 @@ https://www.clamav.net/documents/installation-on-macos-mac-os-x /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" ``` -# Step 2 Install clamav + +# Step 2 Install gtar +``` +brew install gtar +``` + +# Step 3 Install clamav ``` brew install clamav ``` -# Step 3 +# Step 4 ``` sudo su curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/clamav-unofficial-sigs.sh --output /usr/local/bin/clamav-unofficial-sigs.sh @@ -42,19 +48,19 @@ curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master exit ``` -# Step 4 +# Step 5 set your user options ``` sudo pico /etc/clamav-unofficial-sigs/user.conf ``` -# Step 5 +# Step 6 Console (shell) ``` clamav-unofficial-sigs.sh --force ``` -# Step 6 +# Step 7 launchd helper Script (replaces cron) ``` sudo su From f90957e70ff09fc9c99f1c01b3d03f5b859d0083 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 16:08:09 +0200 Subject: [PATCH 017/101] thank you all --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fc49440d..49e6c763 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - Fix: set ownership of last-version-check.txt - Incremented the config to version 92 - BSD read config fix - - Thank you @Mrothyr + - Thank you @dandanio @Mrothyr @m0urs @msapiro @orlitzky @SlothOfAnarchy @jkellerer ### Version 7.0.1 (25 January 2020) - Disable yara project rules duplicated in rxfn.yara (Thanks @dominicraf) From b292ac9b66623a9dacf2b17036a83311bb3195a9 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 16:23:33 +0200 Subject: [PATCH 018/101] thanks @RobbieTheK --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 49e6c763..1e9ea0cc 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - Fix: set ownership of last-version-check.txt - Incremented the config to version 92 - BSD read config fix - - Thank you @dandanio @Mrothyr @m0urs @msapiro @orlitzky @SlothOfAnarchy @jkellerer + - Thank you @dandanio @Mrothyr @m0urs @msapiro @orlitzky @SlothOfAnarchy @jkellerer @RobbieTheK ### Version 7.0.1 (25 January 2020) - Disable yara project rules duplicated in rxfn.yara (Thanks @dominicraf) From c230b76513e4a3b0cfa3a147d14c46254c9fc6bc Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 16:56:05 +0200 Subject: [PATCH 019/101] fallback to host if dig is not used fixes #322 --- README.md | 1 + clamav-unofficial-sigs.sh | 25 ++++++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1e9ea0cc..8285179d 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - Config os.macosx.conf renamed to os.macos.conf - Added tar_executable=gtar to os.macos.conf and os.openbsd.conf - Fix: set ownership of last-version-check.txt + - Fix: fallback to host if dig is not used - Incremented the config to version 92 - BSD read config fix - Thank you @dandanio @Mrothyr @m0urs @msapiro @orlitzky @SlothOfAnarchy @jkellerer @RobbieTheK diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 97e7229c..d34452c8 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2513,23 +2513,30 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then echo "$current_time" > "${work_dir_work_configs}/last-ss-update.txt" xshok_pretty_echo_and_log "Sanesecurity Database & GPG Signature File Updates" "=" xshok_pretty_echo_and_log "Checking for Sanesecurity updates..." - # shellcheck disable=SC2086 - sanesecurity_mirror_ips="$(dig $dig_proxy +ignore +short "$sanesecurity_url")" - # Add fallback to host if dig returns no records + if [ -z "$dig_bin" ] ; then + # shellcheck disable=SC2086 + sanesecurity_mirror_ips="$($dig_bin $dig_proxy +ignore +short "$sanesecurity_url")" + else + sanesecurity_mirror_ips="" + fi + # Add fallback to host if dig returns no records or dig is not used if [ ${#sanesecurity_mirror_ips} -lt 1 ] ; then # shellcheck disable=SC2086 - sanesecurity_mirror_ips="$(host $host_proxy -t A "$sanesecurity_url" | sed -n '/has address/{s/.*address \([^ ]*\).*/\1/;p;}')" + sanesecurity_mirror_ips="$($host_bin $host_proxy -t A "$sanesecurity_url" | sed -n '/has address/{s/.*address \([^ ]*\).*/\1/;p;}')" fi if [ ${#sanesecurity_mirror_ips} -ge 1 ] ; then for sanesecurity_mirror_ip in $sanesecurity_mirror_ips ; do - sanesecurity_mirror_name="" - # shellcheck disable=SC2086 - sanesecurity_mirror_name="$(dig $dig_proxy +short -x "$sanesecurity_mirror_ip" | command sed 's/\.$//')" - # Add fallback to host if dig returns no records + if [ -z "$dig_bin" ] ; then + # shellcheck disable=SC2086 + sanesecurity_mirror_name="$($dig_bin $dig_proxy +short -x "$sanesecurity_mirror_ip" | command sed 's/\.$//')" + else + sanesecurity_mirror_name="" + fi + # Add fallback to host if dig returns no records or dig is not used if [ -z "$sanesecurity_mirror_name" ] ; then # shellcheck disable=SC2086 - sanesecurity_mirror_name="$(host $host_proxy "$sanesecurity_mirror_ip" | sed -n '/name pointer/{s/.*pointer \([^ ]*\).*\.$/\1/;p;}')" + sanesecurity_mirror_name="$($host_bin $host_proxy "$sanesecurity_mirror_ip" | sed -n '/name pointer/{s/.*pointer \([^ ]*\).*\.$/\1/;p;}')" fi sanesecurity_mirror_site_info="$sanesecurity_mirror_name $sanesecurity_mirror_ip" xshok_pretty_echo_and_log "Sanesecurity mirror site used: ${sanesecurity_mirror_site_info}" From b7afd6d9b3c0ad931464bb33966195c7be6b2524 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 17:17:15 +0200 Subject: [PATCH 020/101] minimum config_version increased add option downloader_ignore_ssl_errors remove mailto: from generated cron fixes #325 --- README.md | 1 + clamav-unofficial-sigs.sh | 7 ++++--- config/master.conf | 8 ++++---- config/user.conf | 6 +++--- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8285179d..4be6c375 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - Added tar_executable=gtar to os.macos.conf and os.openbsd.conf - Fix: set ownership of last-version-check.txt - Fix: fallback to host if dig is not used + - Disable cron MAILTO - Incremented the config to version 92 - BSD read config fix - Thank you @dandanio @Mrothyr @m0urs @msapiro @orlitzky @SlothOfAnarchy @jkellerer @RobbieTheK diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index d34452c8..fbfd11d9 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -672,7 +672,8 @@ function install_cron() { # script itself is set to randomize the actual execution time between # 60 - 600 seconds. To Adjust the cron values, edit your configs and run # bash clamav-unofficial-sigs.sh --install-cron to generate a new file. -MAILTO=root +# Uncomment to enable emails to the root user +#MAILTO=root $cron_minute * * * * ${cron_sudo} ${cron_user} [ -x ${cron_script_full_path} ] && ${cron_bash} ${cron_script_full_path} # https://eXtremeSHOK.com ###################################################### @@ -1498,7 +1499,7 @@ EOF # Script Info script_version="7.0.1" script_version_date="2020-01-25" -minimum_required_config_version="91" +minimum_required_config_version="92" minimum_yara_clamav_version="0.100" # Discover script: name, full_path and path @@ -2098,7 +2099,7 @@ else fi # Suppress ssl warnings -if [ "$downloader_ignore_ssl" == "yes" ] ; then +if [ "$downloader_ignore_ssl_errors" == "yes" ] ; then wget_insecure="--no-check-certificate" curl_insecure="--insecure" else diff --git a/config/master.conf b/config/master.conf index ace3e163..d4375a90 100644 --- a/config/master.conf +++ b/config/master.conf @@ -537,7 +537,7 @@ min_sleep_time="60" # Default minimum is 60 seconds (1 minute). # Command to do a full clamd service stop/start #clamd_restart_opt="service clamd restart" -#tar_executable, specofy a custom tar execurtable, ie mac/bsd = gtar +#tar_executable, specofy a custom tar execurtable, ie mac/bsd = g #tar_executable="tar" # Custom Command Paths, these are detected with the which command when not set @@ -573,9 +573,9 @@ enable_gpg="yes" rsync_connect_timeout="60" rsync_max_time="180" -# Enforce HTTPS validation -# Disable the following line to ignore SSL errors leading to insecure transfers -downloader_ignore_ssl="no" # Default is "no" ignore ssl errors and warnings +# HTTPS validation +# Uncomment to allow and ignore SSL errors leading to insecure transfers +# downloader_ignore_ssl_errors="yes" # Default is "no" # Set downloader connection, data transfer timeout limits in seconds. # The defaults settings here are reasonable, only change if you are diff --git a/config/user.conf b/config/user.conf index 67d4f94c..0f95f6bb 100644 --- a/config/user.conf +++ b/config/user.conf @@ -57,9 +57,9 @@ # Uncomment the following line to enable the script user_configuration_complete="yes" -# Enforce HTTPS validation -# Disable the following line to ignore SSL errors leading to insecure transfers -downloader_ignore_ssl="no" +# HTTPS validation +# Uncomment to allow and ignore SSL errors leading to insecure transfers +# downloader_ignore_ssl_errors="yes" # Default is "no" # Proxy Support # If necessary to proxy database downloads, define the rsync, curl, wget, dig, hosr proxy settings here. From 50e96703a4bda6dda6890d66f9e0fdb48fbff752 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 17:21:41 +0200 Subject: [PATCH 021/101] fix #326 --- clamav-unofficial-sigs.sh | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index fbfd11d9..d71d84ac 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2170,9 +2170,11 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then else temp_db="$(xshok_database "$default_dbs_rating" "${sanesecurity_dbs[@]}")" fi - sanesecurity_dbs=( ) - #sanesecurity_dbs=( $temp_db ) - read -r -a sanesecurity_dbs <<< "$temp_db" + sanesecurity_dbs=( ) + if [ ! -z $temp_db ] ; then + #sanesecurity_dbs=( $temp_db ) + read -r -a sanesecurity_dbs <<< "$temp_db" + fi fi fi if [ "$securiteinfo_enabled" == "yes" ] ; then @@ -2182,9 +2184,11 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then else temp_db="$(xshok_database "$default_dbs_rating" "${securiteinfo_dbs[@]}")" fi - securiteinfo_dbs=( ) - #securiteinfo_dbs=( $temp_db ) - read -r -a securiteinfo_dbs <<< "$temp_db" + securiteinfo_dbs=( ) + if [ ! -z $temp_db ] ; then + #securiteinfo_dbs=( $temp_db ) + read -r -a securiteinfo_dbs <<< "$temp_db" + fi fi fi if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then @@ -2194,9 +2198,11 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then else temp_db="$(xshok_database "$default_dbs_rating" "${linuxmalwaredetect_dbs[@]}")" fi - linuxmalwaredetect_dbs=( ) - #linuxmalwaredetect_dbs=( $temp_db ) - read -r -a linuxmalwaredetect_dbs <<< "$temp_db" + linuxmalwaredetect_dbs=( ) + if [ ! -z $temp_db ] ; then + #linuxmalwaredetect_dbs=( $temp_db ) + read -r -a linuxmalwaredetect_dbs <<< "$temp_db" + fi fi fi if [ "$yararulesproject_enabled" == "yes" ] ; then @@ -2207,8 +2213,10 @@ if [ "$yararulesproject_enabled" == "yes" ] ; then temp_db="$(xshok_database "$default_dbs_rating" "${yararulesproject_dbs[@]}")" fi yararulesproject_dbs=( ) - #yararulesproject_dbs=( $temp_db ) - read -r -a yararulesproject_dbs <<< "$temp_db" + if [ ! -z $temp_db ] ; then + #yararulesproject_dbs=( $temp_db ) + read -r -a yararulesproject_dbs <<< "$temp_db" + fi fi fi if [ "$urlhaus_enabled" == "yes" ] ; then @@ -2219,8 +2227,10 @@ if [ "$urlhaus_enabled" == "yes" ] ; then temp_db="$(xshok_database "$default_dbs_rating" "${urlhaus_dbs[@]}")" fi urlhaus_dbs=( ) - #urlhaus_dbs=( $temp_db ) - read -r -a urlhaus_dbs <<< "$temp_db" + if [ ! -z $temp_db ] ; then + #urlhaus_dbs=( $temp_db ) + read -r -a urlhaus_dbs <<< "$temp_db" + fi fi fi # Set the variables for MalwarePatrol From 4885b516354cfada721c686ec8ffe72636f86109 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 19:47:52 +0200 Subject: [PATCH 022/101] fix shellcheck issues --- clamav-unofficial-sigs.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index d71d84ac..2a440d8c 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2171,7 +2171,7 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then temp_db="$(xshok_database "$default_dbs_rating" "${sanesecurity_dbs[@]}")" fi sanesecurity_dbs=( ) - if [ ! -z $temp_db ] ; then + if [ ! -z "$temp_db" ] ; then #sanesecurity_dbs=( $temp_db ) read -r -a sanesecurity_dbs <<< "$temp_db" fi @@ -2185,7 +2185,7 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then temp_db="$(xshok_database "$default_dbs_rating" "${securiteinfo_dbs[@]}")" fi securiteinfo_dbs=( ) - if [ ! -z $temp_db ] ; then + if [ ! -z "$temp_db" ] ; then #securiteinfo_dbs=( $temp_db ) read -r -a securiteinfo_dbs <<< "$temp_db" fi @@ -2199,7 +2199,7 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then temp_db="$(xshok_database "$default_dbs_rating" "${linuxmalwaredetect_dbs[@]}")" fi linuxmalwaredetect_dbs=( ) - if [ ! -z $temp_db ] ; then + if [ ! -z "$temp_db" ] ; then #linuxmalwaredetect_dbs=( $temp_db ) read -r -a linuxmalwaredetect_dbs <<< "$temp_db" fi @@ -2213,7 +2213,7 @@ if [ "$yararulesproject_enabled" == "yes" ] ; then temp_db="$(xshok_database "$default_dbs_rating" "${yararulesproject_dbs[@]}")" fi yararulesproject_dbs=( ) - if [ ! -z $temp_db ] ; then + if [ ! -z "$temp_db" ] ; then #yararulesproject_dbs=( $temp_db ) read -r -a yararulesproject_dbs <<< "$temp_db" fi @@ -2227,7 +2227,7 @@ if [ "$urlhaus_enabled" == "yes" ] ; then temp_db="$(xshok_database "$default_dbs_rating" "${urlhaus_dbs[@]}")" fi urlhaus_dbs=( ) - if [ ! -z $temp_db ] ; then + if [ ! -z "$temp_db" ] ; then #urlhaus_dbs=( $temp_db ) read -r -a urlhaus_dbs <<< "$temp_db" fi @@ -2250,7 +2250,7 @@ else malwarepatrol_product_code=8 fi fi -if [ -z $malwarepatrol_db ] ; then +if [ -z "$malwarepatrol_db" ] ; then malwarepatrol_db="malwarepatrol.db" fi From d6c7e730120e7ab0d6f564e752073222c7d22b9d Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 20:09:12 +0200 Subject: [PATCH 023/101] detect os and use gtar if it is required revert clean config commit --- clamav-unofficial-sigs.sh | 20 ++++++++++++-------- config/os/os.macos.conf | 2 -- config/os/os.openbsd.conf | 2 -- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 2a440d8c..74dbd276 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -1606,13 +1606,17 @@ if [ -x /usr/gnu/bin/grep ] ; then else grep_bin="$(command -v grep 2> /dev/null)" fi -# Detect support for tar or custom tar_executable +# Detect support for tar or gtar + +if [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then + tar_executable="gtar" +else + tar_executable="tar" +fi if [ -z "$tar_bin" ]; then - if [ -z "$tar_executable" ]; then - tar_bin="$(command -v tar 2> /dev/null)" - else tar_bin="$(command -v "$tar_executable" 2> /dev/null)" - fi + else + xshok_pretty_echo_and_log "ERROR: gtar (gnu tar) is missing" fi # Detect support for curl if [ -z "$curl_bin" ]; then @@ -1756,8 +1760,8 @@ for config_file in "${config_files[@]}" ; do clean_config="$(echo "$clean_config" | xargs)" clean_config="$(echo "$clean_config" | sed -e '/^\s*$/d')" # Blank lines - elif [ "$(uname -s)" == "Darwin" ] ; then - # MacOS / OS X fixes, had issues with running with a single command and with SunOS work around.. + elif [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then + # MacOS / OSX / BSD fixes, had issues with running with a single command and with SunOS work around.. clean_config="$(command sed -e '/^#.*/d' "$config_file")" # Comment line clean_config="$(echo "$clean_config" | sed -e 's/#[[:space:]].*//')" # Comment line (duplicated) clean_config="$(echo "$clean_config" | sed -e '/^[[:blank:]]*#/d;s/#.*//')" # Comments at end of line @@ -1772,7 +1776,7 @@ for config_file in "${config_files[@]}" ; do # Delete both trailing and leading whitespace # Delete all trailing whitespace # Delete all empty lines - clean_config="$(command sed -e '/^#./d' -e 's/[[:space:]]#.//' -e 's/#[[:space:]].//' -e $'s/^[ \t]//;s/[ \t]$//' -e '/^\s$/d' "$config_file")" + clean_config="$(command sed -e '/^#.*/d' -e 's/[[:space:]]#.*//' -e 's/#[[:space:]].*//' -e 's/^[ \t]*//;s/[ \t]*$//' -e '/^\s*$/d' "$config_file")" fi diff --git a/config/os/os.macos.conf b/config/os/os.macos.conf index fb5e06f1..2333618c 100644 --- a/config/os/os.macos.conf +++ b/config/os/os.macos.conf @@ -43,8 +43,6 @@ clamd_restart_opt="/usr/local/bin/clamdscan --reload" #gpg_bin="/usr/local/bin/gpg" -tar_executable="gtar" - enable_gpg="no" # https://eXtremeSHOK.com ###################################################### diff --git a/config/os/os.openbsd.conf b/config/os/os.openbsd.conf index 14196d36..9f311879 100644 --- a/config/os/os.openbsd.conf +++ b/config/os/os.openbsd.conf @@ -35,8 +35,6 @@ log_file_path="/var/clamav/log" clamd_restart_opt="rcctl restart clamd" -tar_executable="gtar" - #clamd_socket="/var/run/clamav/clamd.sock" # https://eXtremeSHOK.com ###################################################### From a51705896f8aa5efcf05f265edc703e9aa0b6055 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 20:35:22 +0200 Subject: [PATCH 024/101] travis updates --- .t/ci-test.sh | 10 +++++----- .travis.yml | 49 +++++++++++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index 5fd9240d..2d94ca31 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -15,8 +15,8 @@ if [ -e "/var/lib/clamav/sanesecurity.ftm" ] ; then fi echo "running script as root and verbose and force_wget" -sudo cp -f .t/tests/user_wget.conf /etc/clamav-unofficial-sigs/user.conf -sudo bash /usr/sbin/clamav-unofficial-sigs --verbose +cp -f .t/tests/user_wget.conf /etc/clamav-unofficial-sigs/user.conf +bash /usr/sbin/clamav-unofficial-sigs --verbose if [ "$?" -eq "0" ] ; then echo .. OK else @@ -25,8 +25,8 @@ else fi echo "running script as root and verbose" -sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf -sudo bash /usr/sbin/clamav-unofficial-sigs --verbose +cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf +bash /usr/sbin/clamav-unofficial-sigs --verbose if [ "$?" -eq "0" ] ; then echo .. OK else @@ -35,7 +35,7 @@ else fi echo "running script as clamav and silence" -sudo -u clamav [ -x /usr/sbin/clamav-unofficial-sigs ] && bash /usr/sbin/clamav-unofficial-sigs --force --silence +-u clamav [ -x /usr/sbin/clamav-unofficial-sigs ] && bash /usr/sbin/clamav-unofficial-sigs --force --silence if [ "$?" -eq "0" ] ; then echo .. OK else diff --git a/.travis.yml b/.travis.yml index f9cb4502..204c622b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ # Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com # License: BSD (Berkeley Software Distribution) ################## -language: bash +language: shell +os: linux # Required travis ci environment variables #ci_securiteinfo_authorisation_signature=[secure] @@ -12,13 +13,13 @@ language: bash #ci_malwarepatrol_product_code=[secure] #ci_malwarepatrol_list=[secure] #ci_malwarepatrol_free=[secure] +#ci_codeclimate_repo_token=[secure] #Enable modern build platform, not container based. -sudo: required dist: bionic -# Apparently caching is disabled, since we used modern build platform as we require clamav and sudo ;/ +# Apparently caching is disabled, since we used modern build platform as we require clamav and ;/ # caching the /var/lib/clamav/ will save +/-9mins of run time and allow the use of freshclam signatures #Enable Caching @@ -27,34 +28,34 @@ dist: bionic # - tmp/cache before_install: - - sudo apt-get update -qq - - sudo apt-get install -y ca-certificates curl wget rsync p7zip-full shellcheck -qq - #- sudo pip install bashate - # - "sudo mkdir -p tmp/cache/" + - apt-get update -qq + - apt-get install -y ca-certificates curl wget rsync p7zip-full shellcheck -qq + #- pip install bashate + # - "mkdir -p tmp/cache/" # - "ls -laFh tmp/cache/clamav-dbs" - # - "sudo rsync -rlptDv tmp/cache/ /var/lib/clamav/" - # - "sudo freshclam" + # - "rsync -rlptDv tmp/cache/ /var/lib/clamav/" + # - "freshclam" # - "ls -laFh /var/lib/clamav/" - # - "sudo rsync -rlptDav /var/lib/clamav/ tmp/cache/" - # - "sudo chown -R travis:travis tmp/cache" - # - "sudo chmod -R 777 tmp/cache/" + # - "rsync -rlptDav /var/lib/clamav/ tmp/cache/" + # - "chown -R travis:travis tmp/cache" + # - "chmod -R 777 tmp/cache/" # - "ls -laFh tmp/cache/clamav-dbs/" install: - - sudo mkdir -p /etc/clamav-unofficial-sigs - - sudo cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf - - sudo cp -f config/os/os.ubuntu.conf /etc/clamav-unofficial-sigs/os.conf - - sudo cp -f clamav-unofficial-sigs.sh /usr/sbin/clamav-unofficial-sigs + - mkdir -p /etc/clamav-unofficial-sigs + - cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf + - cp -f config/os/os.ubuntu.conf /etc/clamav-unofficial-sigs/os.conf + - cp -f clamav-unofficial-sigs.sh /usr/sbin/clamav-unofficial-sigs script: - - sudo sh -e .t/ci-shellcheck.sh - - sudo sh -e .t/ci-clamav-download-default-databases.sh - #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh - - sudo sh -e .t/ci-clamav-install-default.sh - - sudo sh -e .t/ci-clamav-install-default-database.sh - - sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf - - sudo sh -e .t/ci-test.sh + - sh -e .t/ci-shellcheck.sh + - sh -e .t/ci-clamav-download-default-databases.sh + #- sh -e .t/ci-clamav-download-default-databases-git.sh + - sh -e .t/ci-clamav-install-default.sh + - sh -e .t/ci-clamav-install-default-database.sh + - cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf + - sh -e .t/ci-test.sh addons: code_climate: - repo_token: 270be84579d01fcbe78b21b91719c2c55cd627f074c8abb83699a8319c980b60 + repo_token: $ci_codeclimate_repo_token From fdaccab7761a13129a4ccf16963b7076e671877a Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 21:16:41 +0200 Subject: [PATCH 025/101] sudo su --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 204c622b..663b047c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,7 @@ # Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com # License: BSD (Berkeley Software Distribution) ################## -language: shell -os: linux +language: bash # Required travis ci environment variables #ci_securiteinfo_authorisation_signature=[secure] @@ -13,7 +12,6 @@ os: linux #ci_malwarepatrol_product_code=[secure] #ci_malwarepatrol_list=[secure] #ci_malwarepatrol_free=[secure] -#ci_codeclimate_repo_token=[secure] #Enable modern build platform, not container based. @@ -28,6 +26,7 @@ dist: bionic # - tmp/cache before_install: + - sudo su - apt-get update -qq - apt-get install -y ca-certificates curl wget rsync p7zip-full shellcheck -qq #- pip install bashate @@ -42,12 +41,14 @@ before_install: # - "ls -laFh tmp/cache/clamav-dbs/" install: + - sudo su - mkdir -p /etc/clamav-unofficial-sigs - cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf - cp -f config/os/os.ubuntu.conf /etc/clamav-unofficial-sigs/os.conf - cp -f clamav-unofficial-sigs.sh /usr/sbin/clamav-unofficial-sigs script: + - sudo su - sh -e .t/ci-shellcheck.sh - sh -e .t/ci-clamav-download-default-databases.sh #- sh -e .t/ci-clamav-download-default-databases-git.sh @@ -58,4 +59,4 @@ script: addons: code_climate: - repo_token: $ci_codeclimate_repo_token + repo_token: 270be84579d01fcbe78b21b91719c2c55cd627f074c8abb83699a8319c980b60 From cd56397b0114aece1b331e91254986fa4c7ebe97 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 21:17:51 +0200 Subject: [PATCH 026/101] corrections --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 663b047c..9b26f745 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ # Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com # License: BSD (Berkeley Software Distribution) ################## -language: bash +language: shell +os: linux # Required travis ci environment variables #ci_securiteinfo_authorisation_signature=[secure] @@ -12,7 +13,7 @@ language: bash #ci_malwarepatrol_product_code=[secure] #ci_malwarepatrol_list=[secure] #ci_malwarepatrol_free=[secure] - +#ci_codeclimate_repo_token=[secure] #Enable modern build platform, not container based. dist: bionic @@ -59,4 +60,4 @@ script: addons: code_climate: - repo_token: 270be84579d01fcbe78b21b91719c2c55cd627f074c8abb83699a8319c980b60 + repo_token: $ci_codeclimate_repo_token From 6abf97db1db742f0d59d9cbcd977ca4da343344c Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 23:25:53 +0200 Subject: [PATCH 027/101] fix inverted logic --- .travis.yml | 29 +- clamav-unofficial-sigs.sh | 866 +++++++++++++++++++------------------- 2 files changed, 449 insertions(+), 446 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9b26f745..771be681 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,9 +27,8 @@ dist: bionic # - tmp/cache before_install: - - sudo su - - apt-get update -qq - - apt-get install -y ca-certificates curl wget rsync p7zip-full shellcheck -qq + - sudo apt-get update -qq + - sudo apt-get install -y ca-certificates curl wget rsync p7zip-full shellcheck -qq #- pip install bashate # - "mkdir -p tmp/cache/" # - "ls -laFh tmp/cache/clamav-dbs" @@ -42,21 +41,19 @@ before_install: # - "ls -laFh tmp/cache/clamav-dbs/" install: - - sudo su - - mkdir -p /etc/clamav-unofficial-sigs - - cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf - - cp -f config/os/os.ubuntu.conf /etc/clamav-unofficial-sigs/os.conf - - cp -f clamav-unofficial-sigs.sh /usr/sbin/clamav-unofficial-sigs + - sudo mkdir -p /etc/clamav-unofficial-sigs + - sudo cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf + - sudo cp -f config/os/os.ubuntu.conf /etc/clamav-unofficial-sigs/os.conf + - sudo cp -f clamav-unofficial-sigs.sh /usr/sbin/clamav-unofficial-sigs script: - - sudo su - - sh -e .t/ci-shellcheck.sh - - sh -e .t/ci-clamav-download-default-databases.sh - #- sh -e .t/ci-clamav-download-default-databases-git.sh - - sh -e .t/ci-clamav-install-default.sh - - sh -e .t/ci-clamav-install-default-database.sh - - cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf - - sh -e .t/ci-test.sh + - sudo sh -e .t/ci-shellcheck.sh + - sudo sh -e .t/ci-clamav-download-default-databases.sh + #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh + - sudo sh -e .t/ci-clamav-install-default.sh + - sudo sh -e .t/ci-clamav-install-default-database.sh + - sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf + - sudo sh -e .t/ci-test.sh addons: code_climate: diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 74dbd276..0ce32f83 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -33,8 +33,8 @@ # Detect to make sure the entire script is avilable, fail if the script is missing contents if [ "$(tail -n 1 "${0}" | head -n 1 | cut -c 1-7)" != "exit \$?" ] ; then - echo "FATAL ERROR: Script is incomplete, please redownload" - exit 1 + echo "FATAL ERROR: Script is incomplete, please redownload" + exit 1 fi # Trap the keyboard interrupt (Ctrl + C) @@ -218,45 +218,45 @@ function xshok_user_group_exists() { # username groupname # type a will make a ** border # type n will make a ++ border function xshok_pretty_echo_and_log() { # "string" "repeating" "count" "type" - #detect if running under cron and silence - mystring="$1" - myrepeating="$2" - mycount="$3" - mytype="$4" - if [ "$comment_silence" != "yes" ] && [ "$force_verbose" != "yes" ]; then - if [ ! -t 1 ] ; then - comment_silence="yes" - fi - fi - # always show errors and alerts - if [ -z "$mytype" ] ; then - shopt -s nocasematch - if [[ "$mystring" =~ "ERROR:" ]] || [[ "$mystring" =~ "ERROR " ]] ; then - mytype="e" - elif [[ "$mystring" =~ "WARNING:" ]] || [[ "$mystring" =~ "WARNING " ]] ; then - mytype="w" - elif [[ "$mystring" =~ "ALERT:" ]] || [[ "$mystring" =~ "ALERT " ]] ; then - mytype="a" - elif [[ "$mystring" =~ "NOTICE:" ]] || [[ "$mystring" =~ "NOTICE " ]] ; then - mytype="n" - fi - fi - if [ "$mytype" == "e" ] || [ "$mytype" == "a" ] ; then - comment_silence="no" - fi - # Handle comments is not silenced or type + #detect if running under cron and silence + mystring="$1" + myrepeating="$2" + mycount="$3" + mytype="$4" + if [ "$comment_silence" != "yes" ] && [ "$force_verbose" != "yes" ]; then + if [ ! -t 1 ] ; then + comment_silence="yes" + fi + fi + # always show errors and alerts + if [ -z "$mytype" ] ; then + shopt -s nocasematch + if [[ "$mystring" =~ "ERROR:" ]] || [[ "$mystring" =~ "ERROR " ]] ; then + mytype="e" + elif [[ "$mystring" =~ "WARNING:" ]] || [[ "$mystring" =~ "WARNING " ]] ; then + mytype="w" + elif [[ "$mystring" =~ "ALERT:" ]] || [[ "$mystring" =~ "ALERT " ]] ; then + mytype="a" + elif [[ "$mystring" =~ "NOTICE:" ]] || [[ "$mystring" =~ "NOTICE " ]] ; then + mytype="n" + fi + fi + if [ "$mytype" == "e" ] || [ "$mytype" == "a" ] ; then + comment_silence="no" + fi + # Handle comments is not silenced or type if [ "$comment_silence" != "yes" ] ; then - if [ -z "$myrepeating" ] ; then - if [ "$mytype" == "e" ] ; then - myrepeating="=" - elif [ "$mytype" == "w" ] ; then - myrepeating="-" - elif [ "$mytype" == "a" ] ; then - myrepeating="*" - elif [ "$mytype" == "n" ] ; then - myrepeating="+" - fi - fi + if [ -z "$myrepeating" ] ; then + if [ "$mytype" == "e" ] ; then + myrepeating="=" + elif [ "$mytype" == "w" ] ; then + myrepeating="-" + elif [ "$mytype" == "a" ] ; then + myrepeating="*" + elif [ "$mytype" == "n" ] ; then + myrepeating="+" + fi + fi if [ -z "$myrepeating" ] ; then echo "${mystring}" else @@ -277,28 +277,28 @@ function xshok_pretty_echo_and_log() { # "string" "repeating" "count" "type" # Handle logging if [ "$enable_log" == "yes" ] ; then - #filter ===, --- - mystring=${1//===} - mystring=${mystring//---} - - if [ ! -z "$mystring" ] ; then - if [ ! -z "$log_pipe_cmd" ] ; then - echo "${mystring}" | $log_pipe_cmd - else - if [ ! -e "${log_file_path}/${log_file_name}" ] ; then - # xshok_mkdir_ownership "$log_file_path" - mkdir -p "$log_file_path" - touch "${log_file_path}/${log_file_name}" 2>/dev/null - perms chown -f "${clam_user}:${clam_group}" "${log_file_path}/${log_file_name}" - fi - if [ ! -w "${log_file_path}/${log_file_name}" ] ; then - echo "WARNING: Logging Disabled, as file not writable: ${log_file_path}/${log_file_name}" - enable_log="no" - else - echo "$(date "+%b %d %T")" "${mystring}" >> "${log_file_path}/${log_file_name}" - fi - fi - fi + #filter ===, --- + mystring=${1//===} + mystring=${mystring//---} + + if [ -n "$mystring" ] ; then + if [ -n "$log_pipe_cmd" ] ; then + echo "${mystring}" | $log_pipe_cmd + else + if [ ! -e "${log_file_path}/${log_file_name}" ] ; then + # xshok_mkdir_ownership "$log_file_path" + mkdir -p "$log_file_path" + touch "${log_file_path}/${log_file_name}" 2>/dev/null + perms chown -f "${clam_user}:${clam_group}" "${log_file_path}/${log_file_name}" + fi + if [ ! -w "${log_file_path}/${log_file_name}" ] ; then + echo "WARNING: Logging Disabled, as file not writable: ${log_file_path}/${log_file_name}" + enable_log="no" + else + echo "$(date "+%b %d %T")" "${mystring}" >> "${log_file_path}/${log_file_name}" + fi + fi + fi fi } @@ -329,55 +329,55 @@ function xshok_draw_time_remaining() { #time_remaining #update_hours #name # Download function function xshok_file_download() { #outputfile #url #notimestamp - if [ "$downloader_debug" == "yes" ] ; then - xshok_pretty_echo_and_log "url: ${2} >> outputfile: ${1} | ${3}" - fi + if [ "$downloader_debug" == "yes" ] ; then + xshok_pretty_echo_and_log "url: ${2} >> outputfile: ${1} | ${3}" + fi if [ "${1}" ] && [ "${2}" ] ; then - if [ -n "$curl_bin" ] ; then - if [ -f "${1}" ] ; then - # shellcheck disable=SC2086 - $curl_bin --fail --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" --time-cond "${1}" --output "${1}" "${2}" 2>&11 - result=$? - else - # shellcheck disable=SC2086 - $curl_bin --fail --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" --output "${1}" "${2}" 2>&11 - result=$? - fi - else - if [ ! "${3}" ] ; then - # the following is required because wget, cannot do --timestamping and --output-document together - this_dir="$(readlink -f "$0")" - output_file="$1" - url="$2" - output_dir="${output_file%/*}" - output_file="${output_file##*/}" - url_file="${url##*/}" - wget_output_link="" - - cd "${output_dir}" || exit - if [ "$output_file" != "$url_file" ] ; then - if [ ! -f "$url_file" ] ; then - if [ ! -f "$output_file" ] ; then - touch "$output_file" - fi - ln -s "$output_file" "$url_file" - wget_output_link="$url_file" - fi - fi - # shellcheck disable=SC2086 - $wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" --timestamping "${2}" 2>&12 - result=$? - if [ ! -n "$wget_output_link" ] ; then - if [ -L "$wget_output_link" ] ; then - rm -f "$wget_output_link" - fi - fi - cd "$this_dir" || exit - else - # shellcheck disable=SC2086 - $wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" --output-document="${1}" "${2}" 2>&12 - result=$? - fi + if [ -n "$curl_bin" ] ; then + if [ -f "${1}" ] ; then + # shellcheck disable=SC2086 + $curl_bin --fail --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" --time-cond "${1}" --output "${1}" "${2}" 2>&11 + result=$? + else + # shellcheck disable=SC2086 + $curl_bin --fail --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" --output "${1}" "${2}" 2>&11 + result=$? + fi + else + if [ ! "${3}" ] ; then + # the following is required because wget, cannot do --timestamping and --output-document together + this_dir="$(readlink -f "$0")" + output_file="$1" + url="$2" + output_dir="${output_file%/*}" + output_file="${output_file##*/}" + url_file="${url##*/}" + wget_output_link="" + + cd "${output_dir}" || exit + if [ "$output_file" != "$url_file" ] ; then + if [ ! -f "$url_file" ] ; then + if [ ! -f "$output_file" ] ; then + touch "$output_file" + fi + ln -s "$output_file" "$url_file" + wget_output_link="$url_file" + fi + fi + # shellcheck disable=SC2086 + $wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" --timestamping "${2}" 2>&12 + result=$? + if [ -z "$wget_output_link" ] ; then + if [ -L "$wget_output_link" ] ; then + rm -f "$wget_output_link" + fi + fi + cd "$this_dir" || exit + else + # shellcheck disable=SC2086 + $wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" --output-document="${1}" "${2}" 2>&12 + result=$? + fi fi return $result fi @@ -435,27 +435,27 @@ function xshok_database() { # rating database_array if [ "$db_name_rating" == "HIGHONLY" ] || [ "$db_name_rating" == "HIGH" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "LOW" ] || [ "$db_name_rating" == "MEDIUMHIGHONLY" ] ; then new_dbs+=( "$db_name" ) fi - elif [ "$current_rating" == "LOWONLY" ] ; then - if [ "$db_name_rating" == "LOWONLY" ] || [ "$db_name_rating" == "LOW" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "MEDIUMONLY" ] ; then - if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "LOWMEDIUMONLY" ] ; then - if [ "$db_name_rating" == "LOWMEDIUMONLY" ] || [ "$db_name_rating" == "LOW" ] || [ "$db_name_rating" == "MEDIUM" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "MEDIUMHIGHONLY" ] ; then - if [ "$db_name_rating" == "MEDIUMHIGHONLY" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "HIGH" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "HIGHONLY" ] ; then - if [ "$db_name_rating" == "HIGHONLY" ] || [ "$db_name_rating" == "HIGH" ] ; then - new_dbs+=( "$db_name" ) - fi - fi + elif [ "$current_rating" == "LOWONLY" ] ; then + if [ "$db_name_rating" == "LOWONLY" ] || [ "$db_name_rating" == "LOW" ] ; then + new_dbs+=( "$db_name" ) + fi + elif [ "$current_rating" == "MEDIUMONLY" ] ; then + if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] ; then + new_dbs+=( "$db_name" ) + fi + elif [ "$current_rating" == "LOWMEDIUMONLY" ] ; then + if [ "$db_name_rating" == "LOWMEDIUMONLY" ] || [ "$db_name_rating" == "LOW" ] || [ "$db_name_rating" == "MEDIUM" ] ; then + new_dbs+=( "$db_name" ) + fi + elif [ "$current_rating" == "MEDIUMHIGHONLY" ] ; then + if [ "$db_name_rating" == "MEDIUMHIGHONLY" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "HIGH" ] ; then + new_dbs+=( "$db_name" ) + fi + elif [ "$current_rating" == "HIGHONLY" ] ; then + if [ "$db_name_rating" == "HIGHONLY" ] || [ "$db_name_rating" == "HIGH" ] ; then + new_dbs+=( "$db_name" ) + fi + fi fi fi fi @@ -687,129 +687,129 @@ EOF # Auto upgrade the master.conf and the function xshok_upgrade() { - if [ "$allow_upgrades" == "no" ] ; then - xshok_pretty_echo_and_log "ERROR: --upgrade has been disabled, allow_upgrades=no" - exit 1 - fi - if ! xshok_is_root ; then - xshok_pretty_echo_and_log "ERROR: Only root can run the upgrade" - exit 1 - fi - - xshok_pretty_echo_and_log "Checking for updates ..." - - found_upgrade="no" - if [ -n "$curl_bin" ] ; then - # shellcheck disable=SC2086 - latest_version="$($curl_bin --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" 2>&11 | $grep_bin "^script_version=" | head -n1 | cut -d '"' -f 2)" - # shellcheck disable=SC2086 - latest_config_version="$($curl_bin --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" 2>&11 | $grep_bin "^config_version=" | head -n1 | cut -d '"' -f 2)" - else - # shellcheck disable=SC2086 - latest_version="$($wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" -O - 2>&12 | $grep_bin "^script_version=" | head -n1 | cut -d '"' -f 2)" - # shellcheck disable=SC2086 - latest_config_version="$($wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" -O - 2>&12 | $grep_bin "^config_version=" | head -n1 | cut -d '"' -f 2)" - fi + if [ "$allow_upgrades" == "no" ] ; then + xshok_pretty_echo_and_log "ERROR: --upgrade has been disabled, allow_upgrades=no" + exit 1 + fi + if ! xshok_is_root ; then + xshok_pretty_echo_and_log "ERROR: Only root can run the upgrade" + exit 1 + fi + + xshok_pretty_echo_and_log "Checking for updates ..." + + found_upgrade="no" + if [ -n "$curl_bin" ] ; then + # shellcheck disable=SC2086 + latest_version="$($curl_bin --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" 2>&11 | $grep_bin "^script_version=" | head -n1 | cut -d '"' -f 2)" + # shellcheck disable=SC2086 + latest_config_version="$($curl_bin --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" 2>&11 | $grep_bin "^config_version=" | head -n1 | cut -d '"' -f 2)" + else + # shellcheck disable=SC2086 + latest_version="$($wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" -O - 2>&12 | $grep_bin "^script_version=" | head -n1 | cut -d '"' -f 2)" + # shellcheck disable=SC2086 + latest_config_version="$($wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" -O - 2>&12 | $grep_bin "^config_version=" | head -n1 | cut -d '"' -f 2)" + fi # config_dir/master.conf - if [ "$latest_config_version" ] ; then - # shellcheck disable=SC2183,SC2086 - if [ "$(printf "%02d%02d%02d%02d" ${latest_config_version//./ })" -gt "$(printf "%02d%02d%02d%02d" ${config_version//./ })" ] ; then - found_upgrade="yes" - xshok_pretty_echo_and_log "ALERT: Upgrading config from v${config_version} to v${latest_config_version}" - if [ -w "${config_dir}/master.conf" ] && [ -f "${config_dir}/master.conf" ] ; then - echo "Downloading https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" - xshok_file_download "${work_dir}/master.conf.tmp" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" "notimestamp" - ret="$?" - if [ "$ret" -ne 0 ] ; then - xshok_pretty_echo_and_log "ERROR: Could not download https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" - exit 1 - fi - if ! $grep_bin -m 1 "config_version" "${work_dir}/master.conf.tmp" > /dev/null 2>&1 ; then - echo "ERROR: Downloaded master.conf is incomplete, please re-run" - exit 1 - fi - # Copy over permissions from old version - OCTAL_MODE="$(stat -c "%a" "${config_dir}/master.conf")" - xshok_pretty_echo_and_log "Running update process" - if ! mv -f "${work_dir}/master.conf.tmp" "${config_dir}/master.conf" ; then - xshok_pretty_echo_and_log "ERROR: failed moving ${work_dir}/master.conf.tmp to ${config_dir}/master.conf" - exit 1 - fi - if ! chmod "$OCTAL_MODE" "${config_dir}/master.conf" ; then - xshok_pretty_echo_and_log "ERROR: unable to set permissions on ${config_dir}/master.conf" - exit 1 - fi - xshok_pretty_echo_and_log "Completed" - else - xshok_pretty_echo_and_log "ERROR: ${config_dir}/master.conf is not a file or is not writable" - exit 1 - fi - fi - fi - - if [ "$latest_version" ] ; then - # shellcheck disable=SC2183,SC2086 - if [ "$(printf "%02d%02d%02d%02d" ${latest_version//./ })" -gt "$(printf "%02d%02d%02d%02d" ${script_version//./ })" ] ; then - found_upgrade="yes" - xshok_pretty_echo_and_log "ALERT: Upgrading script from v${script_version} to v${latest_version}" - if [ -w "${config_dir}/master.conf" ] && [ -f "${config_dir}/master.conf" ] ; then - echo "Downloading https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" - xshok_file_download "${work_dir}/clamav-unofficial-sigs.sh.tmp" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" "notimestamp" - ret=$? - if [ "$ret" -ne 0 ] ; then - xshok_pretty_echo_and_log "ERROR: Could not download https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" - exit 1 - fi - # Detect to make sure the entire script is avilable, fail if the script is missing contents - if [ "$(tail -n 1 "${work_dir}/clamav-unofficial-sigs.sh.tmp" | head -n 1 | cut -c 1-7)" != "exit \$?" ] ; then - echo "ERROR: Downloaded clamav-unofficial-sigs.sh is incomplete, please re-run" - exit 1 - fi - # Copy over permissions from old version - OCTAL_MODE="$(stat -c "%a" "${this_script_full_path}")" - - xshok_pretty_echo_and_log "Inserting update process..." - # Generate the update script - cat > "${work_dir}/xshok_update_script.sh" << EOF + if [ "$latest_config_version" ] ; then + # shellcheck disable=SC2183,SC2086 + if [ "$(printf "%02d%02d%02d%02d" ${latest_config_version//./ })" -gt "$(printf "%02d%02d%02d%02d" ${config_version//./ })" ] ; then + found_upgrade="yes" + xshok_pretty_echo_and_log "ALERT: Upgrading config from v${config_version} to v${latest_config_version}" + if [ -w "${config_dir}/master.conf" ] && [ -f "${config_dir}/master.conf" ] ; then + echo "Downloading https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" + xshok_file_download "${work_dir}/master.conf.tmp" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" "notimestamp" + ret="$?" + if [ "$ret" -ne 0 ] ; then + xshok_pretty_echo_and_log "ERROR: Could not download https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" + exit 1 + fi + if ! $grep_bin -m 1 "config_version" "${work_dir}/master.conf.tmp" > /dev/null 2>&1 ; then + echo "ERROR: Downloaded master.conf is incomplete, please re-run" + exit 1 + fi + # Copy over permissions from old version + OCTAL_MODE="$(stat -c "%a" "${config_dir}/master.conf")" + xshok_pretty_echo_and_log "Running update process" + if ! mv -f "${work_dir}/master.conf.tmp" "${config_dir}/master.conf" ; then + xshok_pretty_echo_and_log "ERROR: failed moving ${work_dir}/master.conf.tmp to ${config_dir}/master.conf" + exit 1 + fi + if ! chmod "$OCTAL_MODE" "${config_dir}/master.conf" ; then + xshok_pretty_echo_and_log "ERROR: unable to set permissions on ${config_dir}/master.conf" + exit 1 + fi + xshok_pretty_echo_and_log "Completed" + else + xshok_pretty_echo_and_log "ERROR: ${config_dir}/master.conf is not a file or is not writable" + exit 1 + fi + fi + fi + + if [ "$latest_version" ] ; then + # shellcheck disable=SC2183,SC2086 + if [ "$(printf "%02d%02d%02d%02d" ${latest_version//./ })" -gt "$(printf "%02d%02d%02d%02d" ${script_version//./ })" ] ; then + found_upgrade="yes" + xshok_pretty_echo_and_log "ALERT: Upgrading script from v${script_version} to v${latest_version}" + if [ -w "${config_dir}/master.conf" ] && [ -f "${config_dir}/master.conf" ] ; then + echo "Downloading https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" + xshok_file_download "${work_dir}/clamav-unofficial-sigs.sh.tmp" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" "notimestamp" + ret=$? + if [ "$ret" -ne 0 ] ; then + xshok_pretty_echo_and_log "ERROR: Could not download https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" + exit 1 + fi + # Detect to make sure the entire script is avilable, fail if the script is missing contents + if [ "$(tail -n 1 "${work_dir}/clamav-unofficial-sigs.sh.tmp" | head -n 1 | cut -c 1-7)" != "exit \$?" ] ; then + echo "ERROR: Downloaded clamav-unofficial-sigs.sh is incomplete, please re-run" + exit 1 + fi + # Copy over permissions from old version + OCTAL_MODE="$(stat -c "%a" "${this_script_full_path}")" + + xshok_pretty_echo_and_log "Inserting update process..." + # Generate the update script + cat > "${work_dir}/xshok_update_script.sh" << EOF #!/usr/bin/env bash echo "Running update process" # Overwrite old file with new if ! mv -f "${work_dir}/clamav-unofficial-sigs.sh.tmp" "${this_script_full_path}" ; then echo "ERROR: failed moving ${work_dir}/clamav-unofficial-sigs.sh.tmp to ${this_script_full_path}" rm -f \$0 - exit 1 + exit 1 fi if ! chmod "$OCTAL_MODE" "${this_script_full_path}" ; then - echo "ERROR: unable to set permissions on ${this_script_full_path}" - rm -f \$0 - exit 1 -fi - echo "Completed" - # echo "---------------------" - # echo "Optional, run as root: " - # echo "clamav-unofficial-sigs.sh --install-all" - echo "---------------------" - echo "Run once as root: " - echo "clamav-unofficial-sigs.sh --force" - - #remove the tmp script before exit - rm -f \$0 + echo "ERROR: unable to set permissions on ${this_script_full_path}" + rm -f \$0 + exit 1 +fi + echo "Completed" + # echo "---------------------" + # echo "Optional, run as root: " + # echo "clamav-unofficial-sigs.sh --install-all" + echo "---------------------" + echo "Run once as root: " + echo "clamav-unofficial-sigs.sh --force" + + #remove the tmp script before exit + rm -f \$0 EOF - # Replaced with $0, so code will update and then call itself with the same parameters it had - #exec "${0}" "$@" - bash_bin="$(command -v bash 2> /dev/null)" - exec "$bash_bin" "${work_dir}/xshok_update_script.sh" - echo "Running once as root" - else - xshok_pretty_echo_and_log "ERROR: ${config_dir}/master.conf is not a file or is not writable" - exit 1 - fi - fi + # Replaced with $0, so code will update and then call itself with the same parameters it had + #exec "${0}" "$@" + bash_bin="$(command -v bash 2> /dev/null)" + exec "$bash_bin" "${work_dir}/xshok_update_script.sh" + echo "Running once as root" + else + xshok_pretty_echo_and_log "ERROR: ${config_dir}/master.conf is not a file or is not writable" + exit 1 + fi + fi fi if [ "$found_upgrade" == "no" ] ; then - xshok_pretty_echo_and_log "No updates available" + xshok_pretty_echo_and_log "No updates available" fi } @@ -820,9 +820,9 @@ function decode_third_party_signature_by_signature_name() { xshok_pretty_echo_and_log "Input a third-party signature name to decode (e.g: Sanesecurity.Junk.15248) or" xshok_pretty_echo_and_log "a hexadecimal encoded data string and press enter:" read -r input - # Remove quotes and .UNOFFICIAL from the whitelist input string + # Remove quotes and .UNOFFICIAL from the whitelist input string input="$(echo "${input}" | tr -d "'" | tr -d '"' | tr -d '`')" - input=${input/\.UNOFFICIAL/} + input=${input/\.UNOFFICIAL/} if echo "${input}" | $grep_bin "\\." > /dev/null ; then cd "$clam_dbs" || exit sig="$($grep_bin "${input}:" ./*.ndb)" @@ -904,8 +904,8 @@ function output_system_configuration_information() { xshok_pretty_echo_and_log "" xshok_pretty_echo_and_log "*** SCRIPT INFORMATION ***" xshok_pretty_echo_and_log "${this_script_name} ${script_version} (${script_version_date})" - xshok_pretty_echo_and_log "Master.conf Version: ${config_version}" - xshok_pretty_echo_and_log "Minimum required config: ${minimum_required_config_version}" + xshok_pretty_echo_and_log "Master.conf Version: ${config_version}" + xshok_pretty_echo_and_log "Minimum required config: ${minimum_required_config_version}" xshok_pretty_echo_and_log "*** SYSTEM INFORMATION ***" $uname_bin -a xshok_pretty_echo_and_log "*** CLAMSCAN LOCATION & VERSION ***" @@ -915,13 +915,13 @@ function output_system_configuration_information() { xshok_pretty_echo_and_log "${rsync_bin}" $rsync_bin --version | head -1 if [ -n "$curl_bin" ] ; then - xshok_pretty_echo_and_log "*** CURL LOCATION & VERSION ***" - xshok_pretty_echo_and_log "${curl_bin}" - $curl_bin --version | head -1 + xshok_pretty_echo_and_log "*** CURL LOCATION & VERSION ***" + xshok_pretty_echo_and_log "${curl_bin}" + $curl_bin --version | head -1 else - xshok_pretty_echo_and_log "*** WGET LOCATION & VERSION ***" - xshok_pretty_echo_and_log "${wget_bin}" - $wget_bin --version | head -1 + xshok_pretty_echo_and_log "*** WGET LOCATION & VERSION ***" + xshok_pretty_echo_and_log "${wget_bin}" + $wget_bin --version | head -1 fi if [ "$enable_gpg" == "yes" ] ; then xshok_pretty_echo_and_log "*** GPG LOCATION & VERSION ***" @@ -941,7 +941,7 @@ function output_system_configuration_information() { else xshok_pretty_echo_and_log "Configuration Directory: ${config_dir}" fi - xshok_pretty_echo_and_log "" + xshok_pretty_echo_and_log "" } # Make a signature database from an ascii file @@ -1181,29 +1181,29 @@ function output_signatures_triggered_during_ham_directory_scan() { # Adds a signature whitelist entry in the newer ClamAV IGN2 format function add_signature_whitelist_entry() { #signature xshok_pretty_echo_and_log "Signature Whitelist" "=" - if [ -n "$1" ] ; then - input="$1" - else - xshok_pretty_echo_and_log "Input a third-party signature name that you wish to whitelist and press enter" - read -r input - fi + if [ -n "$1" ] ; then + input="$1" + else + xshok_pretty_echo_and_log "Input a third-party signature name that you wish to whitelist and press enter" + read -r input + fi if [ -n "$input" ] ; then - xshok_pretty_echo_and_log "Processing: ${input}" + xshok_pretty_echo_and_log "Processing: ${input}" cd "$clam_dbs" || exit - # Remove quotes and .UNOFFICIAL from the string - input="$(echo "${input}" | tr -d "'" | tr -d '"' | tr -d '`"')" - input=${input/\.UNOFFICIAL/} + # Remove quotes and .UNOFFICIAL from the string + input="$(echo "${input}" | tr -d "'" | tr -d '"' | tr -d '`"')" + input=${input/\.UNOFFICIAL/} sig_full="$($grep_bin -H -m 1 "$input" ./*.*db)" - sig_extension=${sig_full%%\:*} - sig_extension=${sig_extension##*\.} - shopt -s nocasematch - if [ "$sig_extension" == "hdb" ] || [ "$sig_extension" == "hsb" ] || [ "$sig_extension" == "hdu " ] || [ "$sig_extension" == "hsu" ] || [ "$sig_extension" == "mdb" ] || [ "$sig_extension" == "msb" ] || [ "$sig_extension" == "mdu" ] || [ "$sig_extension" == "msu" ] ; then - # Hash-based Signature Database - position="4" - else - position="2" - fi + sig_extension=${sig_full%%\:*} + sig_extension=${sig_extension##*\.} + shopt -s nocasematch + if [ "$sig_extension" == "hdb" ] || [ "$sig_extension" == "hsb" ] || [ "$sig_extension" == "hdu " ] || [ "$sig_extension" == "hsu" ] || [ "$sig_extension" == "mdb" ] || [ "$sig_extension" == "msb" ] || [ "$sig_extension" == "mdu" ] || [ "$sig_extension" == "msu" ] ; then + # Hash-based Signature Database + position="4" + else + position="2" + fi sig_name="$(echo "$sig_full" | cut -d ":" -f $position | cut -d "=" -f 1)" if [ -n "$sig_name" ] ; then @@ -1224,7 +1224,7 @@ function add_signature_whitelist_entry() { #signature if [ "$selinux_fixes" == "yes" ] ; then restorecon "${clam_dbs}/local.ign" fi - do_clamd_reload="4" + do_clamd_reload="4" clamscan_reload_dbs xshok_pretty_echo_and_log "Signature '${input}' has been added to my-whitelist.ign2 and" @@ -1377,36 +1377,36 @@ function check_clamav() { # Check for a new version function check_new_version() { - found_upgrade="no" + found_upgrade="no" if [ -n "$curl_bin" ] ; then - # shellcheck disable=SC2086 - latest_version="$($curl_bin --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" 2>&11 | $grep_bin "^script_version=" | head -n1 | cut -d '"' -f 2)" - # shellcheck disable=SC2086 - latest_config_version="$($curl_bin --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" 2>&11 | $grep_bin "^config_version=" | head -n1 | cut -d '"' -f 2)" - else - # shellcheck disable=SC2086 - latest_version="$($wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" -O - 2>&12 | $grep_bin "^script_version=" | head -n1 | cut -d '"' -f 2)" - # shellcheck disable=SC2086 - latest_config_version="$($wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" -O - 2>&12 | $grep_bin "^config_version=" | head -n1 | cut -d '"' -f 2)" - fi + # shellcheck disable=SC2086 + latest_version="$($curl_bin --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" 2>&11 | $grep_bin "^script_version=" | head -n1 | cut -d '"' -f 2)" + # shellcheck disable=SC2086 + latest_config_version="$($curl_bin --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" 2>&11 | $grep_bin "^config_version=" | head -n1 | cut -d '"' -f 2)" + else + # shellcheck disable=SC2086 + latest_version="$($wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/clamav-unofficial-sigs.sh" -O - 2>&12 | $grep_bin "^script_version=" | head -n1 | cut -d '"' -f 2)" + # shellcheck disable=SC2086 + latest_config_version="$($wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/${git_branch}/config/master.conf" -O - 2>&12 | $grep_bin "^config_version=" | head -n1 | cut -d '"' -f 2)" + fi if [ "$latest_version" ] ; then - # shellcheck disable=SC2183,SC2086 - if [ "$(printf "%02d%02d%02d%02d" ${latest_version//./ })" -gt "$(printf "%02d%02d%02d%02d" ${script_version//./ })" ] ; then + # shellcheck disable=SC2183,SC2086 + if [ "$(printf "%02d%02d%02d%02d" ${latest_version//./ })" -gt "$(printf "%02d%02d%02d%02d" ${script_version//./ })" ] ; then xshok_pretty_echo_and_log "ALERT: New version : v${latest_version} @ https://github.com/extremeshok/clamav-unofficial-sigs" - found_upgrade="yes" + found_upgrade="yes" fi fi if [ "$latest_config_version" ] ; then - # shellcheck disable=SC2183,SC2086 - if [ "$(printf "%02d%02d%02d%02d" ${latest_config_version//./ })" -gt "$(printf "%02d%02d%02d%02d" ${config_version//./ })" ] ; then + # shellcheck disable=SC2183,SC2086 + if [ "$(printf "%02d%02d%02d%02d" ${latest_config_version//./ })" -gt "$(printf "%02d%02d%02d%02d" ${config_version//./ })" ] ; then xshok_pretty_echo_and_log "ALERT: New config version : v${latest_config_version} @ https://github.com/extremeshok/clamav-unofficial-sigs" - found_upgrade="yes" + found_upgrade="yes" fi fi if [ "$found_upgrade" == "yes" ] && [ "$allow_upgrades" == "yes" ] ; then - xshok_pretty_echo_and_log "Quickly upgrade, run the following command as root:" - xshok_pretty_echo_and_log "${this_script_name} --upgrade" + xshok_pretty_echo_and_log "Quickly upgrade, run the following command as root:" + xshok_pretty_echo_and_log "${this_script_name} --upgrade" fi } @@ -1508,7 +1508,7 @@ this_script_full_path="${BASH_SOURCE[0]}" while [ -h "$this_script_full_path" ]; do this_script_path="$( cd -P "$( dirname "$this_script_full_path" )" >/dev/null 2>&1 && readlink -f "$0" )" this_script_full_path="$(readlink "$this_script_full_path")" - # if relative symlink, then resolve the path + # if relative symlink, then resolve the path if [[ $this_script_full_path != /* ]] ; then this_script_full_path="$this_script_path/$this_script_full_path" fi @@ -1517,8 +1517,8 @@ this_script_path="$( cd -P "$( dirname "$this_script_full_path" )" >/dev/null 2> this_script_name="$(basename "$this_script_full_path")" if [ -z "$this_script_full_path" ] || [ -z "$this_script_path" ] || [ -z "$this_script_name" ] ; then - echo "ERROR: could not determin script name and fullpath" - exit 1 + echo "ERROR: could not determin script name and fullpath" + exit 1 fi #allow for other negatives besides no. @@ -1557,33 +1557,33 @@ else fi # Default config files if [ -r "${config_dir}/master.conf" ] ; then - config_files+=( "${config_dir}/master.conf" ) + config_files+=( "${config_dir}/master.conf" ) else - xshok_pretty_echo_and_log "ERROR: ${config_dir}/master.conf is not readable" - exit 1 + xshok_pretty_echo_and_log "ERROR: ${config_dir}/master.conf is not readable" + exit 1 fi if [ -r "${config_dir}/os.conf" ] ; then - config_files+=( "${config_dir}/os.conf" ) + config_files+=( "${config_dir}/os.conf" ) else - #find the a suitable os.*.conf file - os_config_number=$(find "$config_dir" -type f -iname "os.*.conf" | wc -l) - if [ "$os_config_number" == "0" ] ; then - xshok_pretty_echo_and_log "WARNING: no os.conf or os.*.conf found" - elif [ "$os_config_number" == "1" ] ; then - config_file="$(find "$config_dir" -type f -iname "os.*.conf" | head -n1)" - if [ -r "${config_file}" ]; then - config_files+=( "${config_file}" ) - else - xshok_pretty_echo_and_log "WARNING: ${config_file} is not readable" - fi - else - xshok_pretty_echo_and_log "WARNING: Too many os.*.conf configs found" - fi + #find the a suitable os.*.conf file + os_config_number=$(find "$config_dir" -type f -iname "os.*.conf" | wc -l) + if [ "$os_config_number" == "0" ] ; then + xshok_pretty_echo_and_log "WARNING: no os.conf or os.*.conf found" + elif [ "$os_config_number" == "1" ] ; then + config_file="$(find "$config_dir" -type f -iname "os.*.conf" | head -n1)" + if [ -r "${config_file}" ]; then + config_files+=( "${config_file}" ) + else + xshok_pretty_echo_and_log "WARNING: ${config_file} is not readable" + fi + else + xshok_pretty_echo_and_log "WARNING: Too many os.*.conf configs found" + fi fi if [ -r "${config_dir}/user.conf" ] ; then - config_files+=( "${config_dir}/user.conf" ) + config_files+=( "${config_dir}/user.conf" ) else - xshok_pretty_echo_and_log "WARNING: ${config_dir}/user.conf is not readable" + xshok_pretty_echo_and_log "WARNING: ${config_dir}/user.conf is not readable" fi # Solaris command -v function returns garbage when the program is not found @@ -1609,26 +1609,26 @@ fi # Detect support for tar or gtar if [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then - tar_executable="gtar" + tar_executable="gtar" else - tar_executable="tar" + tar_executable="tar" fi if [ -z "$tar_bin" ]; then - tar_bin="$(command -v "$tar_executable" 2> /dev/null)" - else - xshok_pretty_echo_and_log "ERROR: gtar (gnu tar) is missing" + tar_bin="$(command -v "$tar_executable" 2> /dev/null)" + else + xshok_pretty_echo_and_log "ERROR: gtar (gnu tar) is missing" fi # Detect support for curl if [ -z "$curl_bin" ]; then - curl_bin="$(command -v curl 2> /dev/null)" + curl_bin="$(command -v curl 2> /dev/null)" fi # Detect support for wget if [ -z "$wget_bin" ]; then - if [ -x /usr/sfw/bin/wget ] ; then - wget_bin="/usr/sfw/bin/wget" - else - wget_bin="$(command -v wget 2> /dev/null)" - fi + if [ -x /usr/sfw/bin/wget ] ; then + wget_bin="/usr/sfw/bin/wget" + else + wget_bin="$(command -v wget 2> /dev/null)" + fi fi if [ -z "$wget_bin" ] && [ -z "$curl_bin" ]; then curl_bin="$(command -v curl 2> /dev/null)" @@ -1638,7 +1638,7 @@ if [ -z "$wget_bin" ] && [ -z "$curl_bin" ]; then fi fi -if [ ! -z "$wget_bin" ] ; then +if [ -n "$wget_bin" ] ; then # wget compression support if $wget_bin --help | $grep_bin -q "compression=TYPE" ; then wget_compression="--compression=auto" @@ -1648,7 +1648,7 @@ if [ ! -z "$wget_bin" ] ; then fi # Detect support for dig or host dig_bin="$(command -v dig 2> /dev/null)" -if [ -z "$dig_bin" ] ; then +if [ -n "$dig_bin" ] ; then host_bin="$(command -v host 2> /dev/null)" if [ -z "$host_bin" ] ; then xshok_pretty_echo_and_log "ERROR: both dig and host commands are missing, One of them is required" @@ -1721,24 +1721,24 @@ if [ "$custom_config" != "no" ] ; then if [ -d "$custom_config" ] ; then # Assign the custom config dir and remove trailing / (removes / and //) shopt -s extglob; config_dir="${custom_config%%+(/)}" - config_files=() - if [ -r "${config_dir}/master.conf" ] ; then - config_files+=( "${config_dir}/master.conf" ) - else - xshok_pretty_echo_and_log "WARNING: ${config_dir}/master.conf not found" - fi - #find the a suitable os.conf or os.*.conf file - config_file="$(find "$config_dir" -type f -iname "os.conf" -o -iname "os.*.conf" | tail -n1)" - if [ -r "${config_file}" ] ; then - config_files+=( "${config_file}" ) - else - xshok_pretty_echo_and_log "WARNING: ${config_dir}/os.conf not found" - fi - if [ -r "${config_dir}/user.conf" ] ; then - config_files+=( "${config_dir}/user.conf" ) - else - xshok_pretty_echo_and_log "WARNING: ${config_dir}/user.conf not found" - fi + config_files=() + if [ -r "${config_dir}/master.conf" ] ; then + config_files+=( "${config_dir}/master.conf" ) + else + xshok_pretty_echo_and_log "WARNING: ${config_dir}/master.conf not found" + fi + #find the a suitable os.conf or os.*.conf file + config_file="$(find "$config_dir" -type f -iname "os.conf" -o -iname "os.*.conf" | tail -n1)" + if [ -r "${config_file}" ] ; then + config_files+=( "${config_file}" ) + else + xshok_pretty_echo_and_log "WARNING: ${config_dir}/os.conf not found" + fi + if [ -r "${config_dir}/user.conf" ] ; then + config_files+=( "${config_dir}/user.conf" ) + else + xshok_pretty_echo_and_log "WARNING: ${config_dir}/user.conf not found" + fi else config_files=( "$custom_config" ) fi @@ -1755,18 +1755,24 @@ for config_file in "${config_files[@]}" ; do clean_config="$(command sed -e '/^#.*/d' "$config_file")" # Comment line #clean_config="$(echo "$clean_config" | sed -e 's/#[[:space:]].*//')" # Comment line (duplicated) clean_config=${clean_config//\#*/} # Comment line (duplicated) + # shellcheck disable=SC2001 clean_config="$(echo "$clean_config" | sed -e '/^[[:blank:]]*#/d;s/#.*//')" # Comments at end of line #clean_config="$(echo "$clean_config" | sed -e 's/^[ \t]*//;s/[ \t]*$//')" # trailing and leading whitespace clean_config="$(echo "$clean_config" | xargs)" + # shellcheck disable=SC2001 clean_config="$(echo "$clean_config" | sed -e '/^\s*$/d')" # Blank lines elif [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then # MacOS / OSX / BSD fixes, had issues with running with a single command and with SunOS work around.. + # shellcheck disable=SC2001 clean_config="$(command sed -e '/^#.*/d' "$config_file")" # Comment line + # shellcheck disable=SC2001 clean_config="$(echo "$clean_config" | sed -e 's/#[[:space:]].*//')" # Comment line (duplicated) + # shellcheck disable=SC2001 clean_config="$(echo "$clean_config" | sed -e '/^[[:blank:]]*#/d;s/#.*//')" # Comments at end of line #clean_config="$(echo "$clean_config" | sed -e 's/^[ \t]*//;s/[ \t]*$//')" # trailing and leading whitespace #clean_config="$(echo "$clean_config" | xargs)" + # shellcheck disable=SC2001 clean_config="$(echo "$clean_config" | sed -e '/^\s*$/d')" # Blank lines else @@ -1931,9 +1937,9 @@ fi shopt -s extglob; clam_dbs="${clam_dbs%%+(/)}" # Force wget over curl. -if [ ! -z "$wget_bin" ] && [ "$force_wget" == "yes" ] ; then - xshok_pretty_echo_and_log "NOTICE: Forcing wget" - curl_bin="" +if [ -n "$wget_bin" ] && [ "$force_wget" == "yes" ] ; then + xshok_pretty_echo_and_log "NOTICE: Forcing wget" + curl_bin="" fi # SANITY checks @@ -1949,9 +1955,9 @@ if [ -z "$uname_bin" ] ; then exit 1 fi if [ -z "$clamscan_bin" ] ; then - if [ "${1}" != "--remove-script" ] ; then - xshok_pretty_echo_and_log "ERROR: clamscan binary (clamscan_bin) not found" - fi + if [ "${1}" != "--remove-script" ] ; then + xshok_pretty_echo_and_log "ERROR: clamscan binary (clamscan_bin) not found" + fi exit 1 fi if [ -z "$rsync_bin" ] ; then @@ -2055,35 +2061,35 @@ if $rsync_bin --help | $grep_bin -q "contimeout" > /dev/null ; then fi if [ "$debug" == "yes" ] ; then - downloader_debug="yes" - clamscan_debug="yes" - curl_debug="yes" - wget_debug="yes" - rsync_debug="yes" + downloader_debug="yes" + clamscan_debug="yes" + curl_debug="yes" + wget_debug="yes" + rsync_debug="yes" fi # Show clamscan errors if [ "$clamscan_debug" == "yes" ] ; then - exec 10>&2 + exec 10>&2 else - exec 10>/dev/null + exec 10>/dev/null fi # Show curl errors if [ "$curl_debug" == "yes" ] ; then - exec 11>&2 + exec 11>&2 else - exec 11>/dev/null + exec 11>/dev/null fi # Show wget errors if [ "$wget_debug" == "yes" ] ; then - exec 12>&2 + exec 12>&2 else - exec 12>/dev/null + exec 12>/dev/null fi # Show rsync errors if [ "$rsync_debug" == "yes" ] ; then - exec 13>&2 + exec 13>&2 else - exec 13>/dev/null + exec 13>/dev/null fi # Silence wget output and only report errors - useful if script is run via cron. @@ -2174,11 +2180,11 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then else temp_db="$(xshok_database "$default_dbs_rating" "${sanesecurity_dbs[@]}")" fi - sanesecurity_dbs=( ) - if [ ! -z "$temp_db" ] ; then - #sanesecurity_dbs=( $temp_db ) - read -r -a sanesecurity_dbs <<< "$temp_db" - fi + sanesecurity_dbs=( ) + if [ -n "$temp_db" ] ; then + #sanesecurity_dbs=( $temp_db ) + read -r -a sanesecurity_dbs <<< "$temp_db" + fi fi fi if [ "$securiteinfo_enabled" == "yes" ] ; then @@ -2188,11 +2194,11 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then else temp_db="$(xshok_database "$default_dbs_rating" "${securiteinfo_dbs[@]}")" fi - securiteinfo_dbs=( ) - if [ ! -z "$temp_db" ] ; then - #securiteinfo_dbs=( $temp_db ) - read -r -a securiteinfo_dbs <<< "$temp_db" - fi + securiteinfo_dbs=( ) + if [ -n "$temp_db" ] ; then + #securiteinfo_dbs=( $temp_db ) + read -r -a securiteinfo_dbs <<< "$temp_db" + fi fi fi if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then @@ -2202,11 +2208,11 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then else temp_db="$(xshok_database "$default_dbs_rating" "${linuxmalwaredetect_dbs[@]}")" fi - linuxmalwaredetect_dbs=( ) - if [ ! -z "$temp_db" ] ; then - #linuxmalwaredetect_dbs=( $temp_db ) - read -r -a linuxmalwaredetect_dbs <<< "$temp_db" - fi + linuxmalwaredetect_dbs=( ) + if [ -n "$temp_db" ] ; then + #linuxmalwaredetect_dbs=( $temp_db ) + read -r -a linuxmalwaredetect_dbs <<< "$temp_db" + fi fi fi if [ "$yararulesproject_enabled" == "yes" ] ; then @@ -2217,10 +2223,10 @@ if [ "$yararulesproject_enabled" == "yes" ] ; then temp_db="$(xshok_database "$default_dbs_rating" "${yararulesproject_dbs[@]}")" fi yararulesproject_dbs=( ) - if [ ! -z "$temp_db" ] ; then - #yararulesproject_dbs=( $temp_db ) - read -r -a yararulesproject_dbs <<< "$temp_db" - fi + if [ -n "$temp_db" ] ; then + #yararulesproject_dbs=( $temp_db ) + read -r -a yararulesproject_dbs <<< "$temp_db" + fi fi fi if [ "$urlhaus_enabled" == "yes" ] ; then @@ -2231,16 +2237,16 @@ if [ "$urlhaus_enabled" == "yes" ] ; then temp_db="$(xshok_database "$default_dbs_rating" "${urlhaus_dbs[@]}")" fi urlhaus_dbs=( ) - if [ ! -z "$temp_db" ] ; then - #urlhaus_dbs=( $temp_db ) - read -r -a urlhaus_dbs <<< "$temp_db" - fi + if [ -n "$temp_db" ] ; then + #urlhaus_dbs=( $temp_db ) + read -r -a urlhaus_dbs <<< "$temp_db" + fi fi fi # Set the variables for MalwarePatrol if [ "$malwarepatrol_product_code" != "8" ] ; then - # assumption, free product code is always 8 (non-free product code is never 8) - malwarepatrol_free="no" + # assumption, free product code is always 8 (non-free product code is never 8) + malwarepatrol_free="no" fi if [ "$malwarepatrol_free" == "yes" ] ; then malwarepatrol_product_code="8" @@ -2255,7 +2261,7 @@ else fi fi if [ -z "$malwarepatrol_db" ] ; then - malwarepatrol_db="malwarepatrol.db" + malwarepatrol_db="malwarepatrol.db" fi malwarepatrol_url="${malwarepatrol_url}?receipt=${malwarepatrol_receipt_code}&product=${malwarepatrol_product_code}&list=${malwarepatrol_list}" @@ -2367,7 +2373,7 @@ if [ ! -s "${work_dir_work_configs}/scan-test.txt" ] ; then fi if [ -z "$git_branch" ] ; then - git_branch="master" + git_branch="master" fi # If rsync proxy is defined in the config file, then export it for use. @@ -2528,29 +2534,29 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then echo "$current_time" > "${work_dir_work_configs}/last-ss-update.txt" xshok_pretty_echo_and_log "Sanesecurity Database & GPG Signature File Updates" "=" xshok_pretty_echo_and_log "Checking for Sanesecurity updates..." - if [ -z "$dig_bin" ] ; then - # shellcheck disable=SC2086 - sanesecurity_mirror_ips="$($dig_bin $dig_proxy +ignore +short "$sanesecurity_url")" - else - sanesecurity_mirror_ips="" - fi - # Add fallback to host if dig returns no records or dig is not used + if [ -n "$dig_bin" ] ; then + # shellcheck disable=SC2086 + sanesecurity_mirror_ips="$($dig_bin $dig_proxy +ignore +short "$sanesecurity_url")" + else + sanesecurity_mirror_ips="" + fi + # Add fallback to host if dig returns no records or dig is not used if [ ${#sanesecurity_mirror_ips} -lt 1 ] ; then - # shellcheck disable=SC2086 + # shellcheck disable=SC2086 sanesecurity_mirror_ips="$($host_bin $host_proxy -t A "$sanesecurity_url" | sed -n '/has address/{s/.*address \([^ ]*\).*/\1/;p;}')" fi if [ ${#sanesecurity_mirror_ips} -ge 1 ] ; then for sanesecurity_mirror_ip in $sanesecurity_mirror_ips ; do - if [ -z "$dig_bin" ] ; then - # shellcheck disable=SC2086 - sanesecurity_mirror_name="$($dig_bin $dig_proxy +short -x "$sanesecurity_mirror_ip" | command sed 's/\.$//')" - else - sanesecurity_mirror_name="" - fi + if [ -n "$dig_bin" ] ; then + # shellcheck disable=SC2086 + sanesecurity_mirror_name="$($dig_bin $dig_proxy +short -x "$sanesecurity_mirror_ip" | command sed 's/\.$//')" + else + sanesecurity_mirror_name="" + fi # Add fallback to host if dig returns no records or dig is not used if [ -z "$sanesecurity_mirror_name" ] ; then - # shellcheck disable=SC2086 + # shellcheck disable=SC2086 sanesecurity_mirror_name="$($host_bin $host_proxy "$sanesecurity_mirror_ip" | sed -n '/name pointer/{s/.*pointer \([^ ]*\).*\.$/\1/;p;}')" fi sanesecurity_mirror_site_info="$sanesecurity_mirror_name $sanesecurity_mirror_ip" @@ -2864,7 +2870,7 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then xshok_file_download "${work_dir_linuxmalwaredetect}/sigpack.tgz" "${linuxmalwaredetect_sigpack_url}" ret="$?" if [ "$ret" -eq 0 ] ; then - # shellcheck disable=SC2035 + # shellcheck disable=SC2035 $tar_bin --strip-components=1 --wildcards --overwrite -xzf "${work_dir_linuxmalwaredetect}/sigpack.tgz" --directory "${work_dir_linuxmalwaredetect}" */rfxn.* for db_file in "${linuxmalwaredetect_dbs[@]}" ; do if [ "$loop" == "1" ] ; then @@ -3688,21 +3694,21 @@ xshok_pretty_echo_and_log "Issue tracker : https://github.com/extremeshok/clamav if [ "$allow_update_checks" != "no" ] ; then - if [ -r "${work_dir_work_configs}/last-version-check.txt" ] ; then - last_version_check="$(cat "${work_dir_work_configs}/last-version-check.txt")" - else - last_version_check="0" - fi - db_file="" - update_check_interval="$((update_check_hours * 3600))" - time_interval="$((current_time - last_version_check))" - if [ "$time_interval" -ge $((update_check_interval - 600)) ] ; then - echo "$current_time" > "${work_dir_work_configs}/last-version-check.txt" - if xshok_is_root ; then - perms chown -f "${clam_user}:${clam_group}" "${work_dir_work_configs}/last-version-check.txt" - fi - check_new_version - fi + if [ -r "${work_dir_work_configs}/last-version-check.txt" ] ; then + last_version_check="$(cat "${work_dir_work_configs}/last-version-check.txt")" + else + last_version_check="0" + fi + db_file="" + update_check_interval="$((update_check_hours * 3600))" + time_interval="$((current_time - last_version_check))" + if [ "$time_interval" -ge $((update_check_interval - 600)) ] ; then + echo "$current_time" > "${work_dir_work_configs}/last-version-check.txt" + if xshok_is_root ; then + perms chown -f "${clam_user}:${clam_group}" "${work_dir_work_configs}/last-version-check.txt" + fi + check_new_version + fi fi @@ -3710,15 +3716,15 @@ xshok_cleanup # Set the permission of the log file, to fix any permission errors, this is done to fix cron errors after running the script as root. if xshok_is_root ; then - if [ "$enable_log" == "yes" ] ; then - # check if the file is owned by root (the current user) - if [ -O "${log_file_path}/${log_file_name}" ] ; then - # checks the file is writable and a file (not a symlink/link) - if [ -w "${log_file_path}/${log_file_name}" ] && [ -f "${log_file_path}/${log_file_name}" ] ; then - perms chown -f "${clam_user}:${clam_group}" "${log_file_path}/${log_file_name}" - fi - fi - fi + if [ "$enable_log" == "yes" ] ; then + # check if the file is owned by root (the current user) + if [ -O "${log_file_path}/${log_file_name}" ] ; then + # checks the file is writable and a file (not a symlink/link) + if [ -w "${log_file_path}/${log_file_name}" ] && [ -f "${log_file_path}/${log_file_name}" ] ; then + perms chown -f "${clam_user}:${clam_group}" "${log_file_path}/${log_file_name}" + fi + fi + fi fi # And lastly we exit, Note: the exit is always on the 2nd last line From c099c049071651d9b0d7a78c22af938dfe347338 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 23:41:43 +0200 Subject: [PATCH 028/101] Fix Commit 7385592a and bug #299 --- clamav-unofficial-sigs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 0ce32f83..3e34f68b 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -345,8 +345,8 @@ function xshok_file_download() { #outputfile #url #notimestamp fi else if [ ! "${3}" ] ; then - # the following is required because wget, cannot do --timestamping and --output-document together - this_dir="$(readlink -f "$0")" + # the following is required because wget, cannot do --timestamping and --output-document together + this_dir="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" output_file="$1" url="$2" output_dir="${output_file%/*}" @@ -1506,14 +1506,14 @@ minimum_yara_clamav_version="0.100" this_script_full_path="${BASH_SOURCE[0]}" # follow the symlinks while [ -h "$this_script_full_path" ]; do - this_script_path="$( cd -P "$( dirname "$this_script_full_path" )" >/dev/null 2>&1 && readlink -f "$0" )" + this_script_path="$( cd -P "$( dirname "$this_script_full_path" )" >/dev/null 2>&1 && pwd )" this_script_full_path="$(readlink "$this_script_full_path")" # if relative symlink, then resolve the path if [[ $this_script_full_path != /* ]] ; then this_script_full_path="$this_script_path/$this_script_full_path" fi done -this_script_path="$( cd -P "$( dirname "$this_script_full_path" )" >/dev/null 2>&1 && readlink -f "$0" )" +this_script_path="$( cd -P "$( dirname "$this_script_full_path" )" >/dev/null 2>&1 && pwd )" this_script_name="$(basename "$this_script_full_path")" if [ -z "$this_script_full_path" ] || [ -z "$this_script_path" ] || [ -z "$this_script_name" ] ; then From cfc5a395702d007da01a7e2c54028e6dec845756 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Tue, 1 Dec 2020 23:56:18 +0200 Subject: [PATCH 029/101] fix travis failing --- .t/ci-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index 2d94ca31..9cd45d02 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -35,7 +35,7 @@ else fi echo "running script as clamav and silence" --u clamav [ -x /usr/sbin/clamav-unofficial-sigs ] && bash /usr/sbin/clamav-unofficial-sigs --force --silence +sudo -u clamav [ -x /usr/sbin/clamav-unofficial-sigs ] && bash /usr/sbin/clamav-unofficial-sigs --force --silence if [ "$?" -eq "0" ] ; then echo .. OK else From cd5b8baef32f38061f7c8576588f7808292dcf90 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 01:07:02 +0200 Subject: [PATCH 030/101] Reworked securiteinfo premium databases (non-free) --- README.md | 26 ++++++++++++------- clamav-unofficial-sigs.sh | 53 ++++++++++++++++++++++++++++----------- config/master.conf | 51 ++++++++++++++++++++++++++----------- config/user.conf | 6 +++++ guides/macos.md | 2 +- 5 files changed, 99 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 4be6c375..5432a585 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,12 @@ Usage of free Yara-Rules Project: http://yararules.com Current limitations of clamav support : http://blog.clamav.net/search/label/yara +### malware.experrt non-free database support (as of December 2020) +Usage of malware.expert : https://www.malware.expert + - 1. Sign up for an account : https://www.malware.expert + - 2. You will recieve an email containing your serial key/number + - 3. Enter the serial key into the config malwareexpert_serial_key: replacing YOUR-SERIAL-KEY with your serial key from the email + ### MalwarePatrol Free/Delayed list support (as of May 2015) Usage of MalwarePatrol 2015 free clamav signatures : https://www.malwarepatrol.net - 1. Sign up for a free account : https://www.malwarepatrol.net/free-guard-upgrade-option/ @@ -199,25 +205,27 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] its associated files and databases from the system ## Change Log +### Version 7.2 ( XX December 2020 : Under Development) + - Added Malware.expert databases (non-free) + - Reworked securiteinfo premium databases (non-free) + - Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) + - Added detection of tar executable (use gtar on mac and bsd) + - Config os.macosx.conf renamed to os.macos.conf + - Fix: set ownership of last-version-check.txt + - Incremented the config to version 93 -### Version 7.1 ( XX December 2020 : Under Development) +### Version 7.1 ( Not Released) - Enforce HTTPS validation by default - Updated sanesecurity publickey.gpg url to use SSL - Ignore yara files that include modules - Enabled yararulesproject rules by default - os.gentoo.conf: disable updates and upgrade checks - - Replace $PWD with readlink -f "$0" - Fix: URLhaus log message - Fix wrong download URL for MalwarePatrol - - Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) - - Added tar_executable to specify a non-default tar executable - - Config os.macosx.conf renamed to os.macos.conf - - Added tar_executable=gtar to os.macos.conf and os.openbsd.conf - - Fix: set ownership of last-version-check.txt - Fix: fallback to host if dig is not used - Disable cron MAILTO - - Incremented the config to version 92 - BSD read config fix + - Incremented the config to version 92 - Thank you @dandanio @Mrothyr @m0urs @msapiro @orlitzky @SlothOfAnarchy @jkellerer @RobbieTheK ### Version 7.0.1 (25 January 2020) @@ -228,7 +236,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] - eXtremeSHOK.com Maintenance - Added urlhaus database - Added extra yararulesproject databases -- Added new linuxmalwaredetect yara file + - Added new linuxmalwaredetect yara file - Automatic upgrades ( --upgrade ) - Added --upgrade command line option - Option to disable automatic upgrades ( allow_upgrades ) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 3e34f68b..1f9692cf 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -345,7 +345,7 @@ function xshok_file_download() { #outputfile #url #notimestamp fi else if [ ! "${3}" ] ; then - # the following is required because wget, cannot do --timestamping and --output-document together + # the following is required because wget, cannot do --timestamping and --output-document together this_dir="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" output_file="$1" url="$2" @@ -1497,9 +1497,9 @@ EOF ################################################################################ # Script Info -script_version="7.0.1" -script_version_date="2020-01-25" -minimum_required_config_version="92" +script_version="7.2" +script_version_date="2020-12-02" +minimum_required_config_version="93" minimum_yara_clamav_version="0.100" # Discover script: name, full_path and path @@ -2018,6 +2018,7 @@ if [ "$force_updates" == "yes" ] ; then xshok_pretty_echo_and_log "NOTICE: forcing updates" sanesecurity_update_hours="0" securiteinfo_update_hours="0" + securiteinfo_premium_update_hours="0" linuxmalwaredetect_update_hours="0" malwarepatrol_update_hours="0" yararulesproject_update_hours="0" @@ -2182,11 +2183,12 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then fi sanesecurity_dbs=( ) if [ -n "$temp_db" ] ; then - #sanesecurity_dbs=( $temp_db ) - read -r -a sanesecurity_dbs <<< "$temp_db" + #sanesecurity_dbs=( $temp_db ) + read -r -a sanesecurity_dbs <<< "$temp_db" fi fi fi +############################################################################################ if [ "$securiteinfo_enabled" == "yes" ] ; then if [ -n "$securiteinfo_dbs" ] ; then if [ -n "$securiteinfo_dbs_rating" ] ; then @@ -2196,11 +2198,23 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then fi securiteinfo_dbs=( ) if [ -n "$temp_db" ] ; then - #securiteinfo_dbs=( $temp_db ) - read -r -a securiteinfo_dbs <<< "$temp_db" + #securiteinfo_dbs=( $temp_db ) + read -r -a securiteinfo_dbs <<< "$temp_db" + fi + fi + if [ -n "$securiteinfo_premium_dbs" ] && [ "$securiteinfo_premium" == "yes" ] ; then + if [ -n "$securiteinfo_dbs_rating" ] ; then + temp_db="$(xshok_database "$securiteinfo_dbs_rating" "${securiteinfo_premium_dbs[@]}")" + else + temp_db="$(xshok_database "$default_dbs_rating" "${securiteinfo_premium_dbs[@]}")" + fi + if [ -n "$temp_db" ] ; then + #securiteinfo_dbs=( $temp_db ) + read -r -a securiteinfo_dbs <<< "$temp_db" fi fi fi +############################################################################################ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then if [ -n "$linuxmalwaredetect_dbs" ] ; then if [ -n "$linuxmalwaredetect_dbs_rating" ] ; then @@ -2210,11 +2224,12 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then fi linuxmalwaredetect_dbs=( ) if [ -n "$temp_db" ] ; then - #linuxmalwaredetect_dbs=( $temp_db ) - read -r -a linuxmalwaredetect_dbs <<< "$temp_db" + #linuxmalwaredetect_dbs=( $temp_db ) + read -r -a linuxmalwaredetect_dbs <<< "$temp_db" fi fi fi +############################################################################################ if [ "$yararulesproject_enabled" == "yes" ] ; then if [ -n "$yararulesproject_dbs" ] ; then if [ -n "$yararulesproject_dbs_rating" ] ; then @@ -2224,11 +2239,12 @@ if [ "$yararulesproject_enabled" == "yes" ] ; then fi yararulesproject_dbs=( ) if [ -n "$temp_db" ] ; then - #yararulesproject_dbs=( $temp_db ) - read -r -a yararulesproject_dbs <<< "$temp_db" + #yararulesproject_dbs=( $temp_db ) + read -r -a yararulesproject_dbs <<< "$temp_db" fi fi fi +############################################################################################ if [ "$urlhaus_enabled" == "yes" ] ; then if [ -n "$urlhaus_dbs" ] ; then if [ -n "$urlhaus_dbs_rating" ] ; then @@ -2243,6 +2259,7 @@ if [ "$urlhaus_enabled" == "yes" ] ; then fi fi fi +############################################################################################ # Set the variables for MalwarePatrol if [ "$malwarepatrol_product_code" != "8" ] ; then # assumption, free product code is always 8 (non-free product code is never 8) @@ -2702,7 +2719,11 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then fi db_file="" loop="" - update_interval="$((securiteinfo_update_hours * 3600))" + if [ "$securiteinfo_premium" == "yes" ] ; then + update_interval="$((securiteinfo_premium_update_hours * 3600))" + else + update_interval="$((securiteinfo_update_hours * 3600))" + fi time_interval="$((current_time - last_securiteinfo_update))" if [ "$time_interval" -ge "$((update_interval - 600))" ] ; then echo "$current_time" > "${work_dir_work_configs}/last-si-update.txt" @@ -2791,7 +2812,11 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then fi else xshok_pretty_echo_and_log "SecuriteInfo Database File Updates" "=" - xshok_draw_time_remaining "$((update_interval - time_interval))" "$securiteinfo_update_hours" "SecuriteInfo" + if [ "$securiteinfo_premium" == "yes" ] ; then + xshok_draw_time_remaining "$((update_interval - time_interval))" "$securiteinfo_premium_update_hours" "SecuriteInfo" + else + xshok_draw_time_remaining "$((update_interval - time_interval))" "$securiteinfo_update_hours" "SecuriteInfo" + fi fi fi fi diff --git a/config/master.conf b/config/master.conf index d4375a90..ae49e25b 100644 --- a/config/master.conf +++ b/config/master.conf @@ -85,6 +85,12 @@ malwarepatrol_list="clamav_basic" # clamav_basic or clamav_ext malwarepatrol_free="yes" malwarepatrol_db="malwarepatrol.db" + +# ========================= +# malware.expert : https://www.malware.expert +# malware.expert 2020 (non-free) clamav signatures +malwareexpert_serial_key="YOUR-SERIAL-KEY" + # ========================= # SecuriteInfo : https://www.SecuriteInfo.com # SecuriteInfo 2015 free clamav signatures @@ -101,8 +107,10 @@ malwarepatrol_db="malwarepatrol.db" # - 6. Enter the authorisation signature into the config securiteinfo_authorisation_signature: replacing YOUR-SIGNATURE-NUMBER with your authorisation signature from the link securiteinfo_authorisation_signature="YOUR-SIGNATURE-NUMBER" +# Enable if you have a commercial/premium/non-free subscription securiteinfo_premium="no" + # ======================== # Database provider update time # ======================== @@ -110,8 +118,10 @@ securiteinfo_premium="no" additional_update_hours="4" # Default is 4 hours (6 downloads daily). linuxmalwaredetect_update_hours="6" # Default is 6 hours (4 downloads daily). malwarepatrol_update_hours="24" # Default is 24 hours (1 downloads daily). +malwareexpert_update_hours="2" # Default is 2 hours (12 downloads daily). sanesecurity_update_hours="2" # Default is 2 hours (12 downloads daily). securiteinfo_update_hours="4" # Default is 4 hours (6 downloads daily). +securiteinfo_premium_update_hours="1" # Default is 1 hours (24 downloads daily). urlhaus_update_hours="0" # Default is 0 hours (Update constantly). yararulesproject_update_hours="24" # Default is 24 hours (1 downloads daily). @@ -122,6 +132,7 @@ yararulesproject_update_hours="24" # Default is 24 hours (1 downloads daily). additional_enabled="yes" # Additional Databases linuxmalwaredetect_enabled="yes" # Linux Malware Detect malwarepatrol_enabled="yes" # Malware Patrol +malwareexpert_enabled="yes" # malware.expert sanesecurity_enabled="yes" # Sanesecurity securiteinfo_enabled="yes" # SecuriteInfo urlhaus_enabled="yes" # urlhaus @@ -210,14 +221,6 @@ spam.ldb|MEDIUM # Spam detected using the new Logical Signature type spear.ndb|MEDIUM # Spear phishing email addresses (autogenerated from data here) spearl.ndb|MEDIUM # Spear phishing urls (autogenerated from data here) -### MALWARE.EXPERT https://malware.expert/ -# LOW -malware.expert.hdb|MEDIUM # statics MD5 pattern for files -# MEDIUM -malware.expert.fp|MEDIUM # found to be false positive malware -malware.expert.ldb|MEDIUM # which use multi-words search for malware in files -malware.expert.ndb|MEDIUM # Generic Hex pattern PHP malware, which can cause false positive alarms - ### FOXHOLE http://sanesecurity.com/foxhole-databases/ # LOW foxhole_filename.cdb|LOW # See Foxhole page for more details @@ -305,11 +308,12 @@ securiteinfopdf.hdb|LOW # Malwares PDF # HIGH spam_marketing.ndb|HIGH # Spam Marketing / spammer blacklist ) #END SECURITEINFO DATABASES -# NON-FREE DATABASES -declare -a securiteinfo_dbs_premium=( #START SECURITEINFO DATABASES + +# SECURITEINFO PREMIUM (NON-FREE) DATABASES +declare -a securiteinfo_premium_dbs=( #START SECURITEINFO DATABASES securiteinfo.mdb|LOW # 0-day Malwares securiteinfo0hour.hdb|LOW # 0-Hour Malwares -) +) #END NON-FREE SECURITEINFO DATABASES # ======================== # LinuxMalwareDetect Database(s) @@ -325,6 +329,21 @@ rfxn.hdb|LOW # MD5 Malware detection signatures rfxn.yara|LOW # Yara Malware detection signatures ) #END LINUXMALWAREDETECT DATABASES +# ======================== +# malware.expert Database(s) +# ======================== +# Add or remove database file names between quote marks as needed. To +# disable any malware.expert database downloads, remove the appropriate +# lines below. +declare -a malwareexpert_dbs=( +# LOW +malware.expert.hdb|MEDIUM # statics MD5 pattern for files +# MEDIUM +malware.expert.fp|MEDIUM # found to be false positive malware +malware.expert.ldb|MEDIUM # which use multi-words search for malware in files +malware.expert.ndb|MEDIUM # Generic Hex pattern PHP malware, which can cause false positive alarms +) #END MALWARE.EXPERT DATABASES + # ======================== # urlhaus Database(s) # ======================== @@ -537,9 +556,6 @@ min_sleep_time="60" # Default minimum is 60 seconds (1 minute). # Command to do a full clamd service stop/start #clamd_restart_opt="service clamd restart" -#tar_executable, specofy a custom tar execurtable, ie mac/bsd = g -#tar_executable="tar" - # Custom Command Paths, these are detected with the which command when not set #clamscan_bin="/usr/bin/clamscan" #curl_bin="/usr/bin/curl" @@ -594,6 +610,7 @@ add_dir="dbs-add" # User defined databases sub-directory gpg_dir="gpg-key" # Sanesecurity GPG Key sub-directory linuxmalwaredetect_dir="dbs-lmd" # Linux Malware Detect sub-directory malwarepatrol_dir="dbs-mbl" # MalwarePatrol sub-directory +malwareexpert_dir="dbs-me" # malware.expertt sub-directory pid_dir="pid" # User defined pid sub-directory sanesecurity_dir="dbs-ss" # Sanesecurity sub-directory securiteinfo_dir="dbs-si" # SecuriteInfo sub-directory @@ -622,6 +639,8 @@ selinux_fixes="no" # Default is "no" ignore ssl errors and warnings # Proxy Support # If necessary to proxy database downloads, define the rsync, curl, wget, dig, hosr proxy settings here. #rsync_proxy="username:password@proxy_host:proxy_port" +# Define rsync to use netcat for socks tunnel +#rsync_connect_prog="nc -X 5 -x socksproxy_host:socksproxy_port %H 873" #curl_proxy="--proxy http://username:password@proxy_host:proxy_port" #wget_proxy="-e http_proxy=http://username:password@proxy_host:proxy_port -e https_proxy=https://username:password@proxy_host:proxy_port" #dig_proxy="@proxy_host -p proxy_host:proxy_port" @@ -663,6 +682,7 @@ selinux_fixes="no" # Default is "no" ignore ssl errors and warnings #work_dir_gpg="" #default: uses work_dir/gpg_dir #work_dir_linuxmalwaredetect="" #default: uses work_dir/linuxmalwaredetect_dir #work_dir_malwarepatrol="" #default: uses work_dir/malwarepatrol_dir +#work_dir_malwareexpert="" #default: uses work_dir/malwareexpert_dir #work_dir_pid="" #default: uses work_dir/pid_dir #work_dir_sanesecurity="" #default: uses work_dir/sanesecurity_dir #work_dir_securiteinfo="" #default: uses work_dir/securiteinfo_dir @@ -680,6 +700,7 @@ user_configuration_complete="no" linuxmalwaredetect_sigpack_url="https://cdn.rfxn.com/downloads/maldet-sigpack.tgz" linuxmalwaredetect_version_url="https://cdn.rfxn.com/downloads/maldet.sigs.ver" malwarepatrol_url="https://lists.malwarepatrol.net/cgi/getfile" +malwareexpert_url="http://signatures.malware.expert" sanesecurity_gpg_url="https://www.sanesecurity.com/publickey.gpg" sanesecurity_url="rsync.sanesecurity.net" securiteinfo_url="https://www.securiteinfo.com/get/signatures" @@ -688,7 +709,7 @@ yararulesproject_url="https://raw.githubusercontent.com/Yara-Rules/rules/master" # ======================== # DO NOT EDIT ! -config_version="92" +config_version="93" ################################################################################ # diff --git a/config/user.conf b/config/user.conf index 0f95f6bb..2fa976ca 100644 --- a/config/user.conf +++ b/config/user.conf @@ -21,6 +21,10 @@ # Please note, it is your responsibility to manage the contents of this file. # Values provided here are just examples, feel free to use any values from the main config file. +# malware.expert 2020 (non-free) clamav signatures +# set to no to enable the commercial subscription databases +#malwareexpert_serial_key="YOUR-SERIAL-KEY" + # set to no to enable the commercial subscription url #malwarepatrol_free="yes" #malwarepatrol_list="clamav_basic" # clamav_basic or clamav_ext @@ -45,6 +49,7 @@ #securiteinfo_dbs_rating="HIGH" #urlhaus_dbs_rating="HIGH" #yararulesproject_dbs_rating="HIGH" +#malwareexpert_dbs_rating="HIGH" # ========================= # Additional signature databases @@ -67,6 +72,7 @@ user_configuration_complete="yes" #dig_proxy="@proxy_host -p proxy_host:proxy_port" #host_proxy="@proxy_host" #does not support port #rsync_proxy="username:password@proxy_host:proxy_port" +# Define rsync to use netcat for socks tunnel #rsync_connect_prog="nc -X 5 -x socksproxy_host:socksproxy_port %H 873" #wget_proxy="-e http_proxy=http://username:password@proxy_host:proxy_port -e https_proxy=https://username:password@proxy_host:proxy_port" diff --git a/guides/macos.md b/guides/macos.md index 6dcd3a29..a5a770a9 100644 --- a/guides/macos.md +++ b/guides/macos.md @@ -26,7 +26,7 @@ https://www.clamav.net/documents/installation-on-macos-mac-os-x ``` -# Step 2 Install gtar +# Step 2 Install gtar (gnu tar) ``` brew install gtar ``` From 47443596569110819c792e65209b1c6b402b64fd Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 01:17:15 +0200 Subject: [PATCH 031/101] sort the config --- config/master.conf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/config/master.conf b/config/master.conf index ae49e25b..e6a6d58a 100644 --- a/config/master.conf +++ b/config/master.conf @@ -117,11 +117,11 @@ securiteinfo_premium="no" # Since the database files are dynamically created, non default values can cause banning, change with caution additional_update_hours="4" # Default is 4 hours (6 downloads daily). linuxmalwaredetect_update_hours="6" # Default is 6 hours (4 downloads daily). -malwarepatrol_update_hours="24" # Default is 24 hours (1 downloads daily). malwareexpert_update_hours="2" # Default is 2 hours (12 downloads daily). +malwarepatrol_update_hours="24" # Default is 24 hours (1 downloads daily). sanesecurity_update_hours="2" # Default is 2 hours (12 downloads daily). -securiteinfo_update_hours="4" # Default is 4 hours (6 downloads daily). securiteinfo_premium_update_hours="1" # Default is 1 hours (24 downloads daily). +securiteinfo_update_hours="4" # Default is 4 hours (6 downloads daily). urlhaus_update_hours="0" # Default is 0 hours (Update constantly). yararulesproject_update_hours="24" # Default is 24 hours (1 downloads daily). @@ -131,8 +131,8 @@ yararulesproject_update_hours="24" # Default is 24 hours (1 downloads daily). # Set to no to disable an entire database, if the database is empty it will also be disabled. additional_enabled="yes" # Additional Databases linuxmalwaredetect_enabled="yes" # Linux Malware Detect -malwarepatrol_enabled="yes" # Malware Patrol malwareexpert_enabled="yes" # malware.expert +malwarepatrol_enabled="yes" # Malware Patrol sanesecurity_enabled="yes" # Sanesecurity securiteinfo_enabled="yes" # SecuriteInfo urlhaus_enabled="yes" # urlhaus @@ -648,21 +648,21 @@ selinux_fixes="no" # Default is "no" ignore ssl errors and warnings # Custom Cron install settings, these are detected and only used if you want to override # the automatic detection and generation of the values when not set, this is mainly to aid package maintainers +#cron_bash="" #default: detected with the which command #cron_dir="" #default: /etc/cron.d #cron_filename="" #default: clamav-unofficial-sigs #cron_minute="" #default: random value between 0-59 -#cron_user="" #default: uses the clam_user -#cron_sudo="no" #default no, yes will append sudo -u before the username -#cron_bash="" #default: detected with the which command #cron_script_full_path="" #default: detected to the fullpath of the script +#cron_sudo="no" #default no, yes will append sudo -u before the username +#cron_user="" #default: uses the clam_user # Custom logrotate install settings, these are detected and only used if you want to override # the automatic detection and generation of the values when not set, this is mainly to aid package maintainers #logrotate_dir="" #default: /etc/logrotate.d #logrotate_filename="" #default: clamav-unofficial-sigs -#logrotate_user="" #default: uses the clam_user #logrotate_group="" #default: uses the clam_group #logrotate_log_file_full_path="" #default: detected to the $log_file_path/$log_file_name +#logrotate_user="" #default: uses the clam_user # Custom man install settings, these are detected and only used if you want to override # the automatic detection and generation of the values when not set, this is mainly to aid package maintainers @@ -681,8 +681,8 @@ selinux_fixes="no" # Default is "no" ignore ssl errors and warnings #work_dir_add="" #default: uses work_dir/add_dir #work_dir_gpg="" #default: uses work_dir/gpg_dir #work_dir_linuxmalwaredetect="" #default: uses work_dir/linuxmalwaredetect_dir -#work_dir_malwarepatrol="" #default: uses work_dir/malwarepatrol_dir #work_dir_malwareexpert="" #default: uses work_dir/malwareexpert_dir +#work_dir_malwarepatrol="" #default: uses work_dir/malwarepatrol_dir #work_dir_pid="" #default: uses work_dir/pid_dir #work_dir_sanesecurity="" #default: uses work_dir/sanesecurity_dir #work_dir_securiteinfo="" #default: uses work_dir/securiteinfo_dir @@ -699,8 +699,8 @@ user_configuration_complete="no" # Database provider URLs linuxmalwaredetect_sigpack_url="https://cdn.rfxn.com/downloads/maldet-sigpack.tgz" linuxmalwaredetect_version_url="https://cdn.rfxn.com/downloads/maldet.sigs.ver" +malwareexpert_url="https://signatures.malware.expert" malwarepatrol_url="https://lists.malwarepatrol.net/cgi/getfile" -malwareexpert_url="http://signatures.malware.expert" sanesecurity_gpg_url="https://www.sanesecurity.com/publickey.gpg" sanesecurity_url="rsync.sanesecurity.net" securiteinfo_url="https://www.securiteinfo.com/get/signatures" From 7276092a4ae4851bf4aa80733e7e90bc01f23c96 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 01:35:15 +0200 Subject: [PATCH 032/101] add support for malwareexpert fixes #321 --- clamav-unofficial-sigs.sh | 195 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 1f9692cf..17f439cb 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -1147,6 +1147,9 @@ function clamscan_integrity_test_specific_database_file() { # databasefile xshok_pretty_echo_and_log "=== Linux Malware Detect ===" ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_linuxmalwaredetect" + xshok_pretty_echo_and_log "=== malware.expert Detect ===" + ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_malwareexpert" + xshok_pretty_echo_and_log "=== Linux Malware Detect ===" ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_yararulesproject" @@ -1871,6 +1874,11 @@ if [ -z "$work_dir_linuxmalwaredetect" ] ; then else shopt -s extglob; work_dir_malwarepatrol="${work_dir_malwarepatrol%%+(/)}" fi +if [ -z "$work_dir_malwareexpert" ] ; then + work_dir_malwareexpert="$(echo "${work_dir}/${malwareexpert_dir}" | sed 's:/*$::')" +else + shopt -s extglob; work_dir_malwareexpert="${work_dir_malwareexpert%%+(/)}" +fi if [ -z "$work_dir_malwarepatrol" ] ; then work_dir_malwarepatrol="$(echo "${work_dir}/${malwarepatrol_dir}" | sed 's:/*$::')" else @@ -2020,6 +2028,7 @@ if [ "$force_updates" == "yes" ] ; then securiteinfo_update_hours="0" securiteinfo_premium_update_hours="0" linuxmalwaredetect_update_hours="0" + malwareexpert_update_hours="0" malwarepatrol_update_hours="0" yararulesproject_update_hours="0" additional_update_hours="0" @@ -2230,6 +2239,21 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then fi fi ############################################################################################ +if [ "$malwareexpert_enabled" == "yes" ] ; then + if [ -n "$malwareexpert_dbs" ] ; then + if [ -n "$malwareexpert_dbs_rating" ] ; then + temp_db="$(xshok_database "$malwareexpert_dbs_rating" "${malwareexpert_dbs[@]}")" + else + temp_db="$(xshok_database "$default_dbs_rating" "${malwareexpert_dbs[@]}")" + fi + malwareexpert_dbs=( ) + if [ -n "$temp_db" ] ; then + #malwareexpert_dbs=( $temp_db ) + read -r -a malwareexpert_dbs <<< "$temp_db" + fi + fi +fi +############################################################################################ if [ "$yararulesproject_enabled" == "yes" ] ; then if [ -n "$yararulesproject_dbs" ] ; then if [ -n "$yararulesproject_dbs_rating" ] ; then @@ -2324,6 +2348,7 @@ xshok_mkdir_ownership "$work_dir" xshok_mkdir_ownership "$work_dir_securiteinfo" xshok_mkdir_ownership "$work_dir_malwarepatrol" xshok_mkdir_ownership "$work_dir_linuxmalwaredetect" +xshok_mkdir_ownership "$work_dir_malwareexpert" xshok_mkdir_ownership "$work_dir_sanesecurity" xshok_mkdir_ownership "$work_dir_yararulesproject" xshok_mkdir_ownership "$work_dir_work_configs" @@ -2441,6 +2466,14 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then done fi fi +if [ "$malwareexpert_enabled" == "yes" ] ; then + if [ -n "${malwareexpert_dbs[0]}" ] ; then + for db in "${malwareexpert_dbs[@]}" ; do + echo "${work_dir_malwareexpert}/${db}" >> "${current_tmp}" + clamav_files + done + fi +fi if [ "$malwarepatrol_enabled" == "yes" ] ; then if [ -n "$malwarepatrol_db" ] ; then echo "${work_dir_malwarepatrol}/${malwarepatrol_db}" >> "${current_tmp}" @@ -3005,6 +3038,168 @@ else fi fi +############################################################################################################################################## +# Check for updated malware.expert database files every set number of hours as defined in the "USER CONFIGURATION" section of this script +############################################################################################################################################## +if [ "$malwareexpert_enabled" == "yes" ] ; then + if [ "$malwareexpert_serial_key" != "YOUR-SERIAL-KEY" ] ; then + if [ -n "${malwareexpert_dbs[0]}" ] ; then + if [ ${#malwareexpert_dbs} -lt 1 ] ; then + xshok_pretty_echo_and_log "Failed malwareexpert_dbs config is invalid or not defined - SKIPPING" + else + rm -f "${work_dir_malwareexpert}/*.gz" + if [ -r "${work_dir_work_configs}/last-malwareexpert-update.txt" ] ; then + last_malwareexpert_update="$(cat "${work_dir_work_configs}/last-malwareexpert-update.txt")" + else + last_malwareexpert_update="0" + fi + db_file="" + loop="" + update_interval="$((malwareexpert_update_hours * 3600))" + time_interval="$((current_time - last_malwareexpert_update))" + if [ "$time_interval" -ge "$((update_interval - 600))" ] ; then + echo "$current_time" > "${work_dir_work_configs}/last-malwareexpert-update.txt" + + xshok_pretty_echo_and_log "malwareexpert Database File Updates" "=" + xshok_pretty_echo_and_log "Checking for malwareexpert updates..." + + # Check for a new version + found_upgrade="no" + if [ -n "$curl_bin" ] ; then + # shellcheck disable=SC2086 + latest_malwareexpert_version="$($curl_bin --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" "$malwareexpert_version_url" 2>&11 | head -n1 | xargs)" + else + # shellcheck disable=SC2086 + latest_malwareexpert_version="$($wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" "$malwareexpert_version_url" -O - 2>&12 | $grep_bin "^script_version=" | head -n1 | xargs)" + fi + + if [ "$latest_malwareexpert_version" ] ; then + # shellcheck disable=SC2183,SC2086 + if [ -f "${work_dir_malwareexpert}/current_malwareexpert_version" ] ; then + current_malwareexpert_version="$(head -n1 "${work_dir_malwareexpert}/current_malwareexpert_version" | xargs)" + else + current_malwareexpert_version="-1" + fi + if [ "$latest_malwareexpert_version" != "$current_malwareexpert_version" ] ; then + xshok_pretty_echo_and_log "malwareexpert Database File Updates" "=" + found_upgrade="yes" + fi + fi + + if [ "$found_upgrade" == "yes" ] ; then + xshok_file_download "${work_dir_malwareexpert}/sigpack.tgz" "${malwareexpert_sigpack_url}" + ret="$?" + if [ "$ret" -eq 0 ] ; then + # shellcheck disable=SC2035 + $tar_bin --strip-components=1 --wildcards --overwrite -xzf "${work_dir_malwareexpert}/sigpack.tgz" --directory "${work_dir_malwareexpert}" */rfxn.* + for db_file in "${malwareexpert_dbs[@]}" ; do + if [ "$loop" == "1" ] ; then + xshok_pretty_echo_and_log "---" + fi + loop="1" + if ! cmp -s "${work_dir_malwareexpert}/${db_file}" "${clam_dbs}/${db_file}" ; then + db_ext="${db_file#*.}" + + xshok_pretty_echo_and_log "Testing updated malwareexpert database file: ${db_file}" + if [ -z "$ham_dir" ] || [ "$db_ext" != "ndb" ] ; then + if $clamscan_bin --quiet -d "${work_dir_malwareexpert}/${db_file}" "${work_dir_work_configs}/scan-test.txt" 2>&10 ; then + xshok_pretty_echo_and_log "Clamscan reports malwareexpert ${db_file} database integrity tested good" + true + else + xshok_pretty_echo_and_log "Clamscan reports malwareexpert ${db_file} database integrity tested BAD" + if [ "$remove_bad_database" == "yes" ] ; then + if rm -f "${work_dir_malwareexpert}/${db_file}" ; then + xshok_pretty_echo_and_log "Removed invalid database: ${work_dir_malwareexpert}/${db_file}" + fi + fi + false + fi && (test "$keep_db_backup" = "yes" && cp -f -p "${clam_dbs}/${db_file}" "${clam_dbs}/${db}_file-bak" 2>/dev/null ; true) && if $rsync_bin -pcqt "${work_dir_malwareexpert}/${db_file}" "$clam_dbs" 2>&13 ; then + perms chown -f "${clam_user}:${clam_group}" "${clam_dbs}/${db_file}" + if [ "$selinux_fixes" == "yes" ] ; then + restorecon "${clam_dbs}/local.ign" + fi + xshok_pretty_echo_and_log "Successfully updated malwareexpert production database file: ${db_file}" + do_clamd_reload=1 + else + xshok_pretty_echo_and_log "Failed to successfully update malwareexpert production database file: ${db_file} - SKIPPING" + fi + else + $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_malwareexpert}/${db_file}" > "${test_dir}/${db_file}" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" + $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" + mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" + if $clamscan_bin --quiet -d "${test_dir}/${db_file}" "${work_dir_work_configs}/scan-test.txt" 2>&10 ; then + xshok_pretty_echo_and_log "Clamscan reports malwareexpert ${db_file} database integrity tested good" + true + else + xshok_pretty_echo_and_log "Clamscan reports malwareexpert ${db_file} database integrity tested BAD" + if [ "$remove_bad_database" == "yes" ] ; then + if rm -f "${work_dir_malwareexpert}/${db_file}" ; then + xshok_pretty_echo_and_log "Removed invalid database: ${work_dir_malwareexpert}/${db_file}" + fi + fi + false + fi && (test "$keep_db_backup" = "yes" && cp -f -p "${clam_dbs}/${db_file}" "${clam_dbs}/${db}_file-bak" 2>/dev/null ; true) && if $rsync_bin -pcqt "${test_dir}/${db_file}" "$clam_dbs" 2>&13 ; then + perms chown -f "${clam_user}:${clam_group}" "${clam_dbs}/${db_file}" + if [ "$selinux_fixes" == "yes" ] ; then + restorecon "${clam_dbs}/${db_file}" + fi + xshok_pretty_echo_and_log "Successfully updated malwareexpert production database file: ${db_file}" + do_clamd_reload=1 + else + xshok_pretty_echo_and_log "Failed to successfully update malwareexpert production database file: ${db_file} - SKIPPING" + fi + fi + fi + + done + #save the current version + echo "$latest_malwareexpert_version" > "${work_dir_malwareexpert}/current_malwareexpert_version" + + else + xshok_pretty_echo_and_log "WARNING: Failed connection to ${malwareexpert_sigpack_url} - SKIPPED malwareexpert update" + fi + else + xshok_pretty_echo_and_log "No malwareexpert database file updates found" "-" + fi + else + xshok_pretty_echo_and_log "malwareexpert Database File Updates" "=" + xshok_draw_time_remaining "$((update_interval - time_interval))" "$malwareexpert_update_hours" "malwareexpert" + fi + fi + fi + fi +else + if [ -n "${malwareexpert_dbs[0]}" ] ; then + if [ "$remove_disabled_databases" == "yes" ] ; then + xshok_pretty_echo_and_log "Removing disabled malwareexpert Database files" + + if [ -f "${work_dir_malwareexpert}/current_malwareexpert_version" ] ; then + rm -f "${work_dir_malwareexpert}/current_malwareexpert_version" + fi + if [ -f "${work_dir_malwareexpert}/sigpack.tgz" ] ; then + rm -f "${work_dir_malwareexpert}/sigpack.tgz" + fi + + for db_file in "${malwareexpert_dbs[@]}" ; do + if echo "$db_file" | $grep_bin -q "|" ; then + db_file="${db_file%|*}" + fi + if [ -r "${work_dir_malwareexpert}/${db_file}" ] ; then + xshok_pretty_echo_and_log "Removing ${work_dir_malwareexpert}/${db_file}" + rm -f "${work_dir_malwareexpert}/${db_file}" + do_clamd_reload=1 + fi + if [ -r "${clam_dbs}/${db_file}" ] ; then + xshok_pretty_echo_and_log "Removing ${clam_dbs}/${db_file}" + rm -f "${clam_dbs}/${db_file}" + do_clamd_reload=1 + fi + done + fi + fi +fi ######################################################################################################################################### # Download MalwarePatrol database file every set number of hours as defined in the "USER CONFIGURATION" section of this script. # From 71efd842d6bd8a231e9968b5d1815c4b87a7702b Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 01:43:30 +0200 Subject: [PATCH 033/101] corrected logic for malware.expert --- clamav-unofficial-sigs.sh | 227 +++++++++++++++++--------------------- 1 file changed, 103 insertions(+), 124 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 17f439cb..f628c8d6 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -3039,149 +3039,128 @@ else fi ############################################################################################################################################## -# Check for updated malware.expert database files every set number of hours as defined in the "USER CONFIGURATION" section of this script +# Check for updated malware.expert database files every set number of hours as defined in the "USER CONFIGURATION" section of this script # ############################################################################################################################################## if [ "$malwareexpert_enabled" == "yes" ] ; then - if [ "$malwareexpert_serial_key" != "YOUR-SERIAL-KEY" ] ; then - if [ -n "${malwareexpert_dbs[0]}" ] ; then - if [ ${#malwareexpert_dbs} -lt 1 ] ; then - xshok_pretty_echo_and_log "Failed malwareexpert_dbs config is invalid or not defined - SKIPPING" + if [ "$malwareexpert_serial_key" != "YOUR-SERIAL-KEY" ] ; then + if [ -n "${malwareexpert_dbs[0]}" ] ; then + if [ ${#malwareexpert_dbs} -lt 1 ] ; then + xshok_pretty_echo_and_log "Failed malwareexpert_dbs config is invalid or not defined - SKIPPING" + else + rm -f "${work_dir_malwareexpert}/*.gz" + if [ -r "${work_dir_work_configs}/last-si-update.txt" ] ; then + last_malwareexpert_update="$(cat "${work_dir_work_configs}/last-si-update.txt")" else - rm -f "${work_dir_malwareexpert}/*.gz" - if [ -r "${work_dir_work_configs}/last-malwareexpert-update.txt" ] ; then - last_malwareexpert_update="$(cat "${work_dir_work_configs}/last-malwareexpert-update.txt")" - else - last_malwareexpert_update="0" - fi - db_file="" - loop="" - update_interval="$((malwareexpert_update_hours * 3600))" - time_interval="$((current_time - last_malwareexpert_update))" - if [ "$time_interval" -ge "$((update_interval - 600))" ] ; then - echo "$current_time" > "${work_dir_work_configs}/last-malwareexpert-update.txt" - - xshok_pretty_echo_and_log "malwareexpert Database File Updates" "=" - xshok_pretty_echo_and_log "Checking for malwareexpert updates..." - - # Check for a new version - found_upgrade="no" - if [ -n "$curl_bin" ] ; then - # shellcheck disable=SC2086 - latest_malwareexpert_version="$($curl_bin --compressed $curl_proxy $curl_insecure $curl_output_level --connect-timeout "${downloader_connect_timeout}" --remote-time --location --retry "${downloader_tries}" --max-time "${downloader_max_time}" "$malwareexpert_version_url" 2>&11 | head -n1 | xargs)" - else - # shellcheck disable=SC2086 - latest_malwareexpert_version="$($wget_bin $wget_compression $wget_proxy $wget_insecure $wget_output_level --connect-timeout="${downloader_connect_timeout}" --random-wait --tries="${downloader_tries}" --timeout="${downloader_max_time}" "$malwareexpert_version_url" -O - 2>&12 | $grep_bin "^script_version=" | head -n1 | xargs)" - fi - - if [ "$latest_malwareexpert_version" ] ; then - # shellcheck disable=SC2183,SC2086 - if [ -f "${work_dir_malwareexpert}/current_malwareexpert_version" ] ; then - current_malwareexpert_version="$(head -n1 "${work_dir_malwareexpert}/current_malwareexpert_version" | xargs)" - else - current_malwareexpert_version="-1" - fi - if [ "$latest_malwareexpert_version" != "$current_malwareexpert_version" ] ; then - xshok_pretty_echo_and_log "malwareexpert Database File Updates" "=" - found_upgrade="yes" - fi + last_malwareexpert_update="0" + fi + db_file="" + loop="" + if [ "$malwareexpert_premium" == "yes" ] ; then + update_interval="$((malwareexpert_premium_update_hours * 3600))" + else + update_interval="$((malwareexpert_update_hours * 3600))" + fi + time_interval="$((current_time - last_malwareexpert_update))" + if [ "$time_interval" -ge "$((update_interval - 600))" ] ; then + echo "$current_time" > "${work_dir_work_configs}/last-si-update.txt" + xshok_pretty_echo_and_log "malwareexpert Database File Updates" "=" + xshok_pretty_echo_and_log "Checking for malwareexpert updates..." + malwareexpert_updates="0" + for db_file in "${malwareexpert_dbs[@]}" ; do + if [ "$loop" == "1" ] ; then + xshok_pretty_echo_and_log "---" fi + xshok_pretty_echo_and_log "Checking for updated malware.expert database file: ${db_file}" + malwareexpert_db_update="0" + xshok_file_download "${work_dir_malwareexpert}/${db_file}" "${malwareexpert_url}/${malwareexpert_serial_key}/${db_file}" + ret="$?" + if [ "$ret" -eq 0 ] ; then + loop="1" + if ! cmp -s "${work_dir_malwareexpert}/${db_file}" "${clam_dbs}/${db_file}" ; then + db_ext="${db_file#*.}" - if [ "$found_upgrade" == "yes" ] ; then - xshok_file_download "${work_dir_malwareexpert}/sigpack.tgz" "${malwareexpert_sigpack_url}" - ret="$?" - if [ "$ret" -eq 0 ] ; then - # shellcheck disable=SC2035 - $tar_bin --strip-components=1 --wildcards --overwrite -xzf "${work_dir_malwareexpert}/sigpack.tgz" --directory "${work_dir_malwareexpert}" */rfxn.* - for db_file in "${malwareexpert_dbs[@]}" ; do - if [ "$loop" == "1" ] ; then - xshok_pretty_echo_and_log "---" - fi - loop="1" - if ! cmp -s "${work_dir_malwareexpert}/${db_file}" "${clam_dbs}/${db_file}" ; then - db_ext="${db_file#*.}" - - xshok_pretty_echo_and_log "Testing updated malwareexpert database file: ${db_file}" - if [ -z "$ham_dir" ] || [ "$db_ext" != "ndb" ] ; then - if $clamscan_bin --quiet -d "${work_dir_malwareexpert}/${db_file}" "${work_dir_work_configs}/scan-test.txt" 2>&10 ; then - xshok_pretty_echo_and_log "Clamscan reports malwareexpert ${db_file} database integrity tested good" - true - else - xshok_pretty_echo_and_log "Clamscan reports malwareexpert ${db_file} database integrity tested BAD" - if [ "$remove_bad_database" == "yes" ] ; then - if rm -f "${work_dir_malwareexpert}/${db_file}" ; then - xshok_pretty_echo_and_log "Removed invalid database: ${work_dir_malwareexpert}/${db_file}" - fi - fi - false - fi && (test "$keep_db_backup" = "yes" && cp -f -p "${clam_dbs}/${db_file}" "${clam_dbs}/${db}_file-bak" 2>/dev/null ; true) && if $rsync_bin -pcqt "${work_dir_malwareexpert}/${db_file}" "$clam_dbs" 2>&13 ; then - perms chown -f "${clam_user}:${clam_group}" "${clam_dbs}/${db_file}" - if [ "$selinux_fixes" == "yes" ] ; then - restorecon "${clam_dbs}/local.ign" - fi - xshok_pretty_echo_and_log "Successfully updated malwareexpert production database file: ${db_file}" - do_clamd_reload=1 - else - xshok_pretty_echo_and_log "Failed to successfully update malwareexpert production database file: ${db_file} - SKIPPING" + xshok_pretty_echo_and_log "Testing updated malware.expert database file: ${db_file}" + if [ -z "$ham_dir" ] || [ "$db_ext" != "ndb" ] ; then + if $clamscan_bin --quiet -d "${work_dir_malwareexpert}/${db_file}" "${work_dir_work_configs}/scan-test.txt" 2>&10 ; then + xshok_pretty_echo_and_log "Clamscan reports malware.expert ${db_file} database integrity tested good" + true + else + xshok_pretty_echo_and_log "Clamscan reports malware.expert ${db_file} database integrity tested BAD" + if [ "$remove_bad_database" == "yes" ] ; then + if rm -f "${work_dir_malwareexpert}/${db_file}" ; then + xshok_pretty_echo_and_log "Removed invalid database: ${work_dir_malwareexpert}/${db_file}" fi - else - $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_malwareexpert}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" - $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" - $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" - mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" - if $clamscan_bin --quiet -d "${test_dir}/${db_file}" "${work_dir_work_configs}/scan-test.txt" 2>&10 ; then - xshok_pretty_echo_and_log "Clamscan reports malwareexpert ${db_file} database integrity tested good" - true - else - xshok_pretty_echo_and_log "Clamscan reports malwareexpert ${db_file} database integrity tested BAD" - if [ "$remove_bad_database" == "yes" ] ; then - if rm -f "${work_dir_malwareexpert}/${db_file}" ; then - xshok_pretty_echo_and_log "Removed invalid database: ${work_dir_malwareexpert}/${db_file}" - fi - fi - false - fi && (test "$keep_db_backup" = "yes" && cp -f -p "${clam_dbs}/${db_file}" "${clam_dbs}/${db}_file-bak" 2>/dev/null ; true) && if $rsync_bin -pcqt "${test_dir}/${db_file}" "$clam_dbs" 2>&13 ; then - perms chown -f "${clam_user}:${clam_group}" "${clam_dbs}/${db_file}" - if [ "$selinux_fixes" == "yes" ] ; then - restorecon "${clam_dbs}/${db_file}" - fi - xshok_pretty_echo_and_log "Successfully updated malwareexpert production database file: ${db_file}" - do_clamd_reload=1 - else - xshok_pretty_echo_and_log "Failed to successfully update malwareexpert production database file: ${db_file} - SKIPPING" + fi + false + fi && (test "$keep_db_backup" = "yes" && cp -f -p "${clam_dbs}/${db_file}" "${clam_dbs}/${db}_file-bak" 2>/dev/null ; true) && if $rsync_bin -pcqt "${work_dir_malwareexpert}/${db_file}" "$clam_dbs" 2>&13 ; then + perms chown -f "${clam_user}:${clam_group}" "${clam_dbs}/${db_file}" + if [ "$selinux_fixes" == "yes" ] ; then + restorecon "${clam_dbs}/${db_file}" + fi + xshok_pretty_echo_and_log "Successfully updated malware.expert production database file: ${db_file}" + malwareexpert_updates=1 + malwareexpert_db_update=1 + do_clamd_reload=1 + else + xshok_pretty_echo_and_log "Failed to successfully update malware.expert production database file: ${db_file} - SKIPPING" + fi + else + $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_malwareexpert}/${db_file}" > "${test_dir}/${db_file}" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" + $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" + mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" + if $clamscan_bin --quiet -d "${test_dir}/${db_file}" "${work_dir_work_configs}/scan-test.txt" 2>&10 ; then + xshok_pretty_echo_and_log "Clamscan reports malware.expert ${db_file} database integrity tested good" + true + else + xshok_pretty_echo_and_log "Clamscan reports malware.expert ${db_file} database integrity tested BAD" + rm -f "${work_dir_malwareexpert}/${db_file}" + if [ "$remove_bad_database" == "yes" ] ; then + if rm -f "${work_dir_malwareexpert}/${db_file}" ; then + xshok_pretty_echo_and_log "Removed invalid database: ${work_dir_malwareexpert}/${db_file}" fi fi + false + fi && (test "$keep_db_backup" = "yes" && cp -f -p "${clam_dbs}/${db_file}" "${clam_dbs}/${db}_file-bak" 2>/dev/null ; true) && if $rsync_bin -pcqt "${test_dir}/${db_file}" "$clam_dbs" 2>&13 ; then + perms chown -f "${clam_user}:${clam_group}" "${clam_dbs}/${db_file}" + if [ "$selinux_fixes" == "yes" ] ; then + restorecon "${clam_dbs}/${db_file}" + fi + xshok_pretty_echo_and_log "Successfully updated malware.expert production database file: ${db_file}" + malwareexpert_updates=1 + malwareexpert_db_update=1 + do_clamd_reload=1 + else + xshok_pretty_echo_and_log "Failed to successfully update malware.expert production database file: ${db_file} - SKIPPING" fi - - done - #save the current version - echo "$latest_malwareexpert_version" > "${work_dir_malwareexpert}/current_malwareexpert_version" - - else - xshok_pretty_echo_and_log "WARNING: Failed connection to ${malwareexpert_sigpack_url} - SKIPPED malwareexpert update" + fi fi else - xshok_pretty_echo_and_log "No malwareexpert database file updates found" "-" + xshok_pretty_echo_and_log "Failed connection to ${malwareexpert_url} - SKIPPED malware.expert ${db_file} update" + fi + if [ "$malwareexpert_db_update" != "1" ] ; then + xshok_pretty_echo_and_log "No updated malware.expert ${db_file} database file found" "-" fi + done + if [ "$malwareexpert_updates" != "1" ] ; then + xshok_pretty_echo_and_log "No malware.expert database file updates found" "-" + fi + else + xshok_pretty_echo_and_log "malware.expert Database File Updates" "=" + if [ "$malwareexpert_premium" == "yes" ] ; then + xshok_draw_time_remaining "$((update_interval - time_interval))" "$malwareexpert_premium_update_hours" "malwareexpert" else - xshok_pretty_echo_and_log "malwareexpert Database File Updates" "=" - xshok_draw_time_remaining "$((update_interval - time_interval))" "$malwareexpert_update_hours" "malwareexpert" + xshok_draw_time_remaining "$((update_interval - time_interval))" "$malwareexpert_update_hours" "malwareexpert" fi fi fi fi + fi else - if [ -n "${malwareexpert_dbs[0]}" ] ; then + if [ -n "$malwareexpert_dbs" ] ; then if [ "$remove_disabled_databases" == "yes" ] ; then - xshok_pretty_echo_and_log "Removing disabled malwareexpert Database files" - - if [ -f "${work_dir_malwareexpert}/current_malwareexpert_version" ] ; then - rm -f "${work_dir_malwareexpert}/current_malwareexpert_version" - fi - if [ -f "${work_dir_malwareexpert}/sigpack.tgz" ] ; then - rm -f "${work_dir_malwareexpert}/sigpack.tgz" - fi - + xshok_pretty_echo_and_log "Removing disabled malware.expert Database files" for db_file in "${malwareexpert_dbs[@]}" ; do if echo "$db_file" | $grep_bin -q "|" ; then db_file="${db_file%|*}" From ee2a603a293a24583694045a7de22670c9e30276 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 02:23:31 +0200 Subject: [PATCH 034/101] add more ci tests --- .t/ci-test.sh | 24 +++++++++++++++-- .t/tests/user.conf | 9 +++---- .t/tests/user_low.conf | 58 +++++++++++++++++++++++++++++++++++++++++ .t/tests/user_wget.conf | 9 +++---- config/user.conf | 2 +- 5 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 .t/tests/user_low.conf diff --git a/.t/ci-test.sh b/.t/ci-test.sh index 9cd45d02..45539543 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -14,7 +14,7 @@ if [ -e "/var/lib/clamav/sanesecurity.ftm" ] ; then rm -f /var/lib/clamav/sanesecurity.ftm fi -echo "running script as root and verbose and force_wget" +echo "running script verbose and force_wget" cp -f .t/tests/user_wget.conf /etc/clamav-unofficial-sigs/user.conf bash /usr/sbin/clamav-unofficial-sigs --verbose if [ "$?" -eq "0" ] ; then @@ -24,7 +24,7 @@ else exit 1 fi -echo "running script as root and verbose" +echo "running script verbose default curl" cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf bash /usr/sbin/clamav-unofficial-sigs --verbose if [ "$?" -eq "0" ] ; then @@ -120,3 +120,23 @@ else echo .. ERROR exit 1 fi + +echo "running script verbose with LOW ratings" +cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user_low.conf +bash /usr/sbin/clamav-unofficial-sigs --verbose +if [ "$?" -eq "0" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi + +echo "EXTRA DEBUG INFORMATION" +echo "================" +echo "/var/lib/clamav/" +echo "================" +ls -laFh /var/lib/clamav/ +echo "================" +echo "/var/lib/clamav-unofficial-sigs/dbs-si" +echo "================" +ls -laFh /var/lib/clamav-unofficial-sigs/dbs-si diff --git a/.t/tests/user.conf b/.t/tests/user.conf index cea9d624..c4c4ab7b 100644 --- a/.t/tests/user.conf +++ b/.t/tests/user.conf @@ -43,15 +43,14 @@ https://raw.githubusercontent.com/wmetcalf/clam-punch/master/miscreantpunch099.l https://raw.githubusercontent.com/wmetcalf/clam-punch/master/MiscreantPunch099-Low.ldb ) #END ADDITIONAL DATABASES - declare -a securiteinfo_dbs=( securiteinfo.ign2|REQUIRED securiteinfo.hdb|LOW javascript.ndb|LOW -securiteinfohtml.hdb|LOW -securiteinfoascii.hdb|LOW -securiteinfopdf.hdb|LOW -securiteinfoandroid.hdb|LOW # +securiteinfohtml.hdb|MEDIUM +securiteinfoascii.hdb|MEDIUM +securiteinfopdf.hdb|HIGH +securiteinfoandroid.hdb|HIGH # spam_marketing.ndb|HIGH ) #END SECURITEINFO DATABASES diff --git a/.t/tests/user_low.conf b/.t/tests/user_low.conf new file mode 100644 index 00000000..92281234 --- /dev/null +++ b/.t/tests/user_low.conf @@ -0,0 +1,58 @@ +################### +# This is property of eXtremeSHOK.com +# You are free to use, modify and distribute, however you may not remove this notice. +# Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com +# License: BSD (Berkeley Software Distribution) +################## + +malwarepatrol_enabled="yes" +malwarepatrol_receipt_code=$ci_malwarepatrol_receipt_code +malwarepatrol_product_code=$ci_malwarepatrol_receipt_code +malwarepatrol_list=$ci_malwarepatrol_receipt_code +malwarepatrol_free=$ci_malwarepatrol_free + +securiteinfo_enabled="yes" +securiteinfo_authorisation_signature=$ci_securiteinfo_authorisation_signature + +sanesecurity_enabled="yes" + +linuxmalwaredetect_enabled="yes" + +# THIS NEEDS TO BE TESTED +yararules_enabled="no" +enable_yararules="no" + +# Default dbs rating +# valid rating: LOW, MEDIUM, HIGH +default_dbs_rating="LOW" + +# Per Database +# These ratings will override the global rating for the specific database +# valid rating: LOW, MEDIUM, HIGH, DISABLE +sanesecurity_dbs_rating="DISABLE" +securiteinfo_dbs_rating="LOW" +linuxmalwaredetect_dbs_rating="DISABLE" +yararulesproject_dbs_rating="DISABLE" + +enable_gpg="no" + +user_configuration_complete="yes" + +declare -a additional_dbs=( +https://raw.githubusercontent.com/wmetcalf/clam-punch/master/miscreantpunch099.ldb +https://raw.githubusercontent.com/wmetcalf/clam-punch/master/MiscreantPunch099-Low.ldb +) #END ADDITIONAL DATABASES + +declare -a securiteinfo_dbs=( +securiteinfo.ign2|REQUIRED +securiteinfo.hdb|LOW +javascript.ndb|LOW +securiteinfohtml.hdb|MEDIUM +securiteinfoascii.hdb|MEDIUM +securiteinfopdf.hdb|HIGH +securiteinfoandroid.hdb|HIGH # +spam_marketing.ndb|HIGH +) #END SECURITEINFO DATABASES + +# Enable all debug options +debug="yes" diff --git a/.t/tests/user_wget.conf b/.t/tests/user_wget.conf index c9339764..8ca0c3e2 100644 --- a/.t/tests/user_wget.conf +++ b/.t/tests/user_wget.conf @@ -43,15 +43,14 @@ https://raw.githubusercontent.com/wmetcalf/clam-punch/master/miscreantpunch099.l https://raw.githubusercontent.com/wmetcalf/clam-punch/master/MiscreantPunch099-Low.ldb ) #END ADDITIONAL DATABASES - declare -a securiteinfo_dbs=( securiteinfo.ign2|REQUIRED securiteinfo.hdb|LOW javascript.ndb|LOW -securiteinfohtml.hdb|LOW -securiteinfoascii.hdb|LOW -securiteinfopdf.hdb|LOW -securiteinfoandroid.hdb|LOW # +securiteinfohtml.hdb|MEDIUM +securiteinfoascii.hdb|MEDIUM +securiteinfopdf.hdb|HIGH +securiteinfoandroid.hdb|HIGH # spam_marketing.ndb|HIGH ) #END SECURITEINFO DATABASES diff --git a/config/user.conf b/config/user.conf index 2fa976ca..4d2cc32b 100644 --- a/config/user.conf +++ b/config/user.conf @@ -45,11 +45,11 @@ # These ratings will override the global rating for the specific database # valid rating: LOW, MEDIUM, HIGH, DISABLE #linuxmalwaredetect_dbs_rating="HIGH" +#malwareexpert_dbs_rating="HIGH" #sanesecurity_dbs_rating="HIGH" #securiteinfo_dbs_rating="HIGH" #urlhaus_dbs_rating="HIGH" #yararulesproject_dbs_rating="HIGH" -#malwareexpert_dbs_rating="HIGH" # ========================= # Additional signature databases From 1dc10f02e17beec1612b73e72015dd338b2749cf Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 02:39:00 +0200 Subject: [PATCH 035/101] begin support for database rating downgrades --- README.md | 23 ++--------------------- config/master.conf | 38 +++++++++++++++++++------------------- 2 files changed, 21 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 5432a585..15786027 100644 --- a/README.md +++ b/README.md @@ -117,59 +117,45 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] master.conf, os.conf or user.conf Default Directory: /etc/clamav-unofficial-sigs - -F, --force Force all databases to be downloaded, could cause ip to be blocked - -h, --help Display this script's help and usage information - -V, --version Output script version and date information - -v, --verbose Be verbose, enabled when not run under cron - -s, --silence Only output error messages, enabled when run under cron - -d, --decode-sig Decode a third-party signature either by signature name (eg: Sanesecurity.Junk.15248) or hexadecimal string. This flag will 'NOT' decode image signatures - -e, --encode-string Hexadecimal encode an entire input string that can be used in any '*.ndb' signature database file - -f, --encode-formatted Hexadecimal encode a formatted input string containing signature spacing fields '{}, (), *', without encoding the spacing fields, so that the encoded signature can be used in any '*.ndb' signature database file - -g, --gpg-verify GPG verify a specific Sanesecurity database file eg: '-g filename.ext' (do not include file path) - -i, --information Output system and configuration information for viewing or possible debugging purposes - -m, --make-database Make a signature database from an ascii file containing data strings, with one data string per line. Additional information is provided when using this flag - -t, --test-database Clamscan integrity test a specific database file eg: '-t filename.ext' (do not include file path) - -o, --output-triggered If HAM directory scanning is enabled in the script's configuration file, then output names of any third-party signatures that triggered during the HAM directory scan - -w, --whitelist Adds a signature whitelist entry in the newer ClamAV IGN2 format to 'my-whitelist.ign2' in order to temporarily resolve a false-positive issue with a specific third-party signature. @@ -177,35 +163,30 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] if the original signature is either modified or removed from the third-party signature database - --check-clamav If ClamD status check is enabled and the socket path is correctly specifiedthen test to see if clamd is running or not - --upgrade Upgrades this script and master.conf to the latest available version - --install-all Install and generate the cron, logroate and man files, autodetects the values based on your config files - --install-cron Install and generate the cron file, autodetects the values based on your config files - --install-logrotate Install and generate the logrotate file, autodetects the values based on your config files - --install-man Install and generate the man file, autodetects the values based on your config files - --remove-script Remove the clamav-unofficial-sigs script and all of its associated files and databases from the system ## Change Log ### Version 7.2 ( XX December 2020 : Under Development) + - Database rating downgrades are now supported, eg, changing from HIGH to LOW will remove the HIGH and MEDIUM rated databases. + - Disabled datbaases are automatically removed - Added Malware.expert databases (non-free) - Reworked securiteinfo premium databases (non-free) - Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) diff --git a/config/master.conf b/config/master.conf index e6a6d58a..d7b48227 100644 --- a/config/master.conf +++ b/config/master.conf @@ -161,7 +161,7 @@ enable_yararules="yes" #Enables yararules in the various databases, automatica # LOWONLY : used only when the rating is low # MEDIUMONLY : used only when the rating is medium # LOWMEDIUMONLY : used only when the rating is medium or low -# DISABLED : never used, or you can also comment the line out if you want +# DISABLED : never used, will automatically remove the present file # # Old Format is still supported, requiring you to comment out files to disable them # old_example_dbs=" @@ -250,7 +250,7 @@ winnow.complex.patterns.ldb|MEDIUM # contain hand generated signatures for malw winnow_phish_complete.ndb|HIGH # Phishing and other malicious urls and compromised hosts **DO NOT USE WITH winnow_phish_complete_url** ### OITC YARA Format rules ### Note: Yara signatures require ClamAV 0.100 or newer to work -##winnow_malware.yara|LOW # detect spam +winnow_malware.yara|LOW # detect spam ### MiscreantPunch http://malwarefor.me/about/ ## MEDIUM @@ -371,26 +371,26 @@ declare -a yararulesproject_dbs=( # Anti debug and anti virtualization techniques used by malware antidebug_antivm/antidebug_antivm.yar|LOW # Aimed toward the detection and existence of Exploit Kits. -#exploit_kits/EK_Angler.yar|LOW # duplicated in rxfn.yara -#exploit_kits/EK_Blackhole.yar|LOW # duplicated in rxfn.yara +exploit_kits/EK_Angler.yar|DISABLED # duplicated in rxfn.yara +exploit_kits/EK_Blackhole.yar|DISABLED # duplicated in rxfn.yara exploit_kits/EK_BleedingLife.yar|LOW # duplicated in rxfn.yara -#exploit_kits/EK_Crimepack.yar|LOW # duplicated in rxfn.yara -#exploit_kits/EK_Eleonore.yar|LOW # duplicated in rxfn.yara -#exploit_kits/EK_Fragus.yar|LOW # duplicated in rxfn.yara -#exploit_kits/EK_Phoenix.yar|LOW # duplicated in rxfn.yara -#exploit_kits/EK_Sakura.yar|LOW # duplicated in rxfn.yara -#exploit_kits/EK_ZeroAcces.yar|LOW # duplicated in rxfn.yara -#exploit_kits/EK_Zerox88.yar|LOW # duplicated in rxfn.yara -#exploit_kits/EK_Zeus.yar|LOW # duplicated in rxfn.yara -# Identification of well-known webshells -#webshells/WShell_APT_Laudanum.yar|LOW # duplicated in rxfn.yara +exploit_kits/EK_Crimepack.yar|DISABLED # duplicated in rxfn.yara +exploit_kits/EK_Eleonore.yar|DISABLED # duplicated in rxfn.yara +exploit_kits/EK_Fragus.yar|DISABLED # duplicated in rxfn.yara +exploit_kits/EK_Phoenix.yar|DISABLED # duplicated in rxfn.yara +exploit_kits/EK_Sakura.yar|DISABLED # duplicated in rxfn.yara +exploit_kits/EK_ZeroAcces.yar|DISABLED # duplicated in rxfn.yara +exploit_kits/EK_Zerox88.yar|DISABLED # duplicated in rxfn.yara +exploit_kits/EK_Zeus.yar|DISABLED # duplicated in rxfn.yara +#Identification of well-known webshells +webshells/WShell_APT_Laudanum.yar|DISABLED # duplicated in rxfn.yara webshells/WShell_ASPXSpy.yar|LOW webshells/WShell_Drupalgeddon2_icos.yar|LOW -#webshells/WShell_PHP_Anuna.yar|LOW # duplicated in rxfn.yara -#webshells/WShell_PHP_in_images.yar|LOW # duplicated in rxfn.yara -#webshells/WShell_THOR_Webshells.yar|LOW # duplicated in rxfn.yara -#webshells/Wshell_ChineseSpam.yar|LOW # duplicated in rxfn.yara -#webshells/Wshell_fire2013.yar|LOW # duplicated in rxfn.yara +webshells/WShell_PHP_Anuna.yar|DISABLED # duplicated in rxfn.yara +webshells/WShell_PHP_in_images.yar|DISABLED # duplicated in rxfn.yara +webshells/WShell_THOR_Webshells.yar|DISABLED # duplicated in rxfn.yara +webshells/Wshell_ChineseSpam.yar|DISABLED # duplicated in rxfn.yara +webshells/Wshell_fire2013.yar|DISABLED # duplicated in rxfn.yara # MEDIUM # Identification of specific Common Vulnerabilities and Exposures (CVEs) cve_rules/CVE-2010-0805.yar|MEDIUM From 56ff57d4ac3b5b3a5b0b3bfd838080fd5b075f8b Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 03:02:08 +0200 Subject: [PATCH 036/101] clarify valid database ratings --- clamav-unofficial-sigs.sh | 42 ++++++++++++++++----------------------- config/master.conf | 2 +- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index f628c8d6..25902708 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -393,7 +393,7 @@ function clamav_files() { # Manage the databases and allow multi-dimensions as well as global overrides # Since the datbases are basically a multi-dimentional associative arrays in bash -# ratings: LOW | MEDIUM | HIGH | REQUIRED | LOWONLY | MEDIUMONLY | LOWMEDIUMONLY | MEDIUMHIGHONLY | HIGHONLY | DISABLED +# ratings: LOW | MEDIUM | HIGH | REQUIRED | LOWONLY | MEDIUMONLY | LOWMEDIUMONLY | DISABLED function xshok_database() { # rating database_array # Assign current_rating="${1}" @@ -428,35 +428,25 @@ function xshok_database() { # rating database_array new_dbs+=( "$db_name" ) fi elif [ "$current_rating" == "MEDIUM" ] ; then - if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "LOW" ] || [ "$db_name_rating" == "LOWMEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUMHIGHONLY" ] ; then + if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "LOW" ] || [ "$db_name_rating" == "LOWMEDIUMONLY" ] ; then new_dbs+=( "$db_name" ) fi elif [ "$current_rating" == "HIGH" ] ; then - if [ "$db_name_rating" == "HIGHONLY" ] || [ "$db_name_rating" == "HIGH" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "LOW" ] || [ "$db_name_rating" == "MEDIUMHIGHONLY" ] ; then + if [ "$db_name_rating" == "HIGH" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "LOW" ]; then new_dbs+=( "$db_name" ) fi - elif [ "$current_rating" == "LOWONLY" ] ; then - if [ "$db_name_rating" == "LOWONLY" ] || [ "$db_name_rating" == "LOW" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "MEDIUMONLY" ] ; then - if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "LOWMEDIUMONLY" ] ; then - if [ "$db_name_rating" == "LOWMEDIUMONLY" ] || [ "$db_name_rating" == "LOW" ] || [ "$db_name_rating" == "MEDIUM" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "MEDIUMHIGHONLY" ] ; then - if [ "$db_name_rating" == "MEDIUMHIGHONLY" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "HIGH" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "HIGHONLY" ] ; then - if [ "$db_name_rating" == "HIGHONLY" ] || [ "$db_name_rating" == "HIGH" ] ; then - new_dbs+=( "$db_name" ) - fi - fi - fi + elif [ "$current_rating" == "LOWONLY" ] ; then + if [ "$db_name_rating" == "LOWONLY" ] || [ "$db_name_rating" == "LOW" ] ; then + new_dbs+=( "$db_name" ) + fi + elif [ "$current_rating" == "MEDIUMONLY" ] ; then + if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] ; then + new_dbs+=( "$db_name" ) + fi + elif [ "$current_rating" == "LOWMEDIUMONLY" ] ; then + if [ "$db_name_rating" == "LOWMEDIUMONLY" ] || [ "$db_name_rating" == "LOW" ] || [ "$db_name_rating" == "MEDIUM" ] ; then + new_dbs+=( "$db_name" ) + fi fi fi done @@ -465,6 +455,8 @@ function xshok_database() { # rating database_array echo "${new_dbs[@]}" | xargs # Remove extra whitespace } + + ################################################################################ # ADDITIONAL PROGRAM FUNCTIONS ################################################################################ diff --git a/config/master.conf b/config/master.conf index d7b48227..677216c2 100644 --- a/config/master.conf +++ b/config/master.conf @@ -174,7 +174,7 @@ default_dbs_rating="MEDIUM" # Per Database # These ratings will override the global rating for the specific database -# valid rating: LOW, MEDIUM, HIGH, DISABLED +# valid ratings: LOW | MEDIUM | HIGH | REQUIRED | LOWONLY | MEDIUMONLY | LOWMEDIUMONLY | MEDIUMHIGHONLY | HIGHONLY | DISABLED #linuxmalwaredetect_dbs_rating="" #sanesecurity_dbs_rating="" #securiteinfo_dbs_rating="" From 9821ede18bcad4b78bbfd519ea6ac640dc6655b7 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 03:03:01 +0200 Subject: [PATCH 037/101] valid ratings: LOW | MEDIUM | HIGH | REQUIRED | LOWONLY | MEDIUMONLY | LOWMEDIUMONLY | DISABLED --- config/master.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/master.conf b/config/master.conf index 677216c2..dc65ae73 100644 --- a/config/master.conf +++ b/config/master.conf @@ -174,7 +174,7 @@ default_dbs_rating="MEDIUM" # Per Database # These ratings will override the global rating for the specific database -# valid ratings: LOW | MEDIUM | HIGH | REQUIRED | LOWONLY | MEDIUMONLY | LOWMEDIUMONLY | MEDIUMHIGHONLY | HIGHONLY | DISABLED +# valid ratings: LOW | MEDIUM | HIGH | REQUIRED | LOWONLY | MEDIUMONLY | LOWMEDIUMONLY | DISABLED #linuxmalwaredetect_dbs_rating="" #sanesecurity_dbs_rating="" #securiteinfo_dbs_rating="" From 3ec97792c3df3b889f2f8cf9c1009e35c3888fc7 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 03:10:18 +0200 Subject: [PATCH 038/101] remove support for very old format of db lists --- clamav-unofficial-sigs.sh | 48 +++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 25902708..a0edefa7 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -412,9 +412,6 @@ function xshok_database() { # rating database_array if [ -z "$current_rating" ] ; then # YARA rules are disabled new_dbs+=( "$db_name" ) else - if [[ ! "$db_name" = *"|"* ]] ; then # This old format - new_dbs+=( "$db_name" ) - else db_name_rating="${db_name#*|}" db_name="${db_name%|*}" @@ -447,8 +444,51 @@ function xshok_database() { # rating database_array if [ "$db_name_rating" == "LOWMEDIUMONLY" ] || [ "$db_name_rating" == "LOW" ] || [ "$db_name_rating" == "MEDIUM" ] ; then new_dbs+=( "$db_name" ) fi - fi + fi fi + fi + done + fi + fi + echo "${new_dbs[@]}" | xargs # Remove extra whitespace +} + +# Manage the databases to be removed and allow multi-dimensions as well as global overrides +# Since the datbases are basically a multi-dimentional associative arrays in bash +# ratings: LOW | MEDIUM | HIGH | REQUIRED | LOWONLY | MEDIUMONLY | LOWMEDIUMONLY | DISABLED +function xshok_remove_database() { # rating database_array + # Assign + current_rating="${1}" + declare -a current_dbs=( "${@:2}" ) + # Zero + declare -a new_dbs=( ) + if [ -n "${current_dbs[0]}" ] ; then + if [ ${#current_dbs} -ge 1 ] ; then + for db_name in "${current_dbs[@]}" ; do + db_name_rating="${db_name#*|}" + db_name="${db_name%|*}" + # Checks + if [ "$enable_yararules" == "no" ] ; then # YARA rules are disabled + if [[ "$db_name" == *".yar"* ]] ; then # If it's the value you want to delete + new_dbs+=( "$db_name" ) + fi + else + if [ "$db_name_rating" == "DISABLED" ] ; then + new_dbs+=( "$db_name" ) + elif [ "$current_rating" == "HIGH" ] ; then + if [ "$db_name_rating" == "LOWONLY" ] || [ "$db_name_rating" == "LOWMEDIUMONLY" ] ||[ "$db_name_rating" == "MEDIUMONLY" ] ; then + new_dbs+=( "$db_name" ) + fi + elif [ "$current_rating" == "MEDIUM" ] ; then + if [ "$db_name_rating" == "HIGH" ] ; then + new_dbs+=( "$db_name" ) + fi + elif [ "$current_rating" == "LOW" ] ; then + if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "HIGH" ]; then + new_dbs+=( "$db_name" ) + fi + fi + fi done fi fi From be43e10d2a835b7c5e8354d87e1adebd2020b067 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 03:18:15 +0200 Subject: [PATCH 039/101] ensure the logic is still working --- clamav-unofficial-sigs.sh | 12 ------------ config/master.conf | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index a0edefa7..9509a2ee 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -432,18 +432,6 @@ function xshok_database() { # rating database_array if [ "$db_name_rating" == "HIGH" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "LOW" ]; then new_dbs+=( "$db_name" ) fi - elif [ "$current_rating" == "LOWONLY" ] ; then - if [ "$db_name_rating" == "LOWONLY" ] || [ "$db_name_rating" == "LOW" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "MEDIUMONLY" ] ; then - if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "LOWMEDIUMONLY" ] ; then - if [ "$db_name_rating" == "LOWMEDIUMONLY" ] || [ "$db_name_rating" == "LOW" ] || [ "$db_name_rating" == "MEDIUM" ] ; then - new_dbs+=( "$db_name" ) - fi fi fi fi diff --git a/config/master.conf b/config/master.conf index dc65ae73..e694ce3c 100644 --- a/config/master.conf +++ b/config/master.conf @@ -174,7 +174,7 @@ default_dbs_rating="MEDIUM" # Per Database # These ratings will override the global rating for the specific database -# valid ratings: LOW | MEDIUM | HIGH | REQUIRED | LOWONLY | MEDIUMONLY | LOWMEDIUMONLY | DISABLED +# valid ratings: LOW | MEDIUM | HIGH | DISABLED #linuxmalwaredetect_dbs_rating="" #sanesecurity_dbs_rating="" #securiteinfo_dbs_rating="" From 20d54c7237c7f4c1489b369240ca0b3ce8236e40 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 03:25:24 +0200 Subject: [PATCH 040/101] break CI to test --- clamav-unofficial-sigs.sh | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 9509a2ee..82132c27 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2207,16 +2207,29 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then if [ -n "$sanesecurity_dbs" ] ; then if [ -n "$sanesecurity_dbs_rating" ] ; then temp_db="$(xshok_database "$sanesecurity_dbs_rating" "${sanesecurity_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$sanesecurity_dbs_rating" "${sanesecurity_dbs[@]}")" else temp_db="$(xshok_database "$default_dbs_rating" "${sanesecurity_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$default_dbs_rating" "${sanesecurity_dbs[@]}")" + fi + sanesecurity_dbs=( ) + if [ -n "$temp_db" ] ; then + read -r -a sanesecurity_dbs <<< "$temp_db" fi - sanesecurity_dbs=( ) - if [ -n "$temp_db" ] ; then - #sanesecurity_dbs=( $temp_db ) - read -r -a sanesecurity_dbs <<< "$temp_db" - fi fi +else + temp__remove_db="$(xshok_remove_database "DISABLED" "${sanesecurity_dbs[@]}")" fi +sanesecurity_remove_dbs=( ) +if [ -n "$temp__remove_db" ] ; then + read -r -a sanesecurity_remove_dbs <<< "$temp__remove_db" +fi + +echo "**********DEBUG :: BEGIN :: FAIL CI *************" +echo "${sanesecurity_remove_dbs[@]}" +echo "**********DEBUG :: END :: FAIL CI *************" + + ############################################################################################ if [ "$securiteinfo_enabled" == "yes" ] ; then if [ -n "$securiteinfo_dbs" ] ; then From e5f6c87073f7f2317bd4a5c3f2393e25882ac311 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 03:26:54 +0200 Subject: [PATCH 041/101] typo --- clamav-unofficial-sigs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 82132c27..cb7f96db 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2218,11 +2218,11 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then fi fi else - temp__remove_db="$(xshok_remove_database "DISABLED" "${sanesecurity_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "DISABLED" "${sanesecurity_dbs[@]}")" fi sanesecurity_remove_dbs=( ) if [ -n "$temp__remove_db" ] ; then - read -r -a sanesecurity_remove_dbs <<< "$temp__remove_db" + read -r -a sanesecurity_remove_dbs <<< "$temp_remove_db" fi echo "**********DEBUG :: BEGIN :: FAIL CI *************" From e5b1686e174156d610faa9a69738a62cc9cbcd27 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 03:46:03 +0200 Subject: [PATCH 042/101] test securiteinfo --- .t/tests/user.conf | 8 ++++---- clamav-unofficial-sigs.sh | 38 +++++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.t/tests/user.conf b/.t/tests/user.conf index c4c4ab7b..ad835ee6 100644 --- a/.t/tests/user.conf +++ b/.t/tests/user.conf @@ -29,10 +29,10 @@ default_dbs_rating="HIGH" # Per Database # These ratings will override the global rating for the specific database # valid rating: LOW, MEDIUM, HIGH, DISABLE -sanesecurity_dbs_rating="HIGH" -#securiteinfo_dbs_rating="" -#linuxmalwaredetect_dbs_rating="" -#yararulesproject_dbs_rating="" +sanesecurity_dbs_rating="LOW" +securiteinfo_dbs_rating="DISABLE" +linuxmalwaredetect_dbs_rating="DISABLE" +yararulesproject_dbs_rating="DISABLE" enable_gpg="no" diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index cb7f96db..6423142c 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2221,22 +2221,18 @@ else temp_remove_db="$(xshok_remove_database "DISABLED" "${sanesecurity_dbs[@]}")" fi sanesecurity_remove_dbs=( ) -if [ -n "$temp__remove_db" ] ; then +if [ -n "$temp_remove_db" ] ; then read -r -a sanesecurity_remove_dbs <<< "$temp_remove_db" fi - -echo "**********DEBUG :: BEGIN :: FAIL CI *************" -echo "${sanesecurity_remove_dbs[@]}" -echo "**********DEBUG :: END :: FAIL CI *************" - - ############################################################################################ if [ "$securiteinfo_enabled" == "yes" ] ; then if [ -n "$securiteinfo_dbs" ] ; then if [ -n "$securiteinfo_dbs_rating" ] ; then temp_db="$(xshok_database "$securiteinfo_dbs_rating" "${securiteinfo_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$securiteinfo_dbs_rating" "${securiteinfo_dbs[@]}")" else temp_db="$(xshok_database "$default_dbs_rating" "${securiteinfo_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$default_dbs_rating" "${securiteinfo_dbs[@]}")" fi securiteinfo_dbs=( ) if [ -n "$temp_db" ] ; then @@ -2244,18 +2240,38 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then read -r -a securiteinfo_dbs <<< "$temp_db" fi fi +else + temp_remove_db="$(xshok_remove_database "DISABLED" "${securiteinfo_dbs[@]}")" +fi +securiteinfo_remove_dbs=( ) +if [ -n "$temp_remove_db" ] ; then + read -r -a securiteinfo_remove_dbs <<< "$temp_remove_db" +fi +if [ "$securiteinfo_enabled" == "yes" ] ; then if [ -n "$securiteinfo_premium_dbs" ] && [ "$securiteinfo_premium" == "yes" ] ; then if [ -n "$securiteinfo_dbs_rating" ] ; then temp_db="$(xshok_database "$securiteinfo_dbs_rating" "${securiteinfo_premium_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$securiteinfo_dbs_rating" "${securiteinfo_premium_dbs[@]}")" else temp_db="$(xshok_database "$default_dbs_rating" "${securiteinfo_premium_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$default_dbs_rating" "${securiteinfo_premium_dbs[@]}")" + fi + if [ -n "$temp_db" ] ; then + #securiteinfo_dbs=( $temp_db ) + read -r -a securiteinfo_dbs <<< "$temp_db" fi - if [ -n "$temp_db" ] ; then - #securiteinfo_dbs=( $temp_db ) - read -r -a securiteinfo_dbs <<< "$temp_db" - fi fi +else + temp_remove_db="$(xshok_remove_database "DISABLED" "${securiteinfo_premium_dbs[@]}")" fi + +if [ -n "$temp_remove_db" ] ; then + read -r -a securiteinfo_remove_dbs <<< "$temp_remove_db" +fi + +echo "**********DEBUG :: BEGIN :: FAIL CI *************" +echo "${securiteinfo_remove_dbs[@]}" +echo "**********DEBUG :: END :: FAIL CI *************" ############################################################################################ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then if [ -n "$linuxmalwaredetect_dbs" ] ; then From 2bc4f1c1b5634ea1dbe831aa51cc4f0e9205632c Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 03:52:44 +0200 Subject: [PATCH 043/101] more debugging --- clamav-unofficial-sigs.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 6423142c..bed177ae 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2269,9 +2269,12 @@ if [ -n "$temp_remove_db" ] ; then read -r -a securiteinfo_remove_dbs <<< "$temp_remove_db" fi -echo "**********DEBUG :: BEGIN :: FAIL CI *************" +echo "**********DEBUG :: BEGIN *************" +echo "sanesecurity_remove_dbs" +echo "${sanesecurity_remove_dbs[@]}" +echo "securiteinfo_remove_dbs" echo "${securiteinfo_remove_dbs[@]}" -echo "**********DEBUG :: END :: FAIL CI *************" +echo "**********DEBUG :: END *************" ############################################################################################ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then if [ -n "$linuxmalwaredetect_dbs" ] ; then From edacc846da3bc23103294cfd2e3f510a906d2cf1 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 04:21:51 +0200 Subject: [PATCH 044/101] generate remove database databases --- clamav-unofficial-sigs.sh | 103 ++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 31 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index bed177ae..68a1ebcd 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2257,76 +2257,90 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then temp_remove_db="$(xshok_remove_database "$default_dbs_rating" "${securiteinfo_premium_dbs[@]}")" fi if [ -n "$temp_db" ] ; then - #securiteinfo_dbs=( $temp_db ) read -r -a securiteinfo_dbs <<< "$temp_db" fi fi else temp_remove_db="$(xshok_remove_database "DISABLED" "${securiteinfo_premium_dbs[@]}")" fi - if [ -n "$temp_remove_db" ] ; then read -r -a securiteinfo_remove_dbs <<< "$temp_remove_db" fi - -echo "**********DEBUG :: BEGIN *************" -echo "sanesecurity_remove_dbs" -echo "${sanesecurity_remove_dbs[@]}" -echo "securiteinfo_remove_dbs" -echo "${securiteinfo_remove_dbs[@]}" -echo "**********DEBUG :: END *************" ############################################################################################ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then if [ -n "$linuxmalwaredetect_dbs" ] ; then if [ -n "$linuxmalwaredetect_dbs_rating" ] ; then temp_db="$(xshok_database "$linuxmalwaredetect_dbs_rating" "${linuxmalwaredetect_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$linuxmalwaredetect_dbs_rating" "${linuxmalwaredetect_dbs[@]}")" else temp_db="$(xshok_database "$default_dbs_rating" "${linuxmalwaredetect_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$default_dbs_rating" "${linuxmalwaredetect_dbs[@]}")" fi linuxmalwaredetect_dbs=( ) if [ -n "$temp_db" ] ; then - #linuxmalwaredetect_dbs=( $temp_db ) read -r -a linuxmalwaredetect_dbs <<< "$temp_db" fi fi +else + temp_remove_db="$(xshok_remove_database "DISABLED" "${linuxmalwaredetect_dbs[@]}")" +fi +linuxmalwaredetect_remove_dbs=( ) +if [ -n "$temp_remove_db" ] ; then + read -r -a linuxmalwaredetect_remove_dbs <<< "$temp_remove_db" fi ############################################################################################ if [ "$malwareexpert_enabled" == "yes" ] ; then if [ -n "$malwareexpert_dbs" ] ; then if [ -n "$malwareexpert_dbs_rating" ] ; then temp_db="$(xshok_database "$malwareexpert_dbs_rating" "${malwareexpert_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$malwareexpert_dbs_rating" "${malwareexpert_dbs[@]}")" else temp_db="$(xshok_database "$default_dbs_rating" "${malwareexpert_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$default_dbs_rating" "${malwareexpert_dbs[@]}")" fi malwareexpert_dbs=( ) if [ -n "$temp_db" ] ; then - #malwareexpert_dbs=( $temp_db ) read -r -a malwareexpert_dbs <<< "$temp_db" fi fi +else + temp_remove_db="$(xshok_remove_database "DISABLED" "${malwareexpert_dbs[@]}")" +fi +malwareexpert_remove_dbs=( ) +if [ -n "$temp_remove_db" ] ; then + read -r -a malwareexpert_remove_dbs <<< "$temp_remove_db" fi ############################################################################################ if [ "$yararulesproject_enabled" == "yes" ] ; then if [ -n "$yararulesproject_dbs" ] ; then if [ -n "$yararulesproject_dbs_rating" ] ; then temp_db="$(xshok_database "$yararulesproject_dbs_rating" "${yararulesproject_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$yararulesproject_dbs_rating" "${yararulesproject_dbs[@]}")" else temp_db="$(xshok_database "$default_dbs_rating" "${yararulesproject_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$default_dbs_rating" "${yararulesproject_dbs[@]}")" fi yararulesproject_dbs=( ) if [ -n "$temp_db" ] ; then - #yararulesproject_dbs=( $temp_db ) read -r -a yararulesproject_dbs <<< "$temp_db" fi fi +else + temp_remove_db="$(xshok_remove_database "DISABLED" "${yararulesproject_dbs[@]}")" +fi +yararulesproject_remove_dbs=( ) +if [ -n "$temp_remove_db" ] ; then + read -r -a yararulesproject_remove_dbs <<< "$temp_remove_db" fi ############################################################################################ if [ "$urlhaus_enabled" == "yes" ] ; then if [ -n "$urlhaus_dbs" ] ; then if [ -n "$urlhaus_dbs_rating" ] ; then temp_db="$(xshok_database "$urlhaus_dbs_rating" "${urlhaus_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$urlhaus_dbs_rating" "${urlhaus_dbs[@]}")" else temp_db="$(xshok_database "$default_dbs_rating" "${urlhaus_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$default_dbs_rating" "${urlhaus_dbs[@]}")" fi urlhaus_dbs=( ) if [ -n "$temp_db" ] ; then @@ -2334,30 +2348,57 @@ if [ "$urlhaus_enabled" == "yes" ] ; then read -r -a urlhaus_dbs <<< "$temp_db" fi fi +else + temp_remove_db="$(xshok_remove_database "DISABLED" "${urlhaus_dbs[@]}")" +fi +urlhaus_remove_dbs=( ) +if [ -n "$temp_remove_db" ] ; then + read -r -a urlhaus_remove_dbs <<< "$temp_remove_db" fi ############################################################################################ -# Set the variables for MalwarePatrol -if [ "$malwarepatrol_product_code" != "8" ] ; then - # assumption, free product code is always 8 (non-free product code is never 8) - malwarepatrol_free="no" -fi -if [ "$malwarepatrol_free" == "yes" ] ; then - malwarepatrol_product_code="8" - malwarepatrol_list="clamav_basic" +if [ "$malwarepatrol_enabled" == "yes" ] ; then + # Set the variables for MalwarePatrol + if [ "$malwarepatrol_product_code" != "8" ] ; then + # assumption, free product code is always 8 (non-free product code is never 8) + malwarepatrol_free="no" + fi + if [ "$malwarepatrol_free" == "yes" ] ; then + malwarepatrol_product_code="8" + malwarepatrol_list="clamav_basic" + else + if [ -z $malwarepatrol_list ] ; then + malwarepatrol_list="clamav_basic" + fi + if [ -z $malwarepatrol_product_code ] ; then + # Not sure, it may be better to return an error. + malwarepatrol_product_code=8 + fi + fi + if [ -z "$malwarepatrol_db" ] ; then + malwarepatrol_db="malwarepatrol.db" + fi + malwarepatrol_url="${malwarepatrol_url}?receipt=${malwarepatrol_receipt_code}&product=${malwarepatrol_product_code}&list=${malwarepatrol_list}" else - if [ -z $malwarepatrol_list ] ; then - malwarepatrol_list="clamav_basic" - fi - if [ -z $malwarepatrol_product_code ] ; then - # Not sure, it may be better to return an error. - malwarepatrol_product_code=8 - fi -fi -if [ -z "$malwarepatrol_db" ] ; then - malwarepatrol_db="malwarepatrol.db" + malwarepatrol_remove_dbs=( "malwarepatrol.db" ) fi +############################################################################################ -malwarepatrol_url="${malwarepatrol_url}?receipt=${malwarepatrol_receipt_code}&product=${malwarepatrol_product_code}&list=${malwarepatrol_list}" +echo "**********DEBUG :: BEGIN *************" +echo "sanesecurity_remove_dbs:" +echo "${sanesecurity_remove_dbs[@]}" +echo "securiteinfo_remove_dbs:" +echo "${securiteinfo_remove_dbs[@]}" +echo "linuxmalwaredetect_remove_dbs:" +echo "${linuxmalwaredetect_remove_dbs[@]}" +echo "malwareexpert_remove_dbs:" +echo "${malwareexpert_remove_dbs[@]}" +echo "yararulesproject_remove_dbs:" +echo "${yararulesproject_remove_dbs[@]}" +echo "urlhaus_remove_dbs:" +echo "${urlhaus_remove_dbs[@]}" +echo "malwarepatrol_remove_dbs:" +echo "${malwarepatrol_remove_dbs[@]}" +echo "**********DEBUG :: END *************" # If "ham_dir" variable is set, then create initial whitelist files (skipped if first-time script run). test_dir="$work_dir/test" From 64eba4e27d01d52448c8cc85213e4f92667d9c7c Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 04:34:49 +0200 Subject: [PATCH 045/101] more progress --- clamav-unofficial-sigs.sh | 39 +++++++++++++++++++++++++++++++++++++++ config/master.conf | 23 +++++++++++------------ 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 68a1ebcd..805d4d73 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -412,6 +412,9 @@ function xshok_database() { # rating database_array if [ -z "$current_rating" ] ; then # YARA rules are disabled new_dbs+=( "$db_name" ) else + if [[ ! "$db_name" = *"|"* ]] ; then # This old format + new_dbs+=( "$db_name" ) + else db_name_rating="${db_name#*|}" db_name="${db_name%|*}" @@ -434,6 +437,7 @@ function xshok_database() { # rating database_array fi fi fi + fi fi done fi @@ -2400,6 +2404,41 @@ echo "malwarepatrol_remove_dbs:" echo "${malwarepatrol_remove_dbs[@]}" echo "**********DEBUG :: END *************" +if [ -n "${sanesecurity_remove_dbs[0]}" ] ; then + for db_file in "${sanesecurity_dbs[@]}" ; do + if [ -f "${work_dir_sanesecurity}/${db_file}" ] ; then + echo "Found: ${work_dir_sanesecurity}/${db_file}" + else + echo "Not: ${work_dir_sanesecurity}/${db_file}" + fi + if [ -f "${clam_dbs}/${db_file}" ] ; then + echo "Found: ${clam_dbs}/${db_file}" + else + echo "Not: ${clam_dbs}/${db_file}" + fi + done +fi +if [ -n "${securiteinfo_remove_dbs[0]}" ] ; then + for db_file in "${securiteinfo_remove_dbs[@]}" ; do + if [ -f "${work_dir_securiteinfo}/${db_file}" ] ; then + echo "Found: ${work_dir_securiteinfo}/${db_file}" + else + echo "Not: ${work_dirsecuriteinfo}/${db_file}" + fi + if [ -f "${clam_dbs}/${db_file}" ] ; then + echo "Found: ${clam_dbs}/${db_file}" + else + echo "Not: ${clam_dbs}/${db_file}" + fi + done +fi + + + +############################################################################################ + + + # If "ham_dir" variable is set, then create initial whitelist files (skipped if first-time script run). test_dir="$work_dir/test" if [ -n "$ham_dir" ] && [ -d "$work_dir" ] && [ ! -d "$test_dir" ] ; then diff --git a/config/master.conf b/config/master.conf index e694ce3c..ffdd50ee 100644 --- a/config/master.conf +++ b/config/master.conf @@ -441,19 +441,18 @@ maldocs/Maldoc_VBA_macro_code.yar|HIGH maldocs/Maldoc_Word_2007_XML_Flat_OPC.yar|HIGH # Yara Rules aimed to detect well-known software packers, that can be used by malware to hide itself. packers/Javascript_exploit_and_obfuscation.yar|HIGH -) #END yararulesproject DATABASES - -declare -a yararulesproject_dbs_blacklisted=( -email/attachment.yar # detects all emails with attachments -email/image.yar # detects all emails with images -email/urls.yar # detects all emails with urls -crypto/crypto_signatures.yar # detects all files which are encrypted +# DISABLED +# NOT SUPPORTED OR CRASHING CLAMAV +email/attachment.yar|DISABLED # detects all emails with attachments +email/image.yar|DISABLED # detects all emails with images +email/urls.yar|DISABLED # detects all emails with urls +crypto/crypto_signatures.yar|DISABLED # detects all files which are encrypted # These files use module includes not supported by ClamAV -packers/packer_compiler_signatures.yar -packers/packer.yar -packers/peid.yar -antidebug_antivm -) +packers/packer_compiler_signatures.yar|DISABLED +packers/packer.yar|DISABLED +packers/peid.yar|DISABLED +antidebug_antivm|DISABLED +) #END yararulesproject DATABASES declare -a yararulesproject_dbs_catagories=( #LOW From 17330a470e3e53c3c4a80cf9745f8b4eeeaf120d Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 04:42:16 +0200 Subject: [PATCH 046/101] pre-test of unused file removal --- clamav-unofficial-sigs.sh | 58 +++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 805d4d73..b9e439b0 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2408,13 +2408,9 @@ if [ -n "${sanesecurity_remove_dbs[0]}" ] ; then for db_file in "${sanesecurity_dbs[@]}" ; do if [ -f "${work_dir_sanesecurity}/${db_file}" ] ; then echo "Found: ${work_dir_sanesecurity}/${db_file}" - else - echo "Not: ${work_dir_sanesecurity}/${db_file}" fi if [ -f "${clam_dbs}/${db_file}" ] ; then echo "Found: ${clam_dbs}/${db_file}" - else - echo "Not: ${clam_dbs}/${db_file}" fi done fi @@ -2422,13 +2418,59 @@ if [ -n "${securiteinfo_remove_dbs[0]}" ] ; then for db_file in "${securiteinfo_remove_dbs[@]}" ; do if [ -f "${work_dir_securiteinfo}/${db_file}" ] ; then echo "Found: ${work_dir_securiteinfo}/${db_file}" - else - echo "Not: ${work_dirsecuriteinfo}/${db_file}" fi if [ -f "${clam_dbs}/${db_file}" ] ; then echo "Found: ${clam_dbs}/${db_file}" - else - echo "Not: ${clam_dbs}/${db_file}" + fi + done +fi +if [ -n "${linuxmalwaredetect_remove_dbs[0]}" ] ; then + for db_file in "${linuxmalwaredetect_remove_dbs[@]}" ; do + if [ -f "${work_dir_linuxmalwaredetect}/${db_file}" ] ; then + echo "Found: ${work_dir_linuxmalwaredetect}/${db_file}" + fi + if [ -f "${clam_dbs}/${db_file}" ] ; then + echo "Found: ${clam_dbs}/${db_file}" + fi + done +fi +if [ -n "${malwareexpert_remove_dbs[0]}" ] ; then + for db_file in "${malwareexpert_remove_dbs[@]}" ; do + if [ -f "${work_dir_malwareexpert}/${db_file}" ] ; then + echo "Found: ${work_dir_malwareexpert}/${db_file}" + fi + if [ -f "${clam_dbs}/${db_file}" ] ; then + echo "Found: ${clam_dbs}/${db_file}" + fi + done +fi +if [ -n "${yararulesproject_remove_dbs[0]}" ] ; then + for db_file in "${yararulesproject_remove_dbs[@]}" ; do + if [ -f "${work_dir_yararulesproject}/${db_file}" ] ; then + echo "Found: ${work_dir_yararulesproject}/${db_file}" + fi + if [ -f "${clam_dbs}/${db_file}" ] ; then + echo "Found: ${clam_dbs}/${db_file}" + fi + done +fi +if [ -n "${urlhaus_remove_dbs[0]}" ] ; then + for db_file in "${urlhaus_remove_dbs[@]}" ; do + if [ -f "${work_dir_urlhaus}/${db_file}" ] ; then + echo "Found: ${work_dir_urlhaus}/${db_file}" + fi + if [ -f "${clam_dbs}/${db_file}" ] ; then + echo "Found: ${clam_dbs}/${db_file}" + fi + done +fi +if [ -n "${malwarepatrol_remove_dbs[0]}" ] ; then + for db_file in "${malwarepatrol_remove_dbs[@]}" ; do + if [ -f "${work_dir_malwarepatrol}/${db_file}" ] ; then + echo "Found: ${work_dir_malwarepatrol}/${db_file}" + fi + if [ -f "${clam_dbs}/${db_file}" ] ; then + echo "Found: ${clam_dbs}/${db_file}" fi done fi From f384692888a2d76961602631e910310e34a45ea4 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 04:50:28 +0200 Subject: [PATCH 047/101] live test of database rating downgrade --- .t/ci-test.sh | 27 +++++++++++++++++-------- clamav-unofficial-sigs.sh | 42 ++++++++++++++++++++++++++------------- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index 45539543..b96d0571 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -130,13 +130,24 @@ else echo .. ERROR exit 1 fi - -echo "EXTRA DEBUG INFORMATION" -echo "================" -echo "/var/lib/clamav/" -echo "================" +echo "===== /var/lib/clamav/ =====" ls -laFh /var/lib/clamav/ -echo "================" -echo "/var/lib/clamav-unofficial-sigs/dbs-si" -echo "================" +echo "===== /var/lib/clamav-unofficial-sigs/dbs-si =====" ls -laFh /var/lib/clamav-unofficial-sigs/dbs-si +echo "================" + + +echo "Was /var/lib/clamav//spam_marketing.ndb removed ?" +if [ -e "/var/lib/clamav//spam_marketing.ndb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "Was /var/lib/clamav-unofficial-sigs/dbs-si/spam_marketing.ndb removed ?" +if [ -e "/var/lib/clamav-unofficial-sigs/dbs-si/spam_marketing.ndb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index b9e439b0..e266a006 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2407,70 +2407,84 @@ echo "**********DEBUG :: END *************" if [ -n "${sanesecurity_remove_dbs[0]}" ] ; then for db_file in "${sanesecurity_dbs[@]}" ; do if [ -f "${work_dir_sanesecurity}/${db_file}" ] ; then - echo "Found: ${work_dir_sanesecurity}/${db_file}" + echo "Removing unused file: ${work_dir_sanesecurity}/${db_file}" + rm -f "${work_dir_sanesecurity}/${db_file}" fi if [ -f "${clam_dbs}/${db_file}" ] ; then - echo "Found: ${clam_dbs}/${db_file}" + echo "Removing unused file: ${clam_dbs}/${db_file}" + rm -f "${clam_dbs}/${db_file}" fi done fi if [ -n "${securiteinfo_remove_dbs[0]}" ] ; then for db_file in "${securiteinfo_remove_dbs[@]}" ; do if [ -f "${work_dir_securiteinfo}/${db_file}" ] ; then - echo "Found: ${work_dir_securiteinfo}/${db_file}" + echo "Removing unused file: ${work_dir_securiteinfo}/${db_file}" + rm -f "${work_dir_securiteinfo}/${db_file}" fi if [ -f "${clam_dbs}/${db_file}" ] ; then - echo "Found: ${clam_dbs}/${db_file}" + echo "Removing unused file: ${clam_dbs}/${db_file}" + rm -f "${clam_dbs}/${db_file}" fi done fi if [ -n "${linuxmalwaredetect_remove_dbs[0]}" ] ; then for db_file in "${linuxmalwaredetect_remove_dbs[@]}" ; do if [ -f "${work_dir_linuxmalwaredetect}/${db_file}" ] ; then - echo "Found: ${work_dir_linuxmalwaredetect}/${db_file}" + echo "Removing unused file: ${work_dir_linuxmalwaredetect}/${db_file}" + rm -f "${work_dir_linuxmalwaredetect}/${db_file}" fi if [ -f "${clam_dbs}/${db_file}" ] ; then - echo "Found: ${clam_dbs}/${db_file}" + echo "Removing unused file: ${clam_dbs}/${db_file}" + rm -f "${clam_dbs}/${db_file}" fi done fi if [ -n "${malwareexpert_remove_dbs[0]}" ] ; then for db_file in "${malwareexpert_remove_dbs[@]}" ; do if [ -f "${work_dir_malwareexpert}/${db_file}" ] ; then - echo "Found: ${work_dir_malwareexpert}/${db_file}" + echo "Removing unused file: ${work_dir_malwareexpert}/${db_file}" + rm -f "${work_dir_malwareexpert}/${db_file}" fi if [ -f "${clam_dbs}/${db_file}" ] ; then - echo "Found: ${clam_dbs}/${db_file}" + echo "Removing unused file: ${clam_dbs}/${db_file}" + rm -f "${clam_dbs}/${db_file}" fi done fi if [ -n "${yararulesproject_remove_dbs[0]}" ] ; then for db_file in "${yararulesproject_remove_dbs[@]}" ; do if [ -f "${work_dir_yararulesproject}/${db_file}" ] ; then - echo "Found: ${work_dir_yararulesproject}/${db_file}" + echo "Removing unused file: ${work_dir_yararulesproject}/${db_file}" + rm -f "${work_dir_yararulesproject}/${db_file}" fi if [ -f "${clam_dbs}/${db_file}" ] ; then - echo "Found: ${clam_dbs}/${db_file}" + echo "Removing unused file: ${clam_dbs}/${db_file}" + rm -f "${clam_dbs}/${db_file}" fi done fi if [ -n "${urlhaus_remove_dbs[0]}" ] ; then for db_file in "${urlhaus_remove_dbs[@]}" ; do if [ -f "${work_dir_urlhaus}/${db_file}" ] ; then - echo "Found: ${work_dir_urlhaus}/${db_file}" + echo "Removing unused file: ${work_dir_urlhaus}/${db_file}" + rm -f "${work_dir_urlhaus}/${db_file}" fi if [ -f "${clam_dbs}/${db_file}" ] ; then - echo "Found: ${clam_dbs}/${db_file}" + echo "Removing unused file: ${clam_dbs}/${db_file}" + rm -f "${clam_dbs}/${db_file}" fi done fi if [ -n "${malwarepatrol_remove_dbs[0]}" ] ; then for db_file in "${malwarepatrol_remove_dbs[@]}" ; do if [ -f "${work_dir_malwarepatrol}/${db_file}" ] ; then - echo "Found: ${work_dir_malwarepatrol}/${db_file}" + echo "Removing unused file: ${work_dir_malwarepatrol}/${db_file}" + rm -f "${work_dir_malwarepatrol}/${db_file}" fi if [ -f "${clam_dbs}/${db_file}" ] ; then - echo "Found: ${clam_dbs}/${db_file}" + echo "Removing unused file: ${clam_dbs}/${db_file}" + rm -f "${clam_dbs}/${db_file}" fi done fi From 525a042c85ec60178912cd32ef4aecfc884a612a Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 04:56:38 +0200 Subject: [PATCH 048/101] improve ci tests for downgrades --- .t/ci-test.sh | 16 ++++++++++------ README.md | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index b96d0571..fff64eb0 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -120,6 +120,11 @@ else echo .. ERROR exit 1 fi +echo "===== BEFORE /var/lib/clamav/ =====" +ls -laFh /var/lib/clamav/ +echo "===== BEFORE /var/lib/clamav-unofficial-sigs/dbs-si =====" +ls -laFh /var/lib/clamav-unofficial-sigs/dbs-si +echo "================" echo "running script verbose with LOW ratings" cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user_low.conf @@ -130,22 +135,21 @@ else echo .. ERROR exit 1 fi -echo "===== /var/lib/clamav/ =====" +echo "===== AFTER /var/lib/clamav/ =====" ls -laFh /var/lib/clamav/ -echo "===== /var/lib/clamav-unofficial-sigs/dbs-si =====" +echo "===== AFTER /var/lib/clamav-unofficial-sigs/dbs-si =====" ls -laFh /var/lib/clamav-unofficial-sigs/dbs-si echo "================" - -echo "Was /var/lib/clamav//spam_marketing.ndb removed ?" -if [ -e "/var/lib/clamav//spam_marketing.ndb" ] ; then +echo "Was /var/lib/clamav/spam_marketing.ndb removed ?" +if [ ! -e "/var/lib/clamav/spam_marketing.ndb" ] ; then echo .. OK else echo .. ERROR exit 1 fi echo "Was /var/lib/clamav-unofficial-sigs/dbs-si/spam_marketing.ndb removed ?" -if [ -e "/var/lib/clamav-unofficial-sigs/dbs-si/spam_marketing.ndb" ] ; then +if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-si/spam_marketing.ndb" ] ; then echo .. OK else echo .. ERROR diff --git a/README.md b/README.md index 15786027..92af9213 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] ## Change Log ### Version 7.2 ( XX December 2020 : Under Development) - Database rating downgrades are now supported, eg, changing from HIGH to LOW will remove the HIGH and MEDIUM rated databases. - - Disabled datbaases are automatically removed + - Disabled databases are automatically removed - Added Malware.expert databases (non-free) - Reworked securiteinfo premium databases (non-free) - Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) From e9431f56eed5fa2f12ffafc7ba6a3b5ff9bf2064 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 05:04:51 +0200 Subject: [PATCH 049/101] ready for 7.2 release ? fixes #179 --- .t/ci-test.sh | 8 ++++---- clamav-unofficial-sigs.sh | 23 +---------------------- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index fff64eb0..1e082424 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -141,15 +141,15 @@ echo "===== AFTER /var/lib/clamav-unofficial-sigs/dbs-si =====" ls -laFh /var/lib/clamav-unofficial-sigs/dbs-si echo "================" -echo "Was /var/lib/clamav/spam_marketing.ndb removed ?" -if [ ! -e "/var/lib/clamav/spam_marketing.ndb" ] ; then +echo "Was /var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb removed ?" +if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then echo .. OK else echo .. ERROR exit 1 fi -echo "Was /var/lib/clamav-unofficial-sigs/dbs-si/spam_marketing.ndb removed ?" -if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-si/spam_marketing.ndb" ] ; then +echo "Was /var/lib/clamav/phish.ndb removed ?" +if [ ! -e "/var/lib/clamav/phish.ndb" ] ; then echo .. OK else echo .. ERROR diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index e266a006..0162b0a7 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2386,24 +2386,7 @@ else malwarepatrol_remove_dbs=( "malwarepatrol.db" ) fi ############################################################################################ - -echo "**********DEBUG :: BEGIN *************" -echo "sanesecurity_remove_dbs:" -echo "${sanesecurity_remove_dbs[@]}" -echo "securiteinfo_remove_dbs:" -echo "${securiteinfo_remove_dbs[@]}" -echo "linuxmalwaredetect_remove_dbs:" -echo "${linuxmalwaredetect_remove_dbs[@]}" -echo "malwareexpert_remove_dbs:" -echo "${malwareexpert_remove_dbs[@]}" -echo "yararulesproject_remove_dbs:" -echo "${yararulesproject_remove_dbs[@]}" -echo "urlhaus_remove_dbs:" -echo "${urlhaus_remove_dbs[@]}" -echo "malwarepatrol_remove_dbs:" -echo "${malwarepatrol_remove_dbs[@]}" -echo "**********DEBUG :: END *************" - +# CLEANUP UNUSED DATABASES, eg when downgrading a database rating or disabling a database if [ -n "${sanesecurity_remove_dbs[0]}" ] ; then for db_file in "${sanesecurity_dbs[@]}" ; do if [ -f "${work_dir_sanesecurity}/${db_file}" ] ; then @@ -2489,12 +2472,8 @@ if [ -n "${malwarepatrol_remove_dbs[0]}" ] ; then done fi - - ############################################################################################ - - # If "ham_dir" variable is set, then create initial whitelist files (skipped if first-time script run). test_dir="$work_dir/test" if [ -n "$ham_dir" ] && [ -d "$work_dir" ] && [ ! -d "$test_dir" ] ; then From e880ec96291fb0485e4d0ff664e2657e7b7c802f Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 05:08:56 +0200 Subject: [PATCH 050/101] i wish... --- .github/FUNDING.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..4808fef3 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms + +github: extremeshok +custom: ['https://paypal.me/AdrianKriel', 'https://www.extremeshok.com'] From d980265ac32cf72a6dce9750d78559d35a2faa9f Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 10:10:39 +0200 Subject: [PATCH 051/101] correct name --- README.md | 6 +++--- clamav-unofficial-sigs.sh | 34 +++++++++++++++++----------------- config/master.conf | 24 ++++++++++++------------ config/user.conf | 2 +- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 92af9213..4452af49 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,8 @@ Usage of free Yara-Rules Project: http://yararules.com Current limitations of clamav support : http://blog.clamav.net/search/label/yara ### malware.experrt non-free database support (as of December 2020) -Usage of malware.expert : https://www.malware.expert - - 1. Sign up for an account : https://www.malware.expert +Usage of Malware Expert : https://www.Malware Expert + - 1. Sign up for an account : https://www.Malware Expert - 2. You will recieve an email containing your serial key/number - 3. Enter the serial key into the config malwareexpert_serial_key: replacing YOUR-SERIAL-KEY with your serial key from the email @@ -187,7 +187,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] ### Version 7.2 ( XX December 2020 : Under Development) - Database rating downgrades are now supported, eg, changing from HIGH to LOW will remove the HIGH and MEDIUM rated databases. - Disabled databases are automatically removed - - Added Malware.expert databases (non-free) + - Added Malware Expert databases (non-free) - Reworked securiteinfo premium databases (non-free) - Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) - Added detection of tar executable (use gtar on mac and bsd) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 0162b0a7..6ce971dc 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -1171,7 +1171,7 @@ function clamscan_integrity_test_specific_database_file() { # databasefile xshok_pretty_echo_and_log "=== Linux Malware Detect ===" ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_linuxmalwaredetect" - xshok_pretty_echo_and_log "=== malware.expert Detect ===" + xshok_pretty_echo_and_log "=== Malware Expert Detect ===" ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_malwareexpert" xshok_pretty_echo_and_log "=== Linux Malware Detect ===" @@ -3206,7 +3206,7 @@ else fi ############################################################################################################################################## -# Check for updated malware.expert database files every set number of hours as defined in the "USER CONFIGURATION" section of this script # +# Check for updated Malware Expert database files every set number of hours as defined in the "USER CONFIGURATION" section of this script # ############################################################################################################################################## if [ "$malwareexpert_enabled" == "yes" ] ; then if [ "$malwareexpert_serial_key" != "YOUR-SERIAL-KEY" ] ; then @@ -3237,7 +3237,7 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then if [ "$loop" == "1" ] ; then xshok_pretty_echo_and_log "---" fi - xshok_pretty_echo_and_log "Checking for updated malware.expert database file: ${db_file}" + xshok_pretty_echo_and_log "Checking for updated Malware Expert database file: ${db_file}" malwareexpert_db_update="0" xshok_file_download "${work_dir_malwareexpert}/${db_file}" "${malwareexpert_url}/${malwareexpert_serial_key}/${db_file}" ret="$?" @@ -3246,13 +3246,13 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then if ! cmp -s "${work_dir_malwareexpert}/${db_file}" "${clam_dbs}/${db_file}" ; then db_ext="${db_file#*.}" - xshok_pretty_echo_and_log "Testing updated malware.expert database file: ${db_file}" + xshok_pretty_echo_and_log "Testing updated Malware Expert database file: ${db_file}" if [ -z "$ham_dir" ] || [ "$db_ext" != "ndb" ] ; then if $clamscan_bin --quiet -d "${work_dir_malwareexpert}/${db_file}" "${work_dir_work_configs}/scan-test.txt" 2>&10 ; then - xshok_pretty_echo_and_log "Clamscan reports malware.expert ${db_file} database integrity tested good" + xshok_pretty_echo_and_log "Clamscan reports Malware Expert ${db_file} database integrity tested good" true else - xshok_pretty_echo_and_log "Clamscan reports malware.expert ${db_file} database integrity tested BAD" + xshok_pretty_echo_and_log "Clamscan reports Malware Expert ${db_file} database integrity tested BAD" if [ "$remove_bad_database" == "yes" ] ; then if rm -f "${work_dir_malwareexpert}/${db_file}" ; then xshok_pretty_echo_and_log "Removed invalid database: ${work_dir_malwareexpert}/${db_file}" @@ -3264,12 +3264,12 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then if [ "$selinux_fixes" == "yes" ] ; then restorecon "${clam_dbs}/${db_file}" fi - xshok_pretty_echo_and_log "Successfully updated malware.expert production database file: ${db_file}" + xshok_pretty_echo_and_log "Successfully updated Malware Expert production database file: ${db_file}" malwareexpert_updates=1 malwareexpert_db_update=1 do_clamd_reload=1 else - xshok_pretty_echo_and_log "Failed to successfully update malware.expert production database file: ${db_file} - SKIPPING" + xshok_pretty_echo_and_log "Failed to successfully update Malware Expert production database file: ${db_file} - SKIPPING" fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_malwareexpert}/${db_file}" > "${test_dir}/${db_file}" @@ -3278,10 +3278,10 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" if $clamscan_bin --quiet -d "${test_dir}/${db_file}" "${work_dir_work_configs}/scan-test.txt" 2>&10 ; then - xshok_pretty_echo_and_log "Clamscan reports malware.expert ${db_file} database integrity tested good" + xshok_pretty_echo_and_log "Clamscan reports Malware Expert ${db_file} database integrity tested good" true else - xshok_pretty_echo_and_log "Clamscan reports malware.expert ${db_file} database integrity tested BAD" + xshok_pretty_echo_and_log "Clamscan reports Malware Expert ${db_file} database integrity tested BAD" rm -f "${work_dir_malwareexpert}/${db_file}" if [ "$remove_bad_database" == "yes" ] ; then if rm -f "${work_dir_malwareexpert}/${db_file}" ; then @@ -3294,27 +3294,27 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then if [ "$selinux_fixes" == "yes" ] ; then restorecon "${clam_dbs}/${db_file}" fi - xshok_pretty_echo_and_log "Successfully updated malware.expert production database file: ${db_file}" + xshok_pretty_echo_and_log "Successfully updated Malware Expert production database file: ${db_file}" malwareexpert_updates=1 malwareexpert_db_update=1 do_clamd_reload=1 else - xshok_pretty_echo_and_log "Failed to successfully update malware.expert production database file: ${db_file} - SKIPPING" + xshok_pretty_echo_and_log "Failed to successfully update Malware Expert production database file: ${db_file} - SKIPPING" fi fi fi else - xshok_pretty_echo_and_log "Failed connection to ${malwareexpert_url} - SKIPPED malware.expert ${db_file} update" + xshok_pretty_echo_and_log "Failed connection to ${malwareexpert_url} - SKIPPED Malware Expert ${db_file} update" fi if [ "$malwareexpert_db_update" != "1" ] ; then - xshok_pretty_echo_and_log "No updated malware.expert ${db_file} database file found" "-" + xshok_pretty_echo_and_log "No updated Malware Expert ${db_file} database file found" "-" fi done if [ "$malwareexpert_updates" != "1" ] ; then - xshok_pretty_echo_and_log "No malware.expert database file updates found" "-" + xshok_pretty_echo_and_log "No Malware Expert database file updates found" "-" fi else - xshok_pretty_echo_and_log "malware.expert Database File Updates" "=" + xshok_pretty_echo_and_log "Malware Expert Database File Updates" "=" if [ "$malwareexpert_premium" == "yes" ] ; then xshok_draw_time_remaining "$((update_interval - time_interval))" "$malwareexpert_premium_update_hours" "malwareexpert" else @@ -3327,7 +3327,7 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then else if [ -n "$malwareexpert_dbs" ] ; then if [ "$remove_disabled_databases" == "yes" ] ; then - xshok_pretty_echo_and_log "Removing disabled malware.expert Database files" + xshok_pretty_echo_and_log "Removing disabled Malware Expert Database files" for db_file in "${malwareexpert_dbs[@]}" ; do if echo "$db_file" | $grep_bin -q "|" ; then db_file="${db_file%|*}" diff --git a/config/master.conf b/config/master.conf index ffdd50ee..90c23d8d 100644 --- a/config/master.conf +++ b/config/master.conf @@ -87,8 +87,8 @@ malwarepatrol_db="malwarepatrol.db" # ========================= -# malware.expert : https://www.malware.expert -# malware.expert 2020 (non-free) clamav signatures +# Malware Expert : https://www.Malware Expert +# Malware Expert 2020 (non-free) clamav signatures malwareexpert_serial_key="YOUR-SERIAL-KEY" # ========================= @@ -131,7 +131,7 @@ yararulesproject_update_hours="24" # Default is 24 hours (1 downloads daily). # Set to no to disable an entire database, if the database is empty it will also be disabled. additional_enabled="yes" # Additional Databases linuxmalwaredetect_enabled="yes" # Linux Malware Detect -malwareexpert_enabled="yes" # malware.expert +malwareexpert_enabled="yes" # Malware Expert malwarepatrol_enabled="yes" # Malware Patrol sanesecurity_enabled="yes" # Sanesecurity securiteinfo_enabled="yes" # SecuriteInfo @@ -330,19 +330,19 @@ rfxn.yara|LOW # Yara Malware detection signatures ) #END LINUXMALWAREDETECT DATABASES # ======================== -# malware.expert Database(s) +# Malware Expert Database(s) # ======================== # Add or remove database file names between quote marks as needed. To -# disable any malware.expert database downloads, remove the appropriate +# disable any Malware Expert database downloads, remove the appropriate # lines below. declare -a malwareexpert_dbs=( # LOW -malware.expert.hdb|MEDIUM # statics MD5 pattern for files +Malware Expert.hdb|MEDIUM # statics MD5 pattern for files # MEDIUM -malware.expert.fp|MEDIUM # found to be false positive malware -malware.expert.ldb|MEDIUM # which use multi-words search for malware in files -malware.expert.ndb|MEDIUM # Generic Hex pattern PHP malware, which can cause false positive alarms -) #END MALWARE.EXPERT DATABASES +Malware Expert.fp|MEDIUM # found to be false positive malware +Malware Expert.ldb|MEDIUM # which use multi-words search for malware in files +Malware Expert.ndb|MEDIUM # Generic Hex pattern PHP malware, which can cause false positive alarms +) #END Malware Expert DATABASES # ======================== # urlhaus Database(s) @@ -609,7 +609,7 @@ add_dir="dbs-add" # User defined databases sub-directory gpg_dir="gpg-key" # Sanesecurity GPG Key sub-directory linuxmalwaredetect_dir="dbs-lmd" # Linux Malware Detect sub-directory malwarepatrol_dir="dbs-mbl" # MalwarePatrol sub-directory -malwareexpert_dir="dbs-me" # malware.expertt sub-directory +malwareexpert_dir="dbs-me" # Malware Expertt sub-directory pid_dir="pid" # User defined pid sub-directory sanesecurity_dir="dbs-ss" # Sanesecurity sub-directory securiteinfo_dir="dbs-si" # SecuriteInfo sub-directory @@ -698,7 +698,7 @@ user_configuration_complete="no" # Database provider URLs linuxmalwaredetect_sigpack_url="https://cdn.rfxn.com/downloads/maldet-sigpack.tgz" linuxmalwaredetect_version_url="https://cdn.rfxn.com/downloads/maldet.sigs.ver" -malwareexpert_url="https://signatures.malware.expert" +malwareexpert_url="https://signatures.Malware Expert" malwarepatrol_url="https://lists.malwarepatrol.net/cgi/getfile" sanesecurity_gpg_url="https://www.sanesecurity.com/publickey.gpg" sanesecurity_url="rsync.sanesecurity.net" diff --git a/config/user.conf b/config/user.conf index 4d2cc32b..f685b39d 100644 --- a/config/user.conf +++ b/config/user.conf @@ -21,7 +21,7 @@ # Please note, it is your responsibility to manage the contents of this file. # Values provided here are just examples, feel free to use any values from the main config file. -# malware.expert 2020 (non-free) clamav signatures +# Malware Expert 2020 (non-free) clamav signatures # set to no to enable the commercial subscription databases #malwareexpert_serial_key="YOUR-SERIAL-KEY" From b717c3650e29a5a034864545389d8a914de2c1a8 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 14:24:55 +0200 Subject: [PATCH 052/101] add ci for malware.expert fix malware.expert database names fix ci user configs being used --- .t/ci-test.sh | 59 ++++++++++++++++++++++++++++++++++++++++------ config/master.conf | 8 +++---- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index 1e082424..d2a44cc2 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -120,14 +120,12 @@ else echo .. ERROR exit 1 fi -echo "===== BEFORE /var/lib/clamav/ =====" +echo "===== HIGH /var/lib/clamav/ =====" ls -laFh /var/lib/clamav/ -echo "===== BEFORE /var/lib/clamav-unofficial-sigs/dbs-si =====" -ls -laFh /var/lib/clamav-unofficial-sigs/dbs-si echo "================" echo "running script verbose with LOW ratings" -cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user_low.conf +cp -f .t/tests/user_low.conf /etc/clamav-unofficial-sigs/user.conf bash /usr/sbin/clamav-unofficial-sigs --verbose if [ "$?" -eq "0" ] ; then echo .. OK @@ -135,10 +133,8 @@ else echo .. ERROR exit 1 fi -echo "===== AFTER /var/lib/clamav/ =====" +echo "===== LOW /var/lib/clamav/ =====" ls -laFh /var/lib/clamav/ -echo "===== AFTER /var/lib/clamav-unofficial-sigs/dbs-si =====" -ls -laFh /var/lib/clamav-unofficial-sigs/dbs-si echo "================" echo "Was /var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb removed ?" @@ -155,3 +151,52 @@ else echo .. ERROR exit 1 fi + +echo "running script verbose with malwareexpert databases" +cp -f .t/tests/user_malwareexpert.conf /etc/clamav-unofficial-sigs/user.conf +bash /usr/sbin/clamav-unofficial-sigs --verbose +if [ "$?" -eq "0" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "===== MALWAREEXPERT /var/lib/clamav/ =====" +ls -laFh /var/lib/clamav/ +echo "================" + +echo "Was /var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb removed ?" +if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "Was /var/lib/clamav/malware.expert.hdb added ?" +if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "Was /var/lib/clamav/malware.expert.fp added ?" +if [ -e "/var/lib/clamav/malware.expert.fp" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "Was /var/lib/clamav/malware.expert.ldb added ?" +if [ -e "/var/lib/clamav/malware.expert.ldb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "Was /var/lib/clamav/malware.expert.ndb added ?" +if [ -e "/var/lib/clamav/malware.expert.ndb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi diff --git a/config/master.conf b/config/master.conf index 90c23d8d..a763a9ed 100644 --- a/config/master.conf +++ b/config/master.conf @@ -337,11 +337,11 @@ rfxn.yara|LOW # Yara Malware detection signatures # lines below. declare -a malwareexpert_dbs=( # LOW -Malware Expert.hdb|MEDIUM # statics MD5 pattern for files +malware.expert.hdb|MEDIUM # statics MD5 pattern for files # MEDIUM -Malware Expert.fp|MEDIUM # found to be false positive malware -Malware Expert.ldb|MEDIUM # which use multi-words search for malware in files -Malware Expert.ndb|MEDIUM # Generic Hex pattern PHP malware, which can cause false positive alarms +malware.expert.fp|MEDIUM # found to be false positive malware +malware.expert.ldb|MEDIUM # which use multi-words search for malware in files +malware.expert.ndb|MEDIUM # Generic Hex pattern PHP malware, which can cause false positive alarms ) #END Malware Expert DATABASES # ======================== From acbdcb55f5d4bc4dfb0698692e3869857c17b4dd Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 14:36:42 +0200 Subject: [PATCH 053/101] more ci fixes --- .t/tests/user_low.conf | 24 +++++++------------ .t/tests/user_malwareexpert.conf | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 15 deletions(-) create mode 100644 .t/tests/user_malwareexpert.conf diff --git a/.t/tests/user_low.conf b/.t/tests/user_low.conf index 92281234..c617b4ff 100644 --- a/.t/tests/user_low.conf +++ b/.t/tests/user_low.conf @@ -5,18 +5,10 @@ # License: BSD (Berkeley Software Distribution) ################## -malwarepatrol_enabled="yes" -malwarepatrol_receipt_code=$ci_malwarepatrol_receipt_code -malwarepatrol_product_code=$ci_malwarepatrol_receipt_code -malwarepatrol_list=$ci_malwarepatrol_receipt_code -malwarepatrol_free=$ci_malwarepatrol_free - -securiteinfo_enabled="yes" -securiteinfo_authorisation_signature=$ci_securiteinfo_authorisation_signature - -sanesecurity_enabled="yes" - -linuxmalwaredetect_enabled="yes" +malwarepatrol_enabled="no" +securiteinfo_enabled="no" +sanesecurity_enabled="no" +linuxmalwaredetect_enabled="no" # THIS NEEDS TO BE TESTED yararules_enabled="no" @@ -29,10 +21,12 @@ default_dbs_rating="LOW" # Per Database # These ratings will override the global rating for the specific database # valid rating: LOW, MEDIUM, HIGH, DISABLE -sanesecurity_dbs_rating="DISABLE" -securiteinfo_dbs_rating="LOW" +sanesecurity_dbs_rating="LOW" linuxmalwaredetect_dbs_rating="DISABLE" -yararulesproject_dbs_rating="DISABLE" +malwareexpert_dbs_rating="DISABLE" +securiteinfo_dbs_rating="DISABLE" +urlhaus_dbs_rating="DISABLE" +yararulesproject_dbs_rating="DISABLE enable_gpg="no" diff --git a/.t/tests/user_malwareexpert.conf b/.t/tests/user_malwareexpert.conf new file mode 100644 index 00000000..920dbfa6 --- /dev/null +++ b/.t/tests/user_malwareexpert.conf @@ -0,0 +1,41 @@ +################### +# This is property of eXtremeSHOK.com +# You are free to use, modify and distribute, however you may not remove this notice. +# Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com +# License: BSD (Berkeley Software Distribution) +################## + +malwarepatrol_enabled="no" +securiteinfo_enabled="no" +sanesecurity_enabled="no" +linuxmalwaredetect_enabled="no" +securiteinfo_authorisation_signature=$ci_securiteinfo_authorisation_signature + +# Malware Expert 2020 (non-free) clamav signatures +# set to no to enable the commercial subscription databases +malwareexpert_serial_key=$ci_malwareexpert_serial_key + +# THIS NEEDS TO BE TESTED +yararules_enabled="no" +enable_yararules="no" + +# Default dbs rating +# valid rating: LOW, MEDIUM, HIGH +default_dbs_rating="DISABLE" + +# Per Database +# These ratings will override the global rating for the specific database +# valid rating: LOW, MEDIUM, HIGH, DISABLE +linuxmalwaredetect_dbs_rating="DISABLE" +malwareexpert_dbs_rating="HIGH" +sanesecurity_dbs_rating="DISABLE" +securiteinfo_dbs_rating="DISABLE" +urlhaus_dbs_rating="DISABLE" +yararulesproject_dbs_rating="DISABLE + +enable_gpg="no" + +user_configuration_complete="yes" + +# Enable all debug options +debug="yes" From 7f4a679201b39ed738fd99da876cabc8ebd892d0 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 2 Dec 2020 14:49:51 +0200 Subject: [PATCH 054/101] missing " --- .t/tests/user_low.conf | 2 +- .t/tests/user_malwareexpert.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.t/tests/user_low.conf b/.t/tests/user_low.conf index c617b4ff..892d71a6 100644 --- a/.t/tests/user_low.conf +++ b/.t/tests/user_low.conf @@ -26,7 +26,7 @@ linuxmalwaredetect_dbs_rating="DISABLE" malwareexpert_dbs_rating="DISABLE" securiteinfo_dbs_rating="DISABLE" urlhaus_dbs_rating="DISABLE" -yararulesproject_dbs_rating="DISABLE +yararulesproject_dbs_rating="DISABLE" enable_gpg="no" diff --git a/.t/tests/user_malwareexpert.conf b/.t/tests/user_malwareexpert.conf index 920dbfa6..52719fbf 100644 --- a/.t/tests/user_malwareexpert.conf +++ b/.t/tests/user_malwareexpert.conf @@ -31,7 +31,7 @@ malwareexpert_dbs_rating="HIGH" sanesecurity_dbs_rating="DISABLE" securiteinfo_dbs_rating="DISABLE" urlhaus_dbs_rating="DISABLE" -yararulesproject_dbs_rating="DISABLE +yararulesproject_dbs_rating="DISABLE" enable_gpg="no" From 2a91009960b5e2fd502f13d8ada168efe07bec1d Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Thu, 3 Dec 2020 04:38:32 +0200 Subject: [PATCH 055/101] more debugging of ci --- .t/ci-test.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index d2a44cc2..9cfab671 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -48,7 +48,7 @@ if [ -e "/var/lib/clamav/sanesecurity.ftm" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "check cron file generation" @@ -62,7 +62,7 @@ if [ "$?" -eq "0" ] ; then fi else echo .. ERROR - exit 1 + #exit 1 fi echo "check logrotate file generation" @@ -72,11 +72,11 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi else echo .. ERROR - exit 1 + #exit 1 fi echo "check man file generation" @@ -86,11 +86,11 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi else echo .. ERROR - exit 1 + #exit 1 fi echo "check database integrity test" @@ -99,7 +99,7 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "check gpg verify test" @@ -108,7 +108,7 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "check clamav-daemon service will start" @@ -118,7 +118,7 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "===== HIGH /var/lib/clamav/ =====" ls -laFh /var/lib/clamav/ @@ -131,7 +131,7 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "===== LOW /var/lib/clamav/ =====" ls -laFh /var/lib/clamav/ @@ -142,14 +142,14 @@ if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "Was /var/lib/clamav/phish.ndb removed ?" if [ ! -e "/var/lib/clamav/phish.ndb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "running script verbose with malwareexpert databases" @@ -159,7 +159,7 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "===== MALWAREEXPERT /var/lib/clamav/ =====" ls -laFh /var/lib/clamav/ @@ -170,33 +170,33 @@ if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "Was /var/lib/clamav/malware.expert.hdb added ?" if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "Was /var/lib/clamav/malware.expert.fp added ?" if [ -e "/var/lib/clamav/malware.expert.fp" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "Was /var/lib/clamav/malware.expert.ldb added ?" if [ -e "/var/lib/clamav/malware.expert.ldb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "Was /var/lib/clamav/malware.expert.ndb added ?" if [ -e "/var/lib/clamav/malware.expert.ndb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi From 0b440f681e311d735e668e75f5b404b24084e224 Mon Sep 17 00:00:00 2001 From: Mark Sapiro Date: Thu, 3 Dec 2020 15:54:54 -0800 Subject: [PATCH 056/101] Update clamav-unofficial-sigs.sh --- clamav-unofficial-sigs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 6ce971dc..9cce5f29 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2388,7 +2388,7 @@ fi ############################################################################################ # CLEANUP UNUSED DATABASES, eg when downgrading a database rating or disabling a database if [ -n "${sanesecurity_remove_dbs[0]}" ] ; then - for db_file in "${sanesecurity_dbs[@]}" ; do + for db_file in "${sanesecurity_remove_dbs[@]}" ; do if [ -f "${work_dir_sanesecurity}/${db_file}" ] ; then echo "Removing unused file: ${work_dir_sanesecurity}/${db_file}" rm -f "${work_dir_sanesecurity}/${db_file}" From aef622142d88b4a803e2c7c70b626751f9e7e957 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Fri, 4 Dec 2020 16:15:58 +0200 Subject: [PATCH 057/101] anable strict ci-testing --- .t/ci-test.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index 9cfab671..6edf0b55 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -48,7 +48,7 @@ if [ -e "/var/lib/clamav/sanesecurity.ftm" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "check cron file generation" @@ -62,7 +62,7 @@ if [ "$?" -eq "0" ] ; then fi else echo .. ERROR - #exit 1 + exit 1 fi echo "check logrotate file generation" @@ -72,11 +72,11 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi else echo .. ERROR - #exit 1 + exit 1 fi echo "check man file generation" @@ -86,11 +86,11 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi else echo .. ERROR - #exit 1 + exit 1 fi echo "check database integrity test" @@ -99,7 +99,7 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "check gpg verify test" @@ -108,7 +108,7 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "check clamav-daemon service will start" @@ -118,7 +118,7 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "===== HIGH /var/lib/clamav/ =====" ls -laFh /var/lib/clamav/ @@ -131,7 +131,7 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "===== LOW /var/lib/clamav/ =====" ls -laFh /var/lib/clamav/ @@ -142,14 +142,14 @@ if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "Was /var/lib/clamav/phish.ndb removed ?" if [ ! -e "/var/lib/clamav/phish.ndb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "running script verbose with malwareexpert databases" @@ -159,7 +159,7 @@ if [ "$?" -eq "0" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "===== MALWAREEXPERT /var/lib/clamav/ =====" ls -laFh /var/lib/clamav/ @@ -170,33 +170,33 @@ if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "Was /var/lib/clamav/malware.expert.hdb added ?" if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "Was /var/lib/clamav/malware.expert.fp added ?" if [ -e "/var/lib/clamav/malware.expert.fp" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "Was /var/lib/clamav/malware.expert.ldb added ?" if [ -e "/var/lib/clamav/malware.expert.ldb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "Was /var/lib/clamav/malware.expert.ndb added ?" if [ -e "/var/lib/clamav/malware.expert.ndb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi From 0b31f985c50790fd5371d0eca32cee461e206879 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Fri, 4 Dec 2020 16:29:46 +0200 Subject: [PATCH 058/101] alas more testing required --- .t/ci-test.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index 6edf0b55..efb3125b 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -142,14 +142,14 @@ if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "Was /var/lib/clamav/phish.ndb removed ?" if [ ! -e "/var/lib/clamav/phish.ndb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "running script verbose with malwareexpert databases" @@ -170,33 +170,33 @@ if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "Was /var/lib/clamav/malware.expert.hdb added ?" if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "Was /var/lib/clamav/malware.expert.fp added ?" if [ -e "/var/lib/clamav/malware.expert.fp" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "Was /var/lib/clamav/malware.expert.ldb added ?" if [ -e "/var/lib/clamav/malware.expert.ldb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi echo "Was /var/lib/clamav/malware.expert.ndb added ?" if [ -e "/var/lib/clamav/malware.expert.ndb" ] ; then echo .. OK else echo .. ERROR - exit 1 + #exit 1 fi From 530fffd314137a8e68e5ab3b9099cf7c39cb1556 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Fri, 4 Dec 2020 22:01:34 +0200 Subject: [PATCH 059/101] test patch #336 --- clamav-unofficial-sigs.sh | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 9cce5f29..13ab0952 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -464,23 +464,22 @@ function xshok_remove_database() { # rating database_array if [[ "$db_name" == *".yar"* ]] ; then # If it's the value you want to delete new_dbs+=( "$db_name" ) fi - else - if [ "$db_name_rating" == "DISABLED" ] ; then + fi + if [ "$db_name_rating" == "DISABLED" ] ; then + new_dbs+=( "$db_name" ) + elif [ "$current_rating" == "HIGH" ] ; then + if [ "$db_name_rating" == "LOWONLY" ] || [ "$db_name_rating" == "LOWMEDIUMONLY" ] ||[ "$db_name_rating" == "MEDIUMONLY" ] ; then new_dbs+=( "$db_name" ) - elif [ "$current_rating" == "HIGH" ] ; then - if [ "$db_name_rating" == "LOWONLY" ] || [ "$db_name_rating" == "LOWMEDIUMONLY" ] ||[ "$db_name_rating" == "MEDIUMONLY" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "MEDIUM" ] ; then - if [ "$db_name_rating" == "HIGH" ] ; then - new_dbs+=( "$db_name" ) - fi - elif [ "$current_rating" == "LOW" ] ; then - if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "HIGH" ]; then - new_dbs+=( "$db_name" ) - fi fi - fi + elif [ "$current_rating" == "MEDIUM" ] ; then + if [ "$db_name_rating" == "HIGH" ] || [ "$db_name_rating" == "LOWONLY" ] ; then + new_dbs+=( "$db_name" ) + fi + elif [ "$current_rating" == "LOW" ] ; then + if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "HIGH" ]; then + new_dbs+=( "$db_name" ) + fi + fi done fi fi From 27d2c19c3c513409b14a1b17c445da9b30e5f94c Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 03:44:36 +0200 Subject: [PATCH 060/101] fixes for yararule databases not being removed when disabled or downgrading eg. medium to low --- clamav-unofficial-sigs.sh | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 13ab0952..d7b26420 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -449,23 +449,19 @@ function xshok_database() { # rating database_array # Since the datbases are basically a multi-dimentional associative arrays in bash # ratings: LOW | MEDIUM | HIGH | REQUIRED | LOWONLY | MEDIUMONLY | LOWMEDIUMONLY | DISABLED function xshok_remove_database() { # rating database_array - # Assign - current_rating="${1}" - declare -a current_dbs=( "${@:2}" ) - # Zero - declare -a new_dbs=( ) - if [ -n "${current_dbs[0]}" ] ; then + # Assign + current_rating="${1}" + declare -a current_dbs=( "${@:2}" ) + # Zero + declare -a new_dbs=( ) if [ ${#current_dbs} -ge 1 ] ; then for db_name in "${current_dbs[@]}" ; do db_name_rating="${db_name#*|}" db_name="${db_name%|*}" # Checks - if [ "$enable_yararules" == "no" ] ; then # YARA rules are disabled - if [[ "$db_name" == *".yar"* ]] ; then # If it's the value you want to delete - new_dbs+=( "$db_name" ) - fi - fi - if [ "$db_name_rating" == "DISABLED" ] ; then + if [ "$enable_yararules" == "no" ] && [[ "$db_name" == *".yar"* ]] ; then # YARA rules are disabled AND it's the value you want to delete + new_dbs+=( "$db_name" ) + elif [ "$db_name_rating" == "DISABLED" ] ; then new_dbs+=( "$db_name" ) elif [ "$current_rating" == "HIGH" ] ; then if [ "$db_name_rating" == "LOWONLY" ] || [ "$db_name_rating" == "LOWMEDIUMONLY" ] ||[ "$db_name_rating" == "MEDIUMONLY" ] ; then @@ -482,8 +478,7 @@ function xshok_remove_database() { # rating database_array fi done fi - fi - echo "${new_dbs[@]}" | xargs # Remove extra whitespace + echo "${new_dbs[@]}" | xargs # Remove extra whitespace } @@ -2335,6 +2330,9 @@ yararulesproject_remove_dbs=( ) if [ -n "$temp_remove_db" ] ; then read -r -a yararulesproject_remove_dbs <<< "$temp_remove_db" fi + + + ############################################################################################ if [ "$urlhaus_enabled" == "yes" ] ; then if [ -n "$urlhaus_dbs" ] ; then @@ -2436,7 +2434,13 @@ if [ -n "${malwareexpert_remove_dbs[0]}" ] ; then fi if [ -n "${yararulesproject_remove_dbs[0]}" ] ; then for db_file in "${yararulesproject_remove_dbs[@]}" ; do - if [ -f "${work_dir_yararulesproject}/${db_file}" ] ; then + if echo "$db_file" | $grep_bin -q "/" ; then + yr_dir="/$(echo "$db_file" | cut -d "/" -f 1)" + db_file="$(echo "$db_file" | cut -d "/" -f 2)" + else + yr_dir="" + fi + if [ -f "${work_dir_yararulesproject}/${yr_dir}${db_file}" ] ; then echo "Removing unused file: ${work_dir_yararulesproject}/${db_file}" rm -f "${work_dir_yararulesproject}/${db_file}" fi From 57e95cf8ecf85f6f41625faa1a362a609de37584 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 04:01:13 +0200 Subject: [PATCH 061/101] malware.expert url fixes --- .t/tests/user_malwareexpert.conf | 3 +-- config/master.conf | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.t/tests/user_malwareexpert.conf b/.t/tests/user_malwareexpert.conf index 52719fbf..8df7b728 100644 --- a/.t/tests/user_malwareexpert.conf +++ b/.t/tests/user_malwareexpert.conf @@ -9,7 +9,6 @@ malwarepatrol_enabled="no" securiteinfo_enabled="no" sanesecurity_enabled="no" linuxmalwaredetect_enabled="no" -securiteinfo_authorisation_signature=$ci_securiteinfo_authorisation_signature # Malware Expert 2020 (non-free) clamav signatures # set to no to enable the commercial subscription databases @@ -26,8 +25,8 @@ default_dbs_rating="DISABLE" # Per Database # These ratings will override the global rating for the specific database # valid rating: LOW, MEDIUM, HIGH, DISABLE -linuxmalwaredetect_dbs_rating="DISABLE" malwareexpert_dbs_rating="HIGH" +linuxmalwaredetect_dbs_rating="DISABLE" sanesecurity_dbs_rating="DISABLE" securiteinfo_dbs_rating="DISABLE" urlhaus_dbs_rating="DISABLE" diff --git a/config/master.conf b/config/master.conf index a763a9ed..ee8a6251 100644 --- a/config/master.conf +++ b/config/master.conf @@ -698,7 +698,7 @@ user_configuration_complete="no" # Database provider URLs linuxmalwaredetect_sigpack_url="https://cdn.rfxn.com/downloads/maldet-sigpack.tgz" linuxmalwaredetect_version_url="https://cdn.rfxn.com/downloads/maldet.sigs.ver" -malwareexpert_url="https://signatures.Malware Expert" +malwareexpert_url="https://signatures.malware.expert" malwarepatrol_url="https://lists.malwarepatrol.net/cgi/getfile" sanesecurity_gpg_url="https://www.sanesecurity.com/publickey.gpg" sanesecurity_url="rsync.sanesecurity.net" From 50307dab47ecc0b98c6a966d67b4f21c5ad128b2 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 04:44:03 +0200 Subject: [PATCH 062/101] more logic for removing databases --- clamav-unofficial-sigs.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index d7b26420..cbec9451 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -458,22 +458,30 @@ function xshok_remove_database() { # rating database_array for db_name in "${current_dbs[@]}" ; do db_name_rating="${db_name#*|}" db_name="${db_name%|*}" + removed="no" # Checks - if [ "$enable_yararules" == "no" ] && [[ "$db_name" == *".yar"* ]] ; then # YARA rules are disabled AND it's the value you want to delete - new_dbs+=( "$db_name" ) - elif [ "$db_name_rating" == "DISABLED" ] ; then + if [ "$db_name_rating" == "DISABLED" ] ; then new_dbs+=( "$db_name" ) + removed="yes" elif [ "$current_rating" == "HIGH" ] ; then if [ "$db_name_rating" == "LOWONLY" ] || [ "$db_name_rating" == "LOWMEDIUMONLY" ] ||[ "$db_name_rating" == "MEDIUMONLY" ] ; then new_dbs+=( "$db_name" ) + removed="yes" fi elif [ "$current_rating" == "MEDIUM" ] ; then if [ "$db_name_rating" == "HIGH" ] || [ "$db_name_rating" == "LOWONLY" ] ; then new_dbs+=( "$db_name" ) + removed="yes" fi elif [ "$current_rating" == "LOW" ] ; then if [ "$db_name_rating" == "MEDIUMONLY" ] || [ "$db_name_rating" == "MEDIUM" ] || [ "$db_name_rating" == "HIGH" ]; then new_dbs+=( "$db_name" ) + removed="yes" + fi + fi + if [ "$removed" == "no" ] ; then # not already removed, process futher + if [ "$enable_yararules" == "no" ] && [[ "$db_name" == *".yar"* ]] ; then # YARA rules are disabled AND it's the value you want to delete + new_dbs+=( "$db_name" ) fi fi done @@ -2218,10 +2226,12 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then else temp_remove_db="$(xshok_remove_database "DISABLED" "${sanesecurity_dbs[@]}")" fi + sanesecurity_remove_dbs=( ) if [ -n "$temp_remove_db" ] ; then read -r -a sanesecurity_remove_dbs <<< "$temp_remove_db" fi + ############################################################################################ if [ "$securiteinfo_enabled" == "yes" ] ; then if [ -n "$securiteinfo_dbs" ] ; then @@ -3212,7 +3222,7 @@ fi # Check for updated Malware Expert database files every set number of hours as defined in the "USER CONFIGURATION" section of this script # ############################################################################################################################################## if [ "$malwareexpert_enabled" == "yes" ] ; then - if [ "$malwareexpert_serial_key" != "YOUR-SERIAL-KEY" ] ; then + if [ "$malwareexpert_serial_key" != "YOUR-SERIAL-KEY" ] && [ -n "$malwareexpert_serial_key" ]; then if [ -n "${malwareexpert_dbs[0]}" ] ; then if [ ${#malwareexpert_dbs} -lt 1 ] ; then xshok_pretty_echo_and_log "Failed malwareexpert_dbs config is invalid or not defined - SKIPPING" From f81c944d50e3fb3231d87cc1a7792322de4bf78e Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 05:04:53 +0200 Subject: [PATCH 063/101] fixed logic with removing disabled --- clamav-unofficial-sigs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index cbec9451..38712b4b 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -454,13 +454,14 @@ function xshok_remove_database() { # rating database_array declare -a current_dbs=( "${@:2}" ) # Zero declare -a new_dbs=( ) + if [ ${#current_dbs} -ge 1 ] ; then for db_name in "${current_dbs[@]}" ; do db_name_rating="${db_name#*|}" db_name="${db_name%|*}" removed="no" # Checks - if [ "$db_name_rating" == "DISABLED" ] ; then + if [ "$current_rating" == "DISABLED" ] ; then new_dbs+=( "$db_name" ) removed="yes" elif [ "$current_rating" == "HIGH" ] ; then From f1e1b9e12d0d162f54dd256d9ca1e64c16abae25 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 05:12:22 +0200 Subject: [PATCH 064/101] enable ci checks --- .t/ci-test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index efb3125b..c6c7ee4c 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -142,14 +142,14 @@ if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "Was /var/lib/clamav/phish.ndb removed ?" if [ ! -e "/var/lib/clamav/phish.ndb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "running script verbose with malwareexpert databases" @@ -170,7 +170,7 @@ if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "Was /var/lib/clamav/malware.expert.hdb added ?" if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then From e48c414a3ec72fe625a5e8de9be3361c9386c80b Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 05:33:52 +0200 Subject: [PATCH 065/101] more updates --- .t/tests/user_malwareexpert.conf | 9 --------- .travis.yml | 1 + 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.t/tests/user_malwareexpert.conf b/.t/tests/user_malwareexpert.conf index 8df7b728..98f3ccee 100644 --- a/.t/tests/user_malwareexpert.conf +++ b/.t/tests/user_malwareexpert.conf @@ -5,19 +5,10 @@ # License: BSD (Berkeley Software Distribution) ################## -malwarepatrol_enabled="no" -securiteinfo_enabled="no" -sanesecurity_enabled="no" -linuxmalwaredetect_enabled="no" - # Malware Expert 2020 (non-free) clamav signatures # set to no to enable the commercial subscription databases malwareexpert_serial_key=$ci_malwareexpert_serial_key -# THIS NEEDS TO BE TESTED -yararules_enabled="no" -enable_yararules="no" - # Default dbs rating # valid rating: LOW, MEDIUM, HIGH default_dbs_rating="DISABLE" diff --git a/.travis.yml b/.travis.yml index 771be681..f834ed48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ language: shell os: linux # Required travis ci environment variables +#ci_malwareexpert_serial_key=[secure] #ci_securiteinfo_authorisation_signature=[secure] #ci_malwarepatrol_receipt_code=[secure] #ci_malwarepatrol_product_code=[secure] From 4f2125529d5fb757181eabf33c4377cd56120a52 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 05:42:28 +0200 Subject: [PATCH 066/101] test with --force --- .t/ci-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index c6c7ee4c..2f5353c3 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -152,9 +152,9 @@ else exit 1 fi -echo "running script verbose with malwareexpert databases" +echo "running script verbose with malware expert databases" cp -f .t/tests/user_malwareexpert.conf /etc/clamav-unofficial-sigs/user.conf -bash /usr/sbin/clamav-unofficial-sigs --verbose +bash /usr/sbin/clamav-unofficial-sigs --verbose --force if [ "$?" -eq "0" ] ; then echo .. OK else From e22676352b89395a6b6ec8b9cbd2460f46201cd2 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 05:48:37 +0200 Subject: [PATCH 067/101] malware.expert fixed ? --- .t/ci-test.sh | 10 +++++----- clamav-unofficial-sigs.sh | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index 2f5353c3..0bca2379 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -154,7 +154,7 @@ fi echo "running script verbose with malware expert databases" cp -f .t/tests/user_malwareexpert.conf /etc/clamav-unofficial-sigs/user.conf -bash /usr/sbin/clamav-unofficial-sigs --verbose --force +bash /usr/sbin/clamav-unofficial-sigs --verbose if [ "$?" -eq "0" ] ; then echo .. OK else @@ -177,26 +177,26 @@ if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "Was /var/lib/clamav/malware.expert.fp added ?" if [ -e "/var/lib/clamav/malware.expert.fp" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "Was /var/lib/clamav/malware.expert.ldb added ?" if [ -e "/var/lib/clamav/malware.expert.ldb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi echo "Was /var/lib/clamav/malware.expert.ndb added ?" if [ -e "/var/lib/clamav/malware.expert.ndb" ] ; then echo .. OK else echo .. ERROR - #exit 1 + exit 1 fi diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 38712b4b..28dd30c6 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -3229,8 +3229,8 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then xshok_pretty_echo_and_log "Failed malwareexpert_dbs config is invalid or not defined - SKIPPING" else rm -f "${work_dir_malwareexpert}/*.gz" - if [ -r "${work_dir_work_configs}/last-si-update.txt" ] ; then - last_malwareexpert_update="$(cat "${work_dir_work_configs}/last-si-update.txt")" + if [ -r "${work_dir_work_configs}/last-me-update.txt" ] ; then + last_malwareexpert_update="$(cat "${work_dir_work_configs}/last-me-update.txt")" else last_malwareexpert_update="0" fi @@ -3243,7 +3243,7 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then fi time_interval="$((current_time - last_malwareexpert_update))" if [ "$time_interval" -ge "$((update_interval - 600))" ] ; then - echo "$current_time" > "${work_dir_work_configs}/last-si-update.txt" + echo "$current_time" > "${work_dir_work_configs}/last-me-update.txt" xshok_pretty_echo_and_log "malwareexpert Database File Updates" "=" xshok_pretty_echo_and_log "Checking for malwareexpert updates..." malwareexpert_updates="0" From f7e173c1096d783084b67f91f98ef97102d13de9 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 06:24:26 +0200 Subject: [PATCH 068/101] Disable databases by setting the rating to "DISABLED" eg. securiteinfo_dbs_rating="DISABLED" will disable all securiteinfo databases --- README.md | 1 + clamav-unofficial-sigs.sh | 40 +++++++++++++++++++++++++++++++++++++++ config/master.conf | 4 ++-- config/user.conf | 2 +- 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4452af49..1dc928d1 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] ### Version 7.2 ( XX December 2020 : Under Development) - Database rating downgrades are now supported, eg, changing from HIGH to LOW will remove the HIGH and MEDIUM rated databases. - Disabled databases are automatically removed + - Disable databases by setting the rating to "DISABLED" eg. securiteinfo_dbs_rating="DISABLED" will disable all securiteinfo databases - Added Malware Expert databases (non-free) - Reworked securiteinfo premium databases (non-free) - Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 28dd30c6..fda3fa84 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2194,6 +2194,46 @@ done xshok_pretty_echo_and_log "Preparing Databases" "=" +if [ "$default_dbs_rating" == "DISABLE" ] ; then + if [ "$sanesecurity_dbs_rating" != "LOW" ] && [ "$sanesecurity_dbs_rating" != "MEDIUM" ] && [ "$sanesecurity_dbs_rating" != "HIGH" ]; then + sanesecurity_enabled="no" + fi + if [ "$linuxmalwaredetect_dbs_rating" != "LOW" ] && [ "$linuxmalwaredetect_dbs_rating" != "MEDIUM" ] && [ "$linuxmalwaredetect_dbs_rating" != "HIGH" ]; then + linuxmalwaredetect_enabled="no" + fi + if [ "$malwareexpert_dbs_rating" != "LOW" ] && [ "$malwareexpert_dbs_rating" != "MEDIUM" ] && [ "$malwareexpert_dbs_rating" != "HIGH" ]; then + malwareexpert_enabled="no" + fi + if [ "$securiteinfo_dbs_rating" != "LOW" ] && [ "$securiteinfo_dbs_rating" != "MEDIUM" ] && [ "$securiteinfo_dbs_rating" != "HIGH" ]; then + securiteinfo_enabled="no" + fi + if [ "$urlhaus_dbs_rating" != "LOW" ] && [ "$urlhaus_dbs_rating" != "MEDIUM" ] && [ "$urlhaus_dbs_rating" != "HIGH" ]; then + urlhaus_enabled="no" + fi + if [ "$yararulesproject_dbs_rating" != "LOW" ] && [ "$yararulesproject_dbs_rating" != "MEDIUM" ] && [ "$yararulesproject_dbs_rating" != "HIGH" ]; then + yararulesproject_enabled="no" + fi +else + if [ "$sanesecurity_dbs_rating" == "DISABLE" ] ; then + sanesecurity_enabled="no" + fi + if [ "$linuxmalwaredetect_dbs_rating" == "DISABLE" ] ; then + linuxmalwaredetect_enabled="no" + fi + if [ "$malwareexpert_dbs_rating" == "DISABLE" ] ; then + malwareexpert_enabled="no" + fi + if [ "$securiteinfo_dbs_rating" == "DISABLE" ] ; then + securiteinfo_enabled="no" + fi + if [ "$urlhaus_dbs_rating" == "DISABLE" ] ; then + urlhaus_enabled="no" + fi + if [ "$yararulesproject_dbs_rating" == "DISABLE" ] ; then + yararulesproject_enabled="no" + fi +fi + # Check yararule support is available if [ "$enable_yararules" == "yes" ] ; then current_clamav_version="$($clamscan_bin -V | cut -d " " -f 2 | cut -d "/" -f 1 | awk -F "." '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }')" diff --git a/config/master.conf b/config/master.conf index ee8a6251..0c69e4c9 100644 --- a/config/master.conf +++ b/config/master.conf @@ -169,12 +169,12 @@ enable_yararules="yes" #Enables yararules in the various databases, automatica # " # Default dbs rating -# valid rating: LOW, MEDIUM, HIGH +# valid rating: LOW, MEDIUM, HIGH, DISABLE default_dbs_rating="MEDIUM" # Per Database # These ratings will override the global rating for the specific database -# valid ratings: LOW | MEDIUM | HIGH | DISABLED +# valid ratings: LOW | MEDIUM | HIGH | DISABLE #linuxmalwaredetect_dbs_rating="" #sanesecurity_dbs_rating="" #securiteinfo_dbs_rating="" diff --git a/config/user.conf b/config/user.conf index f685b39d..d7663d6e 100644 --- a/config/user.conf +++ b/config/user.conf @@ -38,7 +38,7 @@ #securiteinfo_premium="yes" # Default dbs rating (Default: MEDIUM) -# valid rating: LOW, MEDIUM, HIGH +# valid rating: LOW, MEDIUM, HIGH, DISABLE #default_dbs_rating="HIGH" # Per Database From 7261c4a3d0b5a946d8ac634cfd1fc8e0d4e2d4e4 Mon Sep 17 00:00:00 2001 From: Daniel Nowacki Date: Sun, 6 Dec 2020 00:31:21 -0500 Subject: [PATCH 069/101] Fixed spelling, Markdown linting issues --- README.md | 1087 ++++++++++++++++++++++++++++------------------------- 1 file changed, 582 insertions(+), 505 deletions(-) diff --git a/README.md b/README.md index 1dc928d1..4364f6dc 100644 --- a/README.md +++ b/README.md @@ -2,47 +2,56 @@ ClamAV Unofficial Signatures Updater -## Maintained and provided by https://eXtremeSHOK.com +## Maintained and provided by ## Description + The clamav-unofficial-sigs script provides a simple way to download, test, and update third-party signature databases provided by Sanesecurity, FOXHOLE, OITC, Scamnailer, BOFHLAND, CRDF, Porcupine, Securiteinfo, MalwarePatrol, Yara-Rules Project, urlhaus, etc. The script will also generate and install cron, logrotate, and man files. -### Checkout some of our other solutions: https://github.com/extremeshok?tab=repositories +### Checkout some of our other solutions: ### Support / Suggestions / Comments -Please post them on the issue tracker : https://github.com/extremeshok/clamav-unofficial-sigs/issues + +Please post them on the issue tracker: ### Submit Patches / Pull requests to the "dev" Branch ### Required Ports / Firewall Exceptions + * rsync: TCP port 873 -* wget/curl : TCP port 443 +* wget/curl: TCP port 443 ### Supported Operating Systems -Debian, Ubuntu, Raspbian, CentOS (RHEL and clones), OpenBSD, FreeBSD, OpenSUSE, Archlinux, Mac OS X, Slackware, Solaris (Sun OS), pfSense, Zimbra and derivative systems + +Debian, Ubuntu, Raspbian, CentOS (RHEL and clones), OpenBSD, FreeBSD, OpenSUSE, Archlinux, Mac OS X, Slackware, Solaris (Sun OS), pfSense, Zimbra and derivative systems ### Quick Install and Upgrade Guide -https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/INSTALL.md + + ### Operating System Specific Install and Upgrade Guides -* CentOS : https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/guides/centos7.md -* Ubuntu : https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/guides/ubuntu-debian.md -* Debian : https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/guides/ubuntu-debian.md -* Mac OSX : https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/guides/macosx.md -* pFsense : https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/guides/pfsense.md + +* CentOS: +* Ubuntu: +* Debian: +* Mac OSX: +* pFsense: ### UPGRADE INSTRUCTIONS (version 7.0 +) -``` + +```bash clamav-unofficial-sigs.sh --upgrade clamav-unofficial-sigs.sh ``` ### FOR PACKAGE MAINTAINERS / PACKAGERS -Please use the sample package os.*.conf as a base for your os.conf, this will disable automatic updates, update notifications and the uninstallation feature. https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/config/packaging + +Please use the included os.*.conf sample config file as a base for your os.conf, this will disable automatic updates, update notifications and the uninstallation feature. ### Always Run the script once as your superuser to set all the permissions and create the relevant directories ### Advanced Config Overrides + * Default configs are loaded in the following order if they exist: * master.conf -> os.conf -> os.*.conf -> user.conf or your-specified.config * user.conf will always override os.conf and master.conf, os.conf will override master.conf @@ -51,673 +60,741 @@ Please use the sample package os.*.conf as a base for your os.conf, this will di * Specifying a config on the command line (-c | --config) will override the loading of the default configs #### Check if signature are being loaded + **Run the following command to display which signatures are being loaded by clamav ```clamscan --debug 2>&1 /dev/null | grep "loaded"``` #### SELinux cron permission fix + > WARNING - Clamscan reports ________ database integrity tested BAD - SKIPPING **Run the following command to allow clamav selinux support** - ```setsebool -P antivirus_can_scan_system true``` ### Yara Rule Support automatically enabled (as of April 2016) -Since usage yara rules requires clamav 0.100 or above, they will be automatically deactivated if your clamav is older than the required version +Since usage yara rules requires clamav 0.100 or above, they will be automatically deactivated if your clamav is older than the required version ### URLhaus Support (as of January 2020) -Usage of free URLhaus Database: https://urlhaus.abuse.ch -- Enabled by default + +Usage of free URLhaus Database: + +* Enabled by default ### Yara-Rules Project Support (as of June 2015, updated January 2020) -Usage of free Yara-Rules Project: http://yararules.com -- Enabled by default -Current limitations of clamav support : http://blog.clamav.net/search/label/yara +Usage of free Yara-Rules Project: + +* Enabled by default + +Current limitations of clamav support: + +### malware.expert non-free database support (as of December 2020) -### malware.experrt non-free database support (as of December 2020) -Usage of Malware Expert : https://www.Malware Expert - - 1. Sign up for an account : https://www.Malware Expert - - 2. You will recieve an email containing your serial key/number - - 3. Enter the serial key into the config malwareexpert_serial_key: replacing YOUR-SERIAL-KEY with your serial key from the email +Usage of Malware Expert: + +1. Sign up for an account: +1. You will receive an email containing your serial key +1. Enter the serial key into the config malwareexpert_serial_key: replacing YOUR-SERIAL-KEY with your serial key from the email ### MalwarePatrol Free/Delayed list support (as of May 2015) -Usage of MalwarePatrol 2015 free clamav signatures : https://www.malwarepatrol.net - - 1. Sign up for a free account : https://www.malwarepatrol.net/free-guard-upgrade-option/ - - 2. You will recieve an email containing your password/receipt number - - 3. Enter the receipt number into the config malwarepatrol_receipt_code: replacing YOUR-RECEIPT-NUMBER with your receipt number from the email + +Usage of MalwarePatrol 2015 free clamav signatures: + +1. Sign up for a free account: +1. You will receive an email containing your password/receipt number +1. Enter the receipt number into the config malwarepatrol_receipt_code: replacing YOUR-RECEIPT-NUMBER with your receipt number from the email ### SecuriteInfo Free/Delayed list support (as of June 2015) -Usage of SecuriteInfo 2015 free clamav signatures : https://www.securiteinfo.com - - 1. Sign up for a free account : https://www.securiteinfo.com/clients/customers/signup - - 2. You will recieve an email to activate your account and then a followup email with your login name - - 3. Login and navigate to your customer account : https://www.securiteinfo.com/clients/customers/account - - 4. Click on the Setup tab - - 5. You will need to get your unique identifier from one of the download links, they are individual for every user - - 5.1. The 128 character string is after the http://www.securiteinfo.com/get/signatures/ - - 5.2. Example https://www.securiteinfo.com/get/signatures/your_unique_and_very_long_random_string_of_characters/securiteinfo.hdb - Your 128 character authorisation signature would be : your_unique_and_very_long_random_string_of_characters - - 6. Enter the authorisation signature into the config securiteinfo_authorisation_signature: replacing YOUR-SIGNATURE-NUMBER with your authorisation signature from the link + +Usage of SecuriteInfo 2015 free clamav signatures: + +1. Sign up for a free account: +1. You will receive an email to activate your account and then a followup email with your login name +1. Login and navigate to your customer account: +1. Click on the Setup tab +1. You will need to get your unique identifier from one of the download links, they are individual for every user + 1. The 128 character string is after the + 1. Example + Your 128 character authorisation signature would be: your_unique_and_very_long_random_string_of_characters +1. Enter the authorisation signature into the config securiteinfo_authorisation_signature: replacing YOUR-SIGNATURE-NUMBER with your authorisation signature from the link ### Linux Malware Detect support (as of May 2015, updated January 2020) -Usage of free Linux Malware Detect clamav signatures: https://www.rfxn.com/projects/linux-malware-detect/ - - Enabled by default, no configuration required -### Need a database added ? Missing a database or a database not working ? -Please post on the issue tracker : https://github.com/extremeshok/clamav-unofficial-sigs/issues +Usage of free Linux Malware Detect clamav signatures: + +* Enabled by default, no configuration required + +### If you want to add, report a missing one or have a problem with a database + +Please post on the issue tracker: ## USAGE -Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] +```bash +Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] --c, --config Use a specific configuration file or directory +-c, --config Use a specific configuration file or directory eg: '-c /your/dir' or ' -c /your/file.name' - Note: If a directory is specified the directory must contain atleast: + Note: If a directory is specified the directory must contain at least: master.conf, os.conf or user.conf Default Directory: /etc/clamav-unofficial-sigs --F, --force Force all databases to be downloaded, could cause ip to be blocked +-F, --force Force all databases to be downloaded, could cause ip to be blocked --h, --help Display this script's help and usage information +-h, --help Display this script's help and usage information --V, --version Output script version and date information +-V, --version Output script version and date information --v, --verbose Be verbose, enabled when not run under cron +-v, --verbose Be verbose, enabled when not run under cron --s, --silence Only output error messages, enabled when run under cron +-s, --silence Only output error messages, enabled when run under cron --d, --decode-sig Decode a third-party signature either by signature name +-d, --decode-sig Decode a third-party signature either by signature name (eg: Sanesecurity.Junk.15248) or hexadecimal string. This flag will 'NOT' decode image signatures --e, --encode-string Hexadecimal encode an entire input string that can +-e, --encode-string Hexadecimal encode an entire input string that can be used in any '*.ndb' signature database file --f, --encode-formatted Hexadecimal encode a formatted input string containing +-f, --encode-formatted Hexadecimal encode a formatted input string containing signature spacing fields '{}, (), *', without encoding the spacing fields, so that the encoded signature can be used in any '*.ndb' signature database file --g, --gpg-verify GPG verify a specific Sanesecurity database file +-g, --gpg-verify GPG verify a specific Sanesecurity database file eg: '-g filename.ext' (do not include file path) --i, --information Output system and configuration information for +-i, --information Output system and configuration information for viewing or possible debugging purposes --m, --make-database Make a signature database from an ascii file containing +-m, --make-database Make a signature database from an ascii file containing data strings, with one data string per line. Additional information is provided when using this flag --t, --test-database Clamscan integrity test a specific database file +-t, --test-database Clamscan integrity test a specific database file eg: '-t filename.ext' (do not include file path) --o, --output-triggered If HAM directory scanning is enabled in the script's +-o, --output-triggered If HAM directory scanning is enabled in the script's configuration file, then output names of any third-party signatures that triggered during the HAM directory scan --w, --whitelist Adds a signature whitelist entry in the newer ClamAV IGN2 +-w, --whitelist Adds a signature whitelist entry in the newer ClamAV IGN2 format to 'my-whitelist.ign2' in order to temporarily resolve a false-positive issue with a specific third-party signature. Script added whitelist entries will automatically be removed if the original signature is either modified or removed from the third-party signature database ---check-clamav If ClamD status check is enabled and the socket path is correctly - specifiedthen test to see if clamd is running or not +--check-clamav If ClamD status check is enabled and the socket path is correctly + specified then test to see if clamd is running or not ---upgrade Upgrades this script and master.conf to the latest available version +--upgrade Upgrades this script and master.conf to the latest available version ---install-all Install and generate the cron, logroate and man files, autodetects the values +--install-all Install and generate the cron, logrotate and man files, autodetects the values based on your config files ---install-cron Install and generate the cron file, autodetects the values +--install-cron Install and generate the cron file, autodetects the values based on your config files ---install-logrotate Install and generate the logrotate file, autodetects the +--install-logrotate Install and generate the logrotate file, autodetects the values based on your config files ---install-man Install and generate the man file, autodetects the +--install-man Install and generate the man file, autodetects the values based on your config files ---remove-script Remove the clamav-unofficial-sigs script and all of +--remove-script Remove the clamav-unofficial-sigs script and all of its associated files and databases from the system +``` ## Change Log -### Version 7.2 ( XX December 2020 : Under Development) - - Database rating downgrades are now supported, eg, changing from HIGH to LOW will remove the HIGH and MEDIUM rated databases. - - Disabled databases are automatically removed - - Disable databases by setting the rating to "DISABLED" eg. securiteinfo_dbs_rating="DISABLED" will disable all securiteinfo databases - - Added Malware Expert databases (non-free) - - Reworked securiteinfo premium databases (non-free) - - Added malwarepatrol_db to specifiy the extact database name (default: malwarepatrol.db) - - Added detection of tar executable (use gtar on mac and bsd) - - Config os.macosx.conf renamed to os.macos.conf - - Fix: set ownership of last-version-check.txt - - Incremented the config to version 93 + +### Version 7.2 ( XX December 2020: Under Development) + +* Database rating downgrades are now supported, eg, changing from HIGH to LOW will remove the HIGH and MEDIUM rated databases. +* Disabled databases are automatically removed +* Disable databases by setting the rating to "DISABLED" eg. securiteinfo_dbs_rating="DISABLED" will disable all securiteinfo databases +* Added Malware Expert databases (non-free) +* Reworked securiteinfo premium databases (non-free) +* Added malwarepatrol_db to specify the exact database name (default: malwarepatrol.db) +* Added detection of tar executable (use gtar on mac and bsd) +* Config os.macosx.conf renamed to os.macos.conf +* Fix: set ownership of last-version-check.txt +* Incremented the config to version 93 ### Version 7.1 ( Not Released) - - Enforce HTTPS validation by default - - Updated sanesecurity publickey.gpg url to use SSL - - Ignore yara files that include modules - - Enabled yararulesproject rules by default - - os.gentoo.conf: disable updates and upgrade checks - - Fix: URLhaus log message - - Fix wrong download URL for MalwarePatrol - - Fix: fallback to host if dig is not used - - Disable cron MAILTO - - BSD read config fix - - Incremented the config to version 92 - - Thank you @dandanio @Mrothyr @m0urs @msapiro @orlitzky @SlothOfAnarchy @jkellerer @RobbieTheK + +* Enforce HTTPS validation by default +* Updated sanesecurity publickey.gpg url to use SSL +* Ignore yara files that include modules +* Enabled yararulesproject rules by default +* os.gentoo.conf: disable updates and upgrade checks +* Fix: URLhaus log message +* Fix wrong download URL for MalwarePatrol +* Fix: fallback to host if dig is not used +* Disable cron MAILTO +* BSD read config fix +* Incremented the config to version 92 +* Thank you @dandanio @Mrothyr @m0urs @msapiro @orlitzky @SlothOfAnarchy @jkellerer @RobbieTheK ### Version 7.0.1 (25 January 2020) - - Disable yara project rules duplicated in rxfn.yara (Thanks @dominicraf) - - Incremented the config to version 91 + +* Disable yara project rules duplicated in rxfn.yara (Thanks @dominicraf) +* Incremented the config to version 91 ### Version 7.0.0 (24 January 2020) - - eXtremeSHOK.com Maintenance - - Added urlhaus database - - Added extra yararulesproject databases - - Added new linuxmalwaredetect yara file - - Automatic upgrades ( --upgrade ) - - Added --upgrade command line option - - Option to disable automatic upgrades ( allow_upgrades ) - - Option to disable update checks (allow_update_checks) - - Increase download time to 1800 seconds from 600 seconds - - os.conf takes preference over os.***.conf - - Warn if there are multiple os.***.conf files - - More sanity checks to help users and prevent errors - - Better output of --info - - Fix all known bugs - - Implement all suggestions - - Fixed yararulesproject database names - - Correctly silence curl and wget - - New linuxmalwaredetect logic - - New malwarepatrol logic - - Suppress --- and === from the logs - - Update the documentation / guides - - Increase minimum clamav version for yara rules to 0.100 or above - - Fix systemd.timer and systemd.service files - - More travis-ci tests - - Added os.alpine.conf - - Added debug options/mode to config - - Set minimum config required to 90 - - Lots of refactoring and optimizing - - Only check for and notify about script updates every 12hours - - Incremented the config to version 90 + +* eXtremeSHOK.com Maintenance +* Added urlhaus database +* Added extra yararulesproject databases +* Added new linuxmalwaredetect yara file +* Automatic upgrades ( --upgrade ) +* Added --upgrade command line option +* Option to disable automatic upgrades ( allow_upgrades ) +* Option to disable update checks (allow_update_checks) +* Increase download time to 1800 seconds from 600 seconds +* os.conf takes preference over os.***.conf +* Warn if there are multiple os.***.conf files +* More sanity checks to help users and prevent errors +* Better output of --info +* Fix all known bugs +* Implement all suggestions +* Fixed yararulesproject database names +* Correctly silence curl and wget +* New linuxmalwaredetect logic +* New malwarepatrol logic +* Suppress --- and === from the logs +* Update the documentation / guides +* Increase minimum clamav version for yara rules to 0.100 or above +* Fix systemd.timer and systemd.service files +* More travis-ci tests +* Added os.alpine.conf +* Added debug options/mode to config +* Set minimum config required to 90 +* Lots of refactoring and optimizing +* Only check for and notify about script updates every 12hours +* Incremented the config to version 90 ### Version 6.1.1 (02 September 2019) - - eXtremeSHOK.com Maintenance - - Update os.archlinux.conf, thanks @amishmm - - master.conf set default dbs rating to medium - - user.conf better suggested values - - Default to using curl, less logic required (lower cpu) - - force_curl replaced with force_wget - - Fix: suppress all non-error output under cron/non interactive terminal - - Fix: check log file is not a link before setting permissions, only set if owned by root. - - Fix: failed to create symbolic link - - Fix: curl --compress ->> curl --compressed - - Minor enhancement to travis-ci checks - - Incremented the config to version 77 + +* eXtremeSHOK.com Maintenance +* Update os.archlinux.conf, thanks @amishmm +* master.conf set default dbs rating to medium +* user.conf better suggested values +* Default to using curl, less logic required (lower cpu) +* force_curl replaced with force_wget +* Fix: suppress all non-error output under cron/non interactive terminal +* Fix: check log file is not a link before setting permissions, only set if owned by root. +* Fix: failed to create symbolic link +* Fix: curl --compress ->> curl --compressed +* Minor enhancement to travis-ci checks +* Incremented the config to version 77 ### Version 6.1.0 (27 August 2019) - - eXtremeSHOK.com Maintenance - - Thanks Reio Remma & Oliver Nissen - - fail added to all curl commands - - Fix: Missing logic for LOWMEDIUMONLY | MEDIUMHIGHONLY | HIGHONLY databases - - Support for either os.osname.conf or os.conf files (no more needing to rename the os.osname.conf to os.conf) - - Where possible replaced echo with xshok_pretty_echo_and_log - - Refactor xshok_pretty_echo_and_log and make all notices styles consistent - - Silence output when run under cron - - add MAILTO=root to the generated cron file - - Add full proxy support for wget, curl, rsync, dig, host - - Better support for proxy config variables - - New config variable: git_branch (defaults to master for the update checks) - - allow -w signature for quicker whitelisting - - Sanitize whitelist input string (Remove quotes and .UNOFFICIAL) - - Added Full support for Hash-based Signature Databases - - User.conf is pre-configured with default options to allow for quicker setup - - Default sanesecurity and LinuxMalwareDetect to enabled - - Increase default retries from 3 to 5 - - Ensure log file permissions are correct - - Better update comparison check, only notify if newer - - Incremented the config to version 76 + +* eXtremeSHOK.com Maintenance +* Thanks Reio Remma & Oliver Nissen +* fail added to all curl commands +* Fix: Missing logic for LOWMEDIUMONLY | MEDIUMHIGHONLY | HIGHONLY databases +* Support for either os.osname.conf or os.conf files (no more needing to rename the os.osname.conf to os.conf) +* Where possible replaced echo with xshok_pretty_echo_and_log +* Refactor xshok_pretty_echo_and_log and make all notices styles consistent +* Silence output when run under cron +* add MAILTO=root to the generated cron file +* Add full proxy support for wget, curl, rsync, dig, host +* Better support for proxy config variables +* New config variable: git_branch (defaults to master for the update checks) +* allow -w signature for quicker whitelisting +* Sanitize whitelist input string (Remove quotes and .UNOFFICIAL) +* Added Full support for Hash-based Signature Databases +* User.conf is pre-configured with default options to allow for quicker setup +* Default sanesecurity and LinuxMalwareDetect to enabled +* Increase default retries from 3 to 5 +* Ensure log file permissions are correct +* Better update comparison check, only notify if newer +* Incremented the config to version 76 ### Version 6.0.1 - - eXtremeSHOK.com Maintenance - - Fix logging @dominicraf + +* eXtremeSHOK.com Maintenance +* Fix logging @dominicraf ### Version 6.0 - - eXtremeSHOK.com Maintenance & Refactoring - - Add timestamp support (do not re-download not modified files, saves bandwidth) - - wget and curl uses compression for the transfer (detected when supported, saves bandwidth) - - Posix compliance 'which' replaced with 'command -v' - - More escaped characters, shellcheck compliance - - Option added : force_curl , to force the usage of curl instead of wget - - Workaround for wget, which cannot do --timestamping and --output-document together - - Added SECURITEINFO securiteinfoold.hdb - - set malwarepatrol_free = no , when malwarepatrol_product_code != 8 - - Fix: remove hardcoded malwarepatrol_product_code - - Fix: os.macosx.conf service: command not found - - Fix: whitelist a MalwarePatrol signature - - More reliable version checking - - Fix: Clamscan database integrity test - - Fix: version comparison of minimum Yara @bytesplit - - Use custom config directory @Amish - - unzip option -j was removed @wotomg - - ZCS 8.7 updates @tonster - - Logic fixes @Claus-Justus Heine - - Specify correct path for systemd units @SlothOfAnarchy - - Avoid hardcoded path to BASH @rseichter + +* eXtremeSHOK.com Maintenance & Refactoring +* Add timestamp support (do not re-download not modified files, saves bandwidth) +* wget and curl uses compression for the transfer (detected when supported, saves bandwidth) +* Posix compliance 'which' replaced with 'command -v' +* More escaped characters, shellcheck compliance +* Option added: force_curl , to force the usage of curl instead of wget +* Workaround for wget, which cannot do --timestamping and --output-document together +* Added SECURITEINFO securiteinfoold.hdb +* set malwarepatrol_free = no , when malwarepatrol_product_code != 8 +* Fix: remove hardcoded malwarepatrol_product_code +* Fix: os.macosx.conf service: command not found +* Fix: whitelist a MalwarePatrol signature +* More reliable version checking +* Fix: Clamscan database integrity test +* Fix: version comparison of minimum Yara @bytesplit +* Use custom config directory @Amish +* unzip option -j was removed @wotomg +* ZCS 8.7 updates @tonster +* Logic fixes @Claus-Justus Heine +* Specify correct path for systemd units @SlothOfAnarchy +* Avoid hardcoded path to BASH @rseichter ### Version 5.6.2 - - eXtremeSHOK.com Maintenance - - Bug Fix GPG always being disabled, thanks @orlitzky + +* eXtremeSHOK.com Maintenance +* Bug Fix GPG always being disabled, thanks @orlitzky ### Version 5.6.1 - - eXtremeSHOK.com Maintenance - - Packers/Javascript_exploit_and_obfuscation.yar false positive rating increased to HIGH - - Codeclimate fixes - - Incremented the config to version 73 + +* eXtremeSHOK.com Maintenance +* Packers/Javascript_exploit_and_obfuscation.yar false positive rating increased to HIGH +* Codeclimate fixes +* Incremented the config to version 73 ### Version 5.6 - - eXtremeSHOK.com Maintenance - - PGP is now optional and no longer a requirement and pgp support is auto-detected - - Full support for MacOS / OS X and added clamav install guide - - Full support for pfSense and added clamav install guide - - Added os configs for Zimbra and Debian 8 with systemd - - Much better error messages with possible solutions given - - Better checking of possible issues - - Update all SANESECURITY signature databases - - Support for clamav-devel (clamav compiled from source) - - Added full proxy support to wget and curl - - Replace allot of "echo | cut | sed" with bash substitutions - - Added fallbacks/substitutions for various commands - - xshok_file_download and xshok_draw_time_remaining functions added to replace redundant code blocks - - Removed SANESECURITY mbl.ndb as this file is not showing up on the rsync mirrors - - Allow exit code 23 for rsync - - Major refactoring : Normalize comments, quotes, functions, conditions - - Protect various arguments and "POSIX-ize" script integrity - - Enhanced testing with travis-ci, including clamav 0.99 - - Incremented the config to version 72 + +* eXtremeSHOK.com Maintenance +* PGP is now optional and no longer a requirement and pgp support is auto-detected +* Full support for MacOS / OS X and added clamav install guide +* Full support for pfSense and added clamav install guide +* Added os configs for Zimbra and Debian 8 with systemd +* Much better error messages with possible solutions given +* Better checking of possible issues +* Update all SANESECURITY signature databases +* Support for clamav-devel (clamav compiled from source) +* Added full proxy support to wget and curl +* Replace allot of "echo | cut | sed" with bash substitutions +* Added fallbacks/substitutions for various commands +* xshok_file_download and xshok_draw_time_remaining functions added to replace redundant code blocks +* Removed SANESECURITY mbl.ndb as this file is not showing up on the rsync mirrors +* Allow exit code 23 for rsync +* Major refactoring: Normalize comments, quotes, functions, conditions +* Protect various arguments and "POSIX-ize" script integrity +* Enhanced testing with travis-ci, including clamav 0.99 +* Incremented the config to version 72 ### Version 5.4.1 - - eXtremeSHOK.com Maintenance - - Disable installation when either pkg_mgr or pkg_rm is defined. - - Minor refactoring - - Update master.conf with the new Yara-rules project file names - - Incremented the config to version 69 + +* eXtremeSHOK.com Maintenance +* Disable installation when either pkg_mgr or pkg_rm is defined. +* Minor refactoring +* Update master.conf with the new Yara-rules project file names +* Incremented the config to version 69 ### Version 5.4 - - eXtremeSHOK.com Maintenance - - Added Solaris 10 and 11 configs - - When under Solaris we define our own which function - - Define grep_bin variable, use gnu grep on sun os - - Fallback to gpg2 if gpg not found, - - Added support for csw gnupg on solaris - - Trap the keyboard interrupt (ctrl+c) and gracefully exit - - Added CentOS 7 Atomic config @deajan - - Minor refactoring and removing of unused variables - - Removed CRDF signatures as per Sanesecurity #124 - - Added more Yara rule project Rules - - Incremented the config to version 68 + +* eXtremeSHOK.com Maintenance +* Added Solaris 10 and 11 configs +* When under Solaris we define our own which function +* Define grep_bin variable, use gnu grep on sun os +* Fallback to gpg2 if gpg not found, +* Added support for csw gnupg on solaris +* Trap the keyboard interrupt (ctrl+c) and gracefully exit +* Added CentOS 7 Atomic config @deajan +* Minor refactoring and removing of unused variables +* Removed CRDF signatures as per Sanesecurity #124 +* Added more Yara rule project Rules +* Incremented the config to version 68 ### Version 5.3.2 - - eXtremeSHOK.com Maintenance - - Bug Fix: Additional Databases not downloading - - Added sanesecurity_update_hours option to limit updating to once every 2 hours - - Added additional_update_hours option to limit updating to once every 4 hours - - Refactor Additional Database File Update code - - Updated osx config with correct group for homebrew + +* eXtremeSHOK.com Maintenance +* Bug Fix: Additional Databases not downloading +* Added sanesecurity_update_hours option to limit updating to once every 2 hours +* Added additional_update_hours option to limit updating to once every 4 hours +* Refactor Additional Database File Update code +* Updated osx config with correct group for homebrew ### Version 5.3.1 - - eXtremeSHOK.com Maintenance - - Bug Fix: for GPG Signature test FAILED by @DamianoBianchi - - Remove unused $GETOPT - - Refactor clamscan_integrity_test_specific_database_file (--test-database) - - Refactor gpg_verify_specific_sanesecurity_database_file (--gpg-verify) - - Big fix: missing $pid_dir + +* eXtremeSHOK.com Maintenance +* Bug Fix: for GPG Signature test FAILED by @DamianoBianchi +* Remove unused $GETOPT +* Refactor clamscan_integrity_test_specific_database_file (--test-database) +* Refactor gpg_verify_specific_sanesecurity_database_file (--gpg-verify) +* Big fix: missing $pid_dir ### Version 5.3.0 - - eXtremeSHOK.com Maintenance - - Major change: Updated to use new database structure, now allows all low/medium/high databases to be enabled or disabled. - - Major change: curl replaced with wget (will fallback to curl is wget is not installed) - - Major change: script now functions correctly as the clamav user when started under cron - - Added fallback to curl if wget is not available - - Added locking (Enable pid file to prevent issues with multiple instances) - - Added retries to fetching downloads - - Code refactor: if wget repaced with if $? -ne 0 - - Enhancement: Verify the clam_user and clam_group actually exists on the system - - Added function : xshok_user_group_exists, to check if a specific user and group exists - - Bug Fix: setmode only if is root - - Bug Fix: eval not working on certain systems - - Bug fix: rsync output not correctly silenced - - Code refactor: remove legacy `..` with $(...) - - Code refactor: replace [ ... -a ... ] with [ ... ] && [ ... ] - - Code refactor: replace [ ... -o ... ] with [ ... ] || [ ... ] - - Code refactor: replace cat "..." with done < ... from loops - - Code refactor: convert for loops using files to while loops - - Code refactor: read replaced with read -r - - Code refactor: added cd ... || exit , to handle a failed cd - - Code refactor: double quoted all varibles - - Code refactor: refactor all "ls" iterations to use globs - - Defined missing uname_bin variable - - Added function xshok_database - - Set minimum config required to 65 - - Bump config to 65 + +* eXtremeSHOK.com Maintenance +* Major change: Updated to use new database structure, now allows all low/medium/high databases to be enabled or disabled. +* Major change: curl replaced with wget (will fallback to curl is wget is not installed) +* Major change: script now functions correctly as the clamav user when started under cron +* Added fallback to curl if wget is not available +* Added locking (Enable pid file to prevent issues with multiple instances) +* Added retries to fetching downloads +* Code refactor: if wget repaced with if $? -ne 0 +* Enhancement: Verify the clam_user and clam_group actually exists on the system +* Added function: xshok_user_group_exists, to check if a specific user and group exists +* Bug Fix: setmode only if is root +* Bug Fix: eval not working on certain systems +* Bug fix: rsync output not correctly silenced +* Code refactor: remove legacy `..` with $(...) +* Code refactor: replace [ ... -a ... ] with [ ... ] && [ ... ] +* Code refactor: replace [ ... -o ... ] with [ ... ] || [ ... ] +* Code refactor: replace cat "..." with done < ... from loops +* Code refactor: convert for loops using files to while loops +* Code refactor: read replaced with read -r +* Code refactor: added cd ... || exit , to handle a failed cd +* Code refactor: double quoted all varibles +* Code refactor: refactor all "ls" iterations to use globs +* Defined missing uname_bin variable +* Added function xshok_database +* Set minimum config required to 65 +* Bump config to 65 ### Version 5.2.2 - - eXtremeSHOK.com Maintenance - - Added --install-all Install and generate the cron, logroate and man files, autodetects the values $oft based on your config files - - Added functions: xshok_prompt_confirm, xshok_is_file, xshok_is_subdir - - Replaced Y/N prompts with xshok_prompt_confirm - - Bug Fix for disabled databases being removed when the remove_disabled_databases is set to NO (default) - - Added more warnings to remove_script and made it double confirmed - - Remove_script will only remove work_dir if its a sub directory - - Remove_script will only remove files if they are files - - Removed -r switch, --remove-script needs to be used instead of both -r and --remove-script - - Fixed: remove_script not removing logrotate file, cron file, man file + +* eXtremeSHOK.com Maintenance +* Added --install-all Install and generate the cron, logroate and man files, autodetects the values $oft based on your config files +* Added functions: xshok_prompt_confirm, xshok_is_file, xshok_is_subdir +* Replaced Y/N prompts with xshok_prompt_confirm +* Bug Fix for disabled databases being removed when the remove_disabled_databases is set to NO (default) +* Added more warnings to remove_script and made it double confirmed +* Remove_script will only remove work_dir if its a sub directory +* Remove_script will only remove files if they are files +* Removed -r switch, --remove-script needs to be used instead of both -r and --remove-script +* Fixed: remove_script not removing logrotate file, cron file, man file ### Version 5.2.1 - - eXtremeSHOK.com Maintenance - - Minor bugfix for Sanesecurity_sigtest.yara Sanesecurity_spam.yara files being removed incorrectly - - Minor fix: yararulesproject_enabled not yararulesproject_enable + +* eXtremeSHOK.com Maintenance +* Minor bugfix for Sanesecurity_sigtest.yara Sanesecurity_spam.yara files being removed incorrectly +* Minor fix: yararulesproject_enabled not yararulesproject_enable ### Version 5.2.0 - - eXtremeSHOK.com Maintenance - - Refactor some functions - - Added --install-man this will automatically generate and install the man (help) file - - Yararules and yararulesproject enabled by default - - Added clamav version detection to automatically disable yararules and yararulesproject if the current clamav version does not support them - - Database files ending with .yar/.yara/.yararules will automatically be disabled from the database if yara rules are not supported - - Script options are added to the man file - - Fixed hardcoded logrotate and cron in remove_script - - Fixed incorrectly assigned logrotate varibles in install-logrotate - - Config added info for port/package maintainers regarding: pkg_mgr and pkg_rm - - Removed pkg_mgr and pkg_rm from freebsd and openbsd os configs - - Allow overriding of all the individual workdirs, this is mainly to aid package maintainers - - Rename sanesecurity_dir to work_dir_sanesecurity, securiteinfo_dir to work_dir_securiteinfo, malwarepatrol_dir to work_dir_malwarepatrol, yararules_dir to work_dir_yararules, add_dir to work_dir_add, gpg_dir to work_dir_gpg, work_dir_configs to work_dir_work_configs - - Rename yararules_enabled to yararulesproject_enabled - - Rename all yararules to yararulesproject - - Fix to prevent disabled databases processing certian things which will not be used as they are disabled - - Set minimum config required to 62 - - Bump config to 62 + +* eXtremeSHOK.com Maintenance +* Refactor some functions +* Added --install-man this will automatically generate and install the man (help) file +* Yararules and yararulesproject enabled by default +* Added clamav version detection to automatically disable yararules and yararulesproject if the current clamav version does not support them +* Database files ending with .yar/.yara/.yararules will automatically be disabled from the database if yara rules are not supported +* Script options are added to the man file +* Fixed hardcoded logrotate and cron in remove_script +* Fixed incorrectly assigned logrotate varibles in install-logrotate +* Config added info for port/package maintainers regarding: pkg_mgr and pkg_rm +* Removed pkg_mgr and pkg_rm from freebsd and openbsd os configs +* Allow overriding of all the individual workdirs, this is mainly to aid package maintainers +* Rename sanesecurity_dir to work_dir_sanesecurity, securiteinfo_dir to work_dir_securiteinfo, malwarepatrol_dir to work_dir_malwarepatrol, yararules_dir to work_dir_yararules, add_dir to work_dir_add, gpg_dir to work_dir_gpg, work_dir_configs to work_dir_work_configs +* Rename yararules_enabled to yararulesproject_enabled +* Rename all yararules to yararulesproject +* Fix to prevent disabled databases processing certian things which will not be used as they are disabled +* Set minimum config required to 62 +* Bump config to 62 ### Version 5.1.1 - - eXtremeSHOK.com Maintenance - - Added OS X and openbsd configs - - Fixed host fallback sed issues by @MichaelKuch - - Suppress most error messages of chmod and chown - - check permissions before chmod - - Added the config option remove_disabled_databases # Default is "no", if enabled when a database is disabled we will remove the associated database files. - - Added function xshok_mkdir_ownership - - Do not set permissions of the log, cron and logrotate dirs - - Fix: fallback for missing gpg -r option on OS X - - Update sanesecurity signatures - - Bump config to 61 + +* eXtremeSHOK.com Maintenance +* Added OS X and openbsd configs +* Fixed host fallback sed issues by @MichaelKuch +* Suppress most error messages of chmod and chown +* check permissions before chmod +* Added the config option remove_disabled_databases # Default is "no", if enabled when a database is disabled we will remove the associated database files. +* Added function xshok_mkdir_ownership +* Do not set permissions of the log, cron and logrotate dirs +* Fix: fallback for missing gpg -r option on OS X +* Update sanesecurity signatures +* Bump config to 61 ### Version 5.1.0 - - eXtremeSHOK.com Maintenance - - Added --install-cron this will automatically generate and install the cron file - - Added --install-logrotate this will automatically generate and install the logrotate file - - Change official URL of SecuriteInfo signatures - - Added a new database (securiteinfoandroid.hdb) for SecuriteInfo - - Remove database files after disabling a database group by @reneschuster - - Updated Gentoo OS config by @orlitzky - - Regroup functiuons - - Increase travis-ci code testing - - Set minimum config required to 60 - - Bump config to 60 + +* eXtremeSHOK.com Maintenance +* Added --install-cron this will automatically generate and install the cron file +* Added --install-logrotate this will automatically generate and install the logrotate file +* Change official URL of SecuriteInfo signatures +* Added a new database (securiteinfoandroid.hdb) for SecuriteInfo +* Remove database files after disabling a database group by @reneschuster +* Updated Gentoo OS config by @orlitzky +* Regroup functiuons +* Increase travis-ci code testing +* Set minimum config required to 60 +* Bump config to 60 ### Version 5.0.6 - - eXtremeSHOK.com Maintenance - - Updated winnow databases as per information from Tom @ OITC - - Bump config to 58 + +* eXtremeSHOK.com Maintenance +* Updated winnow databases as per information from Tom @ OITC +* Bump config to 58 ### Version 5.0.5 - - eXtremeSHOK.com Maintenance - - Add support for specifying a custom config dir or file with (--config) -c option - - Removed default_config - - Added travis-ci build testing - - Updates to the help and usage display - - Added sanity testing of sanesecurity_dbs, securiteinfo_dbs, linuxmalwaredetect_dbs, yararules_dbs, add_dbs - - Added function xshok_array_count - - Prevent some issues with an incomplete or only a user.conf being loaded - - Added fallback to host if dig returns no records - - Check there are Sanesecurity mirror ips before we attempt to rsync - - Important binaries have been aliased (clamscan, rsync, curl, gpg) and allow their paths to be overridden - - Added sanity checks to make sure the binaries and workdir is defined - - Custom Binary Paths added to the config (clamscan_bin, rsync_bin, curl_bin, gpg_bin) - - Bump config to 57 - - Added initial centos6 + cpanel os config - - Bugfix Only start logging once all the configs have been loaded - - Rename $version to script_version - - Default malwarePatrol to the free version - - Added script version checks + +* eXtremeSHOK.com Maintenance +* Add support for specifying a custom config dir or file with (--config) -c option +* Removed default_config +* Added travis-ci build testing +* Updates to the help and usage display +* Added sanity testing of sanesecurity_dbs, securiteinfo_dbs, linuxmalwaredetect_dbs, yararules_dbs, add_dbs +* Added function xshok_array_count +* Prevent some issues with an incomplete or only a user.conf being loaded +* Added fallback to host if dig returns no records +* Check there are Sanesecurity mirror ips before we attempt to rsync +* Important binaries have been aliased (clamscan, rsync, curl, gpg) and allow their paths to be overridden +* Added sanity checks to make sure the binaries and workdir is defined +* Custom Binary Paths added to the config (clamscan_bin, rsync_bin, curl_bin, gpg_bin) +* Bump config to 57 +* Added initial centos6 + cpanel os config +* Bugfix Only start logging once all the configs have been loaded +* Rename $version to script_version +* Default malwarePatrol to the free version +* Added script version checks ### Version 5.0.4 - - eXtremeSHOK.com Maintenance - - Added/Updated OS configs: CentOS 7, FreeBSD, Slackware - - Added clamd_reload_opt to fix issues with centos7 conf - - Fix --remove-script should call remove_script() function by @IdahoPL - - Add OS specific settings to logrotate - - Increased default timeout values - - Attempt to Silence more output - - Create the log_file_path directory before we touch the file. - - Updated config file to remove the $work_dir varible from dir names - - Remove trailing / from directory names - - Initial support for Travis-Ci testing - - Fixed config option enable_logging -> logging_enabled - - Config updated to 56 due to changes + +* eXtremeSHOK.com Maintenance +* Added/Updated OS configs: CentOS 7, FreeBSD, Slackware +* Added clamd_reload_opt to fix issues with centos7 conf +* Fix --remove-script should call remove_script() function by @IdahoPL +* Add OS specific settings to logrotate +* Increased default timeout values +* Attempt to Silence more output +* Create the log_file_path directory before we touch the file. +* Updated config file to remove the $work_dir varible from dir names +* Remove trailing / from directory names +* Initial support for Travis-Ci testing +* Fixed config option enable_logging -> logging_enabled +* Config updated to 56 due to changes ### Version 5.0.3 - - eXtremeSHOK.com Maintenance - - Added OS configs: OpenSUSE, Archlinux, Gentoo, Raspbian, FreeBSD - - Fixed config option enable_logging -> logging_enabled + +* eXtremeSHOK.com Maintenance +* Added OS configs: OpenSUSE, Archlinux, Gentoo, Raspbian, FreeBSD +* Fixed config option enable_logging -> logging_enabled ### Version 5.0.2 - - eXtremeSHOK.com Maintenance - - Detect if the entire script is available/complete - - Fix for Missing space between "] + +* eXtremeSHOK.com Maintenance +* Detect if the entire script is available/complete +* Fix for Missing space between "] ### Version 5.0.1 - - eXtremeSHOK.com Maintenance - - Disable logging if the log file is not writable. - - Do not attempt to log before a config is loaded + +* eXtremeSHOK.com Maintenance +* Disable logging if the log file is not writable. +* Do not attempt to log before a config is loaded ### Version 5.0.0 - - eXtremeSHOK.com Maintenance - - Added porcupine.hsb : Sha256 Hashes of VBS and JSE malware Database from sanesecurity - - Fix for missing $ for clamd_pid an incorrect variable definition - - Fixes for not removing dirs by @msapiro - - Updates to account for changed names and addition of sub-directories for Yara-Rules by @msapiro - - Use MD5 with MalwarePatrol by @olivier2557 - - Suppress the header and config loading message if running via cron - - Added systemd files by @falon - - Added config option remove_bad_database, a database with a BAD integrity check will be removed - - Fixed broken whitelisting of malwarepatrol signatures - - Replaced Version command option -v with -V - - Added command option -v (--verbose) to force verbose output - - Removed config options: silence_ssl, curl_silence, rsync_silence, gpg_silence, comment_silence - - Added ignore_ssl option to supress ssl errors and warnings, ie operate in insecure mode. - - Replaced test-database command option -s with -t - - Replaced output-triggered command option -t with -o - - Added command option -s (--silence) to force silenced output - - Default verbose for terminal and silence for cron - - Added RHEL/Centos 7 config settings - - Added short option (-F) to Force all databases to be downloaded, could cause ip to be blocked" - - Fixed removal of failed databases, disbale with option "remove_bad_database" - - Removed config options: clamd_start, clamd_stop - - Full rewrite of the config handling, master.conf -> os.conf -> user.conf or your-specified.config - - Configs loaded from the /etc/clamav-unofficial-sigs dir - - Added various os.conf files to ease setup - - Added selinux_fixes config option, this will run restorecon on the database files - - minor code refactoring and reindenting + +* eXtremeSHOK.com Maintenance +* Added porcupine.hsb: Sha256 Hashes of VBS and JSE malware Database from sanesecurity +* Fix for missing $ for clamd_pid an incorrect variable definition +* Fixes for not removing dirs by @msapiro +* Updates to account for changed names and addition of sub-directories for Yara-Rules by @msapiro +* Use MD5 with MalwarePatrol by @olivier2557 +* Suppress the header and config loading message if running via cron +* Added systemd files by @falon +* Added config option remove_bad_database, a database with a BAD integrity check will be removed +* Fixed broken whitelisting of malwarepatrol signatures +* Replaced Version command option -v with -V +* Added command option -v (--verbose) to force verbose output +* Removed config options: silence_ssl, curl_silence, rsync_silence, gpg_silence, comment_silence +* Added ignore_ssl option to supress ssl errors and warnings, ie operate in insecure mode. +* Replaced test-database command option -s with -t +* Replaced output-triggered command option -t with -o +* Added command option -s (--silence) to force silenced output +* Default verbose for terminal and silence for cron +* Added RHEL/Centos 7 config settings +* Added short option (-F) to Force all databases to be downloaded, could cause ip to be blocked" +* Fixed removal of failed databases, disbale with option "remove_bad_database" +* Removed config options: clamd_start, clamd_stop +* Full rewrite of the config handling, master.conf -> os.conf -> user.conf or your-specified.config +* Configs loaded from the /etc/clamav-unofficial-sigs dir +* Added various os.conf files to ease setup +* Added selinux_fixes config option, this will run restorecon on the database files +* minor code refactoring and reindenting ### Version 4.9.3 - - eXtremeSHOK.com Maintenance - - Various Bug Fixes - - Last release of 4.x.x base - - minor code refactoring + +* eXtremeSHOK.com Maintenance +* Various Bug Fixes +* Last release of 4.x.x base +* minor code refactoring ### Version 4.9.2 - - eXtremeSHOK.com Maintenance - - Added function xshok_check_s2 to prevent possible errors with -c and no configfile path - - minor code refactoring + +* eXtremeSHOK.com Maintenance +* Added function xshok_check_s2 to prevent possible errors with -c and no configfile path +* minor code refactoring ### Version 4.9.1 - - eXtremeSHOK.com Maintenance - - OS X compatibility fix by stewardle - - missing $ in $yararules_enabled + +* eXtremeSHOK.com Maintenance +* OS X compatibility fix by stewardle +* missing $ in $yararules_enabled ### Version 4.9 - - eXtremeSHOK.com Maintenance - - Code Refactoring - - New function clamscan_reload_dbs, will first try and reload the clam database, if reload fails will restart clamd - - Added Function xshok_pretty_echo_and_log, far easier and cleaner way to output and log information - - Removed functions comment, log - - Removed config option reload_opt - - Added config option clamd_restart_opt - - Added support for # characters in config values, ie malwarepatrol subscription key contains a # - - Minor formatting and code consitency changes - - 10% Smaller script size - - Config updated to 53 due to changes + +* eXtremeSHOK.com Maintenance +* Code Refactoring +* New function clamscan_reload_dbs, will first try and reload the clam database, if reload fails will restart clamd +* Added Function xshok_pretty_echo_and_log, far easier and cleaner way to output and log information +* Removed functions comment, log +* Removed config option reload_opt +* Added config option clamd_restart_opt +* Added support for # characters in config values, ie malwarepatrol subscription key contains a # +* Minor formatting and code consitency changes +* 10% Smaller script size +* Config updated to 53 due to changes ### Version 4.8 - - eXtremeSHOK.com Maintenance - - Added long option (--force) to Force all databases to be downloaded, could cause ip to be blocked" - - added config option: malwarepatrol_free="yes", set to "no" to enable commercial subscription url - - added support for commercial malwarepatrol subscription - - Grammar fix in config - - SELINUX cronjob fix added to readme - - Corrects tput warning when used without TERM (like in cron) - - Config updated to 52 due to changes + +* eXtremeSHOK.com Maintenance +* Added long option (--force) to Force all databases to be downloaded, could cause ip to be blocked" +* added config option: malwarepatrol_free="yes", set to "no" to enable commercial subscription url +* added support for commercial malwarepatrol subscription +* Grammar fix in config +* SELINUX cronjob fix added to readme +* Corrects tput warning when used without TERM (like in cron) +* Config updated to 52 due to changes ### Version 4.7 - - eXtremeSHOK.com Maintenance - - Code Refactoring - - Complete rewrite of the main case selector (program options) - - Added long options (--decode-sig, --encode-string, --encode-formatted, --gpg-verify, --information, --make-database, --remove-script, --test-database, --output-triggered) - - Replaced clamd-status.sh with --check-clamav - - Removed CHANGELOG, changelog has been replaced by this part of the readme and the git commit log. - - Config updated to 51 due to changes + +* eXtremeSHOK.com Maintenance +* Code Refactoring +* Complete rewrite of the main case selector (program options) +* Added long options (--decode-sig, --encode-string, --encode-formatted, --gpg-verify, --information, --make-database, --remove-script, --test-database, --output-triggered) +* Replaced clamd-status.sh with --check-clamav +* Removed CHANGELOG, changelog has been replaced by this part of the readme and the git commit log. +* Config updated to 51 due to changes ### Version 4.6.1 - - eXtremeSHOK.com Maintenance - - Code Refactoring - - Added generic options (--help --version --config) - - Correctly handle generic options before the main case selector - - Sanitize the config before the main case selector (option) - - Rewrite and formatting of the usage options - - Removed the version information code as this is always printed + +* eXtremeSHOK.com Maintenance +* Code Refactoring +* Added generic options (--help --version --config) +* Correctly handle generic options before the main case selector +* Sanitize the config before the main case selector (option) +* Rewrite and formatting of the usage options +* Removed the version information code as this is always printed ### Version 4.6 - - eXtremeSHOK.com Maintenance - - Code Refactoring - - Removed custom config forced to use the same filename as the default config - - Change file checks from exists to exists and is readable - - Removed legacy config checks - - Full support for custom config files for all tasks - - Removed function: no_default_config + +* eXtremeSHOK.com Maintenance +* Code Refactoring +* Removed custom config forced to use the same filename as the default config +* Change file checks from exists to exists and is readable +* Removed legacy config checks +* Full support for custom config files for all tasks +* Removed function: no_default_config ### Version 4.5.3 - - eXtremeSHOK.com Maintenance - - badmacro.ndb rule support for sanesecurity - - Sanesecurity_sigtest.yara rule support for sanesecurity - - Sanesecurity_spam.yara rule support for sanesecurity - - Changed required_config_version to minimum_required_config_version - - Script now supports a minimum config version to allow for out of sync config and script versions + +* eXtremeSHOK.com Maintenance +* badmacro.ndb rule support for sanesecurity +* Sanesecurity_sigtest.yara rule support for sanesecurity +* Sanesecurity_spam.yara rule support for sanesecurity +* Changed required_config_version to minimum_required_config_version +* Script now supports a minimum config version to allow for out of sync config and script versions ### Version 4.5.2 - - eXtremeSHOK.com Maintenance - - hackingteam.hsb rule support for sanesecurity + +* eXtremeSHOK.com Maintenance +* hackingteam.hsb rule support for sanesecurity ### Version 4.5.1 - - eXtremeSHOK.com Maintenance - - Beta YARA rule support for sanesecurity - - Config updated to 4.8 due to changes - - Bugfix "securiteinfo_enabled" should be "$securiteinfo_enabled" + +* eXtremeSHOK.com Maintenance +* Beta YARA rule support for sanesecurity +* Config updated to 4.8 due to changes +* Bugfix "securiteinfo_enabled" should be "$securiteinfo_enabled" ### Version 4.5.0 - - eXtremeSHOK.com Maintenance - - Initial YARA rule support for sanesecurity - - Added Yara-Rules project Database - - Added config option to quickly enable/disable an entire database - - Config updated to 4.7 due to changes - - Note: Yara rules require clamav 0.99+ - - Bugfix removed unused linuxmalwaredetect_authorisation_signature varible from script + +* eXtremeSHOK.com Maintenance +* Initial YARA rule support for sanesecurity +* Added Yara-Rules project Database +* Added config option to quickly enable/disable an entire database +* Config updated to 4.7 due to changes +* Note: Yara rules require clamav 0.99+ +* Bugfix removed unused linuxmalwaredetect_authorisation_signature varible from script ### Version 4.4.5 - - eXtremeSHOK.com Maintenance - - Updated SecuriteInfo setup instructions + +* eXtremeSHOK.com Maintenance +* Updated SecuriteInfo setup instructions ### Version 4.4.4 - - eXtremeSHOK.com Maintenance - - Committed patch-1 by SecuriteInfo (clean up of SecuriteInfo databases) - - Fixed double $surl_insecure + +* eXtremeSHOK.com Maintenance +* Committed patch-1 by SecuriteInfo (clean up of SecuriteInfo databases) +* Fixed double $surl_insecure ### Version 4.4.3 - - eXtremeSHOK.com Maintenance - - Bugfix for SecuriteInfo not downloading by Colin Waring - - Default will now silence ssl errors caused by ssl certificate errors - - Config updated to 4.6 due to new varible: silence_ssl + +* eXtremeSHOK.com Maintenance +* Bugfix for SecuriteInfo not downloading by Colin Waring +* Default will now silence ssl errors caused by ssl certificate errors +* Config updated to 4.6 due to new varible: silence_ssl ### Version 4.4.2 - - eXtremeSHOK.com Maintenance - - Improved config error checking - - Config updated to 4.5, due to invalid default dbs-si value - - Fix debug varible being present - - Bug fix for ubuntu 14.04 with sed being aliased - - Explicitly set bash as the shell + +* eXtremeSHOK.com Maintenance +* Improved config error checking +* Config updated to 4.5, due to invalid default dbs-si value +* Fix debug varible being present +* Bug fix for ubuntu 14.04 with sed being aliased +* Explicitly set bash as the shell ### Version 4.4.1 - - eXtremeSHOK.com Maintenance - - Added error checking to detect if the config could be broken. + +* eXtremeSHOK.com Maintenance +* Added error checking to detect if the config could be broken. ### Version 4.4.0 - - eXtremeSHOK.com Maintenance - - Code refactoring: - - Added full support for Linux Malware Detect clamav databases - - Config updated to 4.4 + +* eXtremeSHOK.com Maintenance +* Code refactoring: +* Added full support for Linux Malware Detect clamav databases +* Config updated to 4.4 ### Version 4.3.0 - - eXtremeSHOK.com Maintenance - - Code refactoring: group and move functions to top of script - - Complete rewrite of securiteinfo support, full support for Free/Delayed clamav by securiteinfo.com ;-P + +* eXtremeSHOK.com Maintenance +* Code refactoring: group and move functions to top of script +* Complete rewrite of securiteinfo support, full support for Free/Delayed clamav by securiteinfo.com ;-P Note: securite info requires you to create a free account and add your authorisation code to the config. - - Config updated to 4.3 - - Restructured Config +* Config updated to 4.3 +* Restructured Config ### Version 4.2.0 - - eXtremeSHOK.com Maintenance - - Replace annoying si_ , mbl_, ss_ with actual names ie. securiteinfo_ malwarepatrol_ sanesecurity_ - - Complete rewrite of malwarepatrol support, full support for Free/Delayed clamav ;-P + +* eXtremeSHOK.com Maintenance +* Replace annoying si_, mbl_, ss_, with actual names ie. securiteinfo_, malwarepatrol_, sanesecurity_ +* Complete rewrite of malwarepatrol support, full support for Free/Delayed clamav ;-P Note: malware patrol requires you to create a free account and add your "purchase" code to the config. - - More fixes to config prasing and stripping of comments and whitespace - - Code refactoring: remove empty commands: echo "" and comment "" - - Config version detection and enforcing +* More fixes to config prasing and stripping of comments and whitespace +* Code refactoring: remove empty commands: echo "" and comment "" +* Config version detection and enforcing ### Version 4.1.0 - - eXtremeSHOK.com Maintenance - - Fix on default enable of foxhole medium and High false positive sources - - grammatical corrections to some comments and log output - - sig-boundary patch by Alan Stern - - create intermediate monitor-ign-old.txt to prevent reading and writing of local.ign by Alan Stern + +* eXtremeSHOK.com Maintenance +* Fix on default enable of foxhole medium and High false positive sources +* grammatical corrections to some comments and log output +* sig-boundary patch by Alan Stern +* create intermediate monitor-ign-old.txt to prevent reading and writing of local.ign by Alan Stern ### Version 4.0.0 (Released 9 May 2015) - - eXtremeSHOK.com Maintenance - - Enabled all low false positive sources by default - - Added all Sanesecurity database files - - Disabled all med/high false positive sources by default - - Set default configs to work out of the box on a centos system - - Silence cron job - - Set correct paths throughout the script - - Updated Installation Instructions - - Updated Paths for removal - - Updated Default locations to reflect installation instructions - - Fix: correctly remove comments and blanklines from config before eval - - Remove: invalid config values (eg. EXPORT path) - - Fix: correctly check if rsync was successful - -## Script updates can be found at: -### https://github.com/extremeshok/clamav-unofficial-sigs + +* eXtremeSHOK.com Maintenance +* Enabled all low false positive sources by default +* Added all Sanesecurity database files +* Disabled all med/high false positive sources by default +* Set default configs to work out of the box on a centos system +* Silence cron job +* Set correct paths throughout the script +* Updated Installation Instructions +* Updated Paths for removal +* Updated Default locations to reflect installation instructions +* Fix: correctly remove comments and blanklines from config before eval +* Remove: invalid config values (eg. EXPORT path) +* Fix: correctly check if rsync was successful + +## Script updates can be found at + +### From 638e2db460b0697edaf919946956b2de3315a1d1 Mon Sep 17 00:00:00 2001 From: Daniel Nowacki Date: Sun, 6 Dec 2020 00:51:11 -0500 Subject: [PATCH 070/101] perform spell check and md linting --- INSTALL.md | 78 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 7578a673..6055cfb0 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,72 +1,90 @@ -# GENERAL INFORMATION +# clamav-unofficial-sigs.sh install + +## GENERAL INFORMATION + This is property of eXtremeSHOK.com You are free to use, modify and distribute, however you may not remove this notice. Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com License: BSD (Berkeley Software Distribution) -Script updates can be found at: https://github.com/extremeshok/clamav-unofficial-sigs +Script updates can be found at: -# Operating System Specific Install Guides -* CentOS : https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/guides/centos7.md -* Ubuntu : https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/guides/ubuntu-debian.md -* Debian : https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/guides/ubuntu-debian.md -* Mac OSX : https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/guides/macosx.md -* pFsense : https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/guides/pfsense.md +## Operating System Specific Install Guides -# GENERIC UPGRADE INSTRUCTIONS (version 7.0 +) -``` +* CentOS : +* Ubuntu : +* Debian : +* Mac OSX : +* pFsense : + +## GENERIC UPGRADE INSTRUCTIONS (version 7.0 +) + +```bash clamav-unofficial-sigs.sh --upgrade clamav-unofficial-sigs.sh --force ``` -# GENERIC UPGRADE INSTRUCTIONS (version 6.1 and below) -``` +## GENERIC UPGRADE INSTRUCTIONS (version 6.1 and below) + +```bash wget https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/clamav-unofficial-sigs.sh -O /usr/local/sbin/clamav-unofficial-sigs.sh && chmod 755 /usr/local/sbin/clamav-unofficial-sigs.sh wget https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/master.conf -O /etc/clamav-unofficial-sigs/master.conf clamav-unofficial-sigs.sh --force ``` -# GENERIC INSTALLATION INSTRUCTIONS +## GENERIC INSTALLATION INSTRUCTIONS + +### Install -## Install Run the following commands in shell (console/terminal) -``` + +```bash mkdir -p /usr/local/sbin/ wget https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/clamav-unofficial-sigs.sh -O /usr/local/sbin/clamav-unofficial-sigs.sh && chmod 755 /usr/local/sbin/clamav-unofficial-sigs.sh mkdir -p /etc/clamav-unofficial-sigs/ wget https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/master.conf -O /etc/clamav-unofficial-sigs/master.conf wget https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/user.conf -O /etc/clamav-unofficial-sigs/user.conf ``` -Select your operating system config from https://github.com/extremeshok/clamav-unofficial-sigs/tree/master/config/ -**replace os.ubuntu.conf with your required config, centos7/8 = os.centos.conf , debian9/10 = os.debian.conf ** -``` + +Select your operating system config from +**replace os.ubuntu.conf with your required config, centos7/8 = os.centos.conf , debian9/10 = os.debian.conf** + +```bash os_conf="os.ubuntu.conf" wget "https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/os/${os_conf}" -O /etc/clamav-unofficial-sigs/os.conf ``` -### Optional: configure your user config /etc/clamav-unofficial-sigs/user.conf +#### Optional: configure your user config /etc/clamav-unofficial-sigs/user.conf + +### RUN THE SCRIPT ONCE AS ROOT -## RUN THE SCRIPT ONCE AS ROOT ensure there are no errors, fix any missing dependencies script must run once as your superuser to set all the permissions and create the relevant directories -``` + +```bash /usr/local/sbin/clamav-unofficial-sigs.sh --force ``` -### Install logrotate and Man files -``` +#### Install logrotate and man files + +```bash /usr/local/sbin/clamav-unofficial-sigs.sh --install-logrotate /usr/local/sbin/clamav-unofficial-sigs.sh --install-man ``` -### Install Systemd configs or use cron -#### cron -``` +#### Install Systemd configs or use cron + +##### cron + +```bash /usr/local/sbin/clamav-unofficial-sigs.sh --install-cron ``` -### OR -#### systemd -``` + +##### OR + +##### Systemd + +```bash mkdir -p /etc/systemd/system/ wget https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/systemd/clamav-unofficial-sigs.service -O /etc/systemd/system/clamav-unofficial-sigs.service wget https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/systemd/clamav-unofficial-sigs.timer -O /etc/systemd/system/clamav-unofficial-sigs.timer @@ -76,4 +94,4 @@ systemctl enable clamav-unofficial-sigs.timer systemctl start clamav-unofficial-sigs.timer ``` -### Script updates can be found at: https://github.com/extremeshok/clamav-unofficial-sigs +## Script updates can be found at: From 8154523b46ea73ea8099315d65efade6c1d67f62 Mon Sep 17 00:00:00 2001 From: Daniel Nowacki Date: Sun, 6 Dec 2020 01:13:19 -0500 Subject: [PATCH 071/101] add markdownlint to travis --- .travis.yml | 120 ++++++++++++++++++++++++++++------------------------ 1 file changed, 65 insertions(+), 55 deletions(-) diff --git a/.travis.yml b/.travis.yml index f834ed48..5b43921e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,58 +4,68 @@ # Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com # License: BSD (Berkeley Software Distribution) ################## -language: shell -os: linux - -# Required travis ci environment variables -#ci_malwareexpert_serial_key=[secure] -#ci_securiteinfo_authorisation_signature=[secure] -#ci_malwarepatrol_receipt_code=[secure] -#ci_malwarepatrol_product_code=[secure] -#ci_malwarepatrol_list=[secure] -#ci_malwarepatrol_free=[secure] -#ci_codeclimate_repo_token=[secure] - -#Enable modern build platform, not container based. -dist: bionic - -# Apparently caching is disabled, since we used modern build platform as we require clamav and ;/ -# caching the /var/lib/clamav/ will save +/-9mins of run time and allow the use of freshclam signatures - -#Enable Caching -#cache: -# directories: -# - tmp/cache - -before_install: - - sudo apt-get update -qq - - sudo apt-get install -y ca-certificates curl wget rsync p7zip-full shellcheck -qq - #- pip install bashate - # - "mkdir -p tmp/cache/" - # - "ls -laFh tmp/cache/clamav-dbs" - # - "rsync -rlptDv tmp/cache/ /var/lib/clamav/" - # - "freshclam" - # - "ls -laFh /var/lib/clamav/" - # - "rsync -rlptDav /var/lib/clamav/ tmp/cache/" - # - "chown -R travis:travis tmp/cache" - # - "chmod -R 777 tmp/cache/" - # - "ls -laFh tmp/cache/clamav-dbs/" - -install: - - sudo mkdir -p /etc/clamav-unofficial-sigs - - sudo cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf - - sudo cp -f config/os/os.ubuntu.conf /etc/clamav-unofficial-sigs/os.conf - - sudo cp -f clamav-unofficial-sigs.sh /usr/sbin/clamav-unofficial-sigs - -script: - - sudo sh -e .t/ci-shellcheck.sh - - sudo sh -e .t/ci-clamav-download-default-databases.sh - #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh - - sudo sh -e .t/ci-clamav-install-default.sh - - sudo sh -e .t/ci-clamav-install-default-database.sh - - sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf - - sudo sh -e .t/ci-test.sh - -addons: - code_climate: - repo_token: $ci_codeclimate_repo_token +matrix: + include: + - language: node_js + node_js: + - '6' + install: + - npm install -g markdownlint-cli + script: + - markdownlint *.md + + - language: shell + os: linux + + # Required travis ci environment variables + #ci_malwareexpert_serial_key=[secure] + #ci_securiteinfo_authorisation_signature=[secure] + #ci_malwarepatrol_receipt_code=[secure] + #ci_malwarepatrol_product_code=[secure] + #ci_malwarepatrol_list=[secure] + #ci_malwarepatrol_free=[secure] + #ci_codeclimate_repo_token=[secure] + + #Enable modern build platform, not container based. + dist: bionic + + # Apparently caching is disabled, since we used modern build platform as we require clamav and ;/ + # caching the /var/lib/clamav/ will save +/-9mins of run time and allow the use of freshclam signatures + + #Enable Caching + #cache: + # directories: + # - tmp/cache + + before_install: + - sudo apt-get update -qq + - sudo apt-get install -y ca-certificates curl wget rsync p7zip-full shellcheck -qq + #- pip install bashate + # - "mkdir -p tmp/cache/" + # - "ls -laFh tmp/cache/clamav-dbs" + # - "rsync -rlptDv tmp/cache/ /var/lib/clamav/" + # - "freshclam" + # - "ls -laFh /var/lib/clamav/" + # - "rsync -rlptDav /var/lib/clamav/ tmp/cache/" + # - "chown -R travis:travis tmp/cache" + # - "chmod -R 777 tmp/cache/" + # - "ls -laFh tmp/cache/clamav-dbs/" + + install: + - sudo mkdir -p /etc/clamav-unofficial-sigs + - sudo cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf + - sudo cp -f config/os/os.ubuntu.conf /etc/clamav-unofficial-sigs/os.conf + - sudo cp -f clamav-unofficial-sigs.sh /usr/sbin/clamav-unofficial-sigs + + script: + - sudo sh -e .t/ci-shellcheck.sh + - sudo sh -e .t/ci-clamav-download-default-databases.sh + #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh + - sudo sh -e .t/ci-clamav-install-default.sh + - sudo sh -e .t/ci-clamav-install-default-database.sh + - sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf + - sudo sh -e .t/ci-test.sh + + addons: + code_climate: + repo_token: $ci_codeclimate_repo_token From 8cded4108f0cd6af63a5e65860b2caf8d48a5bc4 Mon Sep 17 00:00:00 2001 From: Daniel Nowacki Date: Sun, 6 Dec 2020 01:26:31 -0500 Subject: [PATCH 072/101] use bionic build for markdownlint --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5b43921e..b9acb4a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ matrix: include: - language: node_js + dist: + - 'bionic' node_js: - '6' install: From 10abaa0634157fa1f0c903c301e08e5eedb534a3 Mon Sep 17 00:00:00 2001 From: Daniel Nowacki Date: Sun, 6 Dec 2020 01:30:25 -0500 Subject: [PATCH 073/101] update node to ver. 8 for markdownlint --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b9acb4a7..2682c6fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,8 @@ matrix: include: - language: node_js - dist: - - 'bionic' node_js: - - '6' + - '8' install: - npm install -g markdownlint-cli script: From 3944c2ebf7b70ccf16d058e3d123b896373cbf16 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 13:29:55 +0200 Subject: [PATCH 074/101] travis: extra checks to ensure malwareexpert_serial_key is defined before failing --- .t/ci-test.sh | 58 +++++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index 0bca2379..b05a1c0f 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -172,31 +172,35 @@ else echo .. ERROR exit 1 fi -echo "Was /var/lib/clamav/malware.expert.hdb added ?" -if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then - echo .. OK -else - echo .. ERROR - exit 1 -fi -echo "Was /var/lib/clamav/malware.expert.fp added ?" -if [ -e "/var/lib/clamav/malware.expert.fp" ] ; then - echo .. OK -else - echo .. ERROR - exit 1 -fi -echo "Was /var/lib/clamav/malware.expert.ldb added ?" -if [ -e "/var/lib/clamav/malware.expert.ldb" ] ; then - echo .. OK -else - echo .. ERROR - exit 1 -fi -echo "Was /var/lib/clamav/malware.expert.ndb added ?" -if [ -e "/var/lib/clamav/malware.expert.ndb" ] ; then - echo .. OK -else - echo .. ERROR - exit 1 + +if [ "$malwareexpert_serial_key" != "" ] && [ -n "$ci_malwareexpert_serial_key" ]; then + + echo "Was /var/lib/clamav/malware.expert.hdb added ?" + if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then + echo .. OK + else + echo .. ERROR + exit 1 + fi + echo "Was /var/lib/clamav/malware.expert.fp added ?" + if [ -e "/var/lib/clamav/malware.expert.fp" ] ; then + echo .. OK + else + echo .. ERROR + exit 1 + fi + echo "Was /var/lib/clamav/malware.expert.ldb added ?" + if [ -e "/var/lib/clamav/malware.expert.ldb" ] ; then + echo .. OK + else + echo .. ERROR + exit 1 + fi + echo "Was /var/lib/clamav/malware.expert.ndb added ?" + if [ -e "/var/lib/clamav/malware.expert.ndb" ] ; then + echo .. OK + else + echo .. ERROR + exit 1 + fi fi From f3b6ceca756ccaddcb305e1c21126086b31a8a00 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 14:01:14 +0200 Subject: [PATCH 075/101] -z not -n --- .t/ci-test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index b05a1c0f..e82f11dd 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -173,8 +173,7 @@ else exit 1 fi -if [ "$malwareexpert_serial_key" != "" ] && [ -n "$ci_malwareexpert_serial_key" ]; then - +if [ -z "$ci_malwareexpert_serial_key" ]; then echo "Was /var/lib/clamav/malware.expert.hdb added ?" if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then echo .. OK From c29b3d22a34d2d423482add586cbbb16389cb04d Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 14:47:30 +0200 Subject: [PATCH 076/101] fixes for markdown linting --- .travis.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2682c6fa..1088d1c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,15 +7,23 @@ matrix: include: - language: node_js - node_js: - - '8' + os: linux + #Enable modern build platform, not container based. + dist: bionic +# node_js: +# - '8' + before_install: + - nvm install 14.6.0 + - nvm use 14.6.0 install: - - npm install -g markdownlint-cli + - npm install --global markdownlint-cli script: - markdownlint *.md - + - language: shell os: linux + #Enable modern build platform, not container based. + dist: bionic # Required travis ci environment variables #ci_malwareexpert_serial_key=[secure] @@ -26,8 +34,6 @@ matrix: #ci_malwarepatrol_free=[secure] #ci_codeclimate_repo_token=[secure] - #Enable modern build platform, not container based. - dist: bionic # Apparently caching is disabled, since we used modern build platform as we require clamav and ;/ # caching the /var/lib/clamav/ will save +/-9mins of run time and allow the use of freshclam signatures From 528369c97ced119d0a18f655ffdc0784af727bd8 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 14:58:45 +0200 Subject: [PATCH 077/101] config for markdown lint --- .markdownlint.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .markdownlint.json diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 00000000..9264fdbd --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,24 @@ +{ + "first-header-h1": false, + "first-line-h1": false, + "line-length": { + "code_blocks": false + }, + "no-hard-tabs": { + "code_blocks": false + }, + "no-inline-html": { + "allowed_elements": [ + "br", + "figcaption", + "figure", + "p", + "span", + "ruby", + "rt" + ] + }, + "no-trailing-punctuation": { + "punctuation": ".,;:!" + } +} From cf0ba015c7790d7cc2878c866c0fc97b2084a09c Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 15:10:35 +0200 Subject: [PATCH 078/101] suppress line length --- .markdownlint.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.markdownlint.json b/.markdownlint.json index 9264fdbd..5c0737d5 100644 --- a/.markdownlint.json +++ b/.markdownlint.json @@ -1,7 +1,8 @@ { - "first-header-h1": false, - "first-line-h1": false, + /* "first-header-h1": false, + "first-line-h1": false, */ "line-length": { + "strict": false, "code_blocks": false }, "no-hard-tabs": { From 0195408541b8ea8b4cbbf9f93473fa132fa8befc Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 15:20:55 +0200 Subject: [PATCH 079/101] second attempt at markdown config to ignore MD013 --- .markdownlint.json | 47 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/.markdownlint.json b/.markdownlint.json index 5c0737d5..914025be 100644 --- a/.markdownlint.json +++ b/.markdownlint.json @@ -1,25 +1,26 @@ { - /* "first-header-h1": false, - "first-line-h1": false, */ - "line-length": { - "strict": false, - "code_blocks": false - }, - "no-hard-tabs": { - "code_blocks": false - }, - "no-inline-html": { - "allowed_elements": [ - "br", - "figcaption", - "figure", - "p", - "span", - "ruby", - "rt" - ] - }, - "no-trailing-punctuation": { - "punctuation": ".,;:!" - } + "MD010": { + "code_blocks": false + }, + "MD013": false, + "line-length": { + "code_blocks": false + }, + "no-hard-tabs": { + "code_blocks": false + }, + "no-inline-html": { + "allowed_elements": [ + "br", + "figcaption", + "figure", + "p", + "span", + "ruby", + "rt" + ] + }, + "no-trailing-punctuation": { + "punctuation": ".,;:!" + } } From b518d3d9dc55316cbb667d999eb87e2e6e8a9c94 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 15:30:20 +0200 Subject: [PATCH 080/101] parallel shellcheck and another attempt to silence md013 --- .markdownlint.json | 25 +------------------------ .travis.yml | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 32 deletions(-) diff --git a/.markdownlint.json b/.markdownlint.json index 914025be..1344b312 100644 --- a/.markdownlint.json +++ b/.markdownlint.json @@ -1,26 +1,3 @@ { - "MD010": { - "code_blocks": false - }, - "MD013": false, - "line-length": { - "code_blocks": false - }, - "no-hard-tabs": { - "code_blocks": false - }, - "no-inline-html": { - "allowed_elements": [ - "br", - "figcaption", - "figure", - "p", - "span", - "ruby", - "rt" - ] - }, - "no-trailing-punctuation": { - "punctuation": ".,;:!" - } + "MD013": false } diff --git a/.travis.yml b/.travis.yml index 1088d1c3..25c44e6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,24 +2,33 @@ # This is property of eXtremeSHOK.com # You are free to use, modify and distribute, however you may not remove this notice. # Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com -# License: BSD (Berkeley Software Distribution) +# License: BSD (Berkeley Software Distribution) ################## matrix: include: +#### MARKDOWN-LINT - language: node_js os: linux #Enable modern build platform, not container based. dist: bionic -# node_js: -# - '8' before_install: - nvm install 14.6.0 - nvm use 14.6.0 install: - npm install --global markdownlint-cli script: + - cat .markdownlint.json - markdownlint *.md - +#### SHELLCHECK + - language: shell + os: linux + #Enable modern build platform, not container based. + dist: bionic + install: + - apt-get install -y shellcheck -qq + script: + - sudo sh -e .t/ci-shellcheck.sh +#### LOGIC AND CLAMAV TESTING - language: shell os: linux #Enable modern build platform, not container based. @@ -44,8 +53,8 @@ matrix: # - tmp/cache before_install: - - sudo apt-get update -qq - - sudo apt-get install -y ca-certificates curl wget rsync p7zip-full shellcheck -qq + - apt-get update -qq + - apt-get install -y ca-certificates curl wget rsync p7zip-full -qq #- pip install bashate # - "mkdir -p tmp/cache/" # - "ls -laFh tmp/cache/clamav-dbs" @@ -64,7 +73,6 @@ matrix: - sudo cp -f clamav-unofficial-sigs.sh /usr/sbin/clamav-unofficial-sigs script: - - sudo sh -e .t/ci-shellcheck.sh - sudo sh -e .t/ci-clamav-download-default-databases.sh #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh - sudo sh -e .t/ci-clamav-install-default.sh @@ -74,4 +82,4 @@ matrix: addons: code_climate: - repo_token: $ci_codeclimate_repo_token + repo_token: $ci_codeclimate_repo_token From f1cd320b1dd0f5984555188412472217124b5f00 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 15:36:57 +0200 Subject: [PATCH 081/101] upgrade travis to focal ? --- .travis.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25c44e6a..9c011606 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,29 +10,28 @@ matrix: - language: node_js os: linux #Enable modern build platform, not container based. - dist: bionic - before_install: - - nvm install 14.6.0 - - nvm use 14.6.0 + dist: focal + # before_install: + # - nvm install 14.6.0 + # - nvm use 14.6.0 install: - - npm install --global markdownlint-cli + - sudo npm install --global markdownlint-cli script: - - cat .markdownlint.json - markdownlint *.md #### SHELLCHECK - language: shell os: linux #Enable modern build platform, not container based. - dist: bionic + dist: focal install: - - apt-get install -y shellcheck -qq + - sudo apt-get install -y shellcheck -qq script: - sudo sh -e .t/ci-shellcheck.sh #### LOGIC AND CLAMAV TESTING - language: shell os: linux #Enable modern build platform, not container based. - dist: bionic + dist: focal # Required travis ci environment variables #ci_malwareexpert_serial_key=[secure] @@ -53,8 +52,8 @@ matrix: # - tmp/cache before_install: - - apt-get update -qq - - apt-get install -y ca-certificates curl wget rsync p7zip-full -qq + - sudo apt-get update -qq + - sudo apt-get install -y ca-certificates curl wget rsync p7zip-full -qq #- pip install bashate # - "mkdir -p tmp/cache/" # - "ls -laFh tmp/cache/clamav-dbs" From 048b05c5a2c1ffbab02f15ddb8146d9caffa59d3 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 15:47:01 +0200 Subject: [PATCH 082/101] begin initial tests for macos / osx --- .travis.yml | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c011606..5419b6ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,11 +11,11 @@ matrix: os: linux #Enable modern build platform, not container based. dist: focal - # before_install: - # - nvm install 14.6.0 - # - nvm use 14.6.0 + before_install: + - nvm install 14.6.0 + - nvm use 14.6.0 install: - - sudo npm install --global markdownlint-cli + - npm install --global markdownlint-cli script: - markdownlint *.md #### SHELLCHECK @@ -27,7 +27,7 @@ matrix: - sudo apt-get install -y shellcheck -qq script: - sudo sh -e .t/ci-shellcheck.sh -#### LOGIC AND CLAMAV TESTING +#### LOGIC AND CLAMAV TESTING : UBUNTU LINUX - language: shell os: linux #Enable modern build platform, not container based. @@ -42,15 +42,6 @@ matrix: #ci_malwarepatrol_free=[secure] #ci_codeclimate_repo_token=[secure] - - # Apparently caching is disabled, since we used modern build platform as we require clamav and ;/ - # caching the /var/lib/clamav/ will save +/-9mins of run time and allow the use of freshclam signatures - - #Enable Caching - #cache: - # directories: - # - tmp/cache - before_install: - sudo apt-get update -qq - sudo apt-get install -y ca-certificates curl wget rsync p7zip-full -qq @@ -82,3 +73,18 @@ matrix: addons: code_climate: repo_token: $ci_codeclimate_repo_token +#### LOGIC AND CLAMAV TESTING : MACOS / OSX + - language: shell + os: osx + #Enable modern build platform, not container based. + dist: focal + addons: + homebrew: + packages: + - gtar clamav + install: + - mkdir -p /usr/local/bin + - mkdir -p /etc/clamav-unofficial-sigs + - cp -f clamav-unofficial-sigs.sh /usr/local/bin/clamav-unofficial-sigs.sh + - cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf + - cp -f config/os/os.macos.conf /etc/clamav-unofficial-sigs/os.conf From b9664c837c4878d52b92d4880fc009c820a62fb6 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 16:03:07 +0200 Subject: [PATCH 083/101] more travis macos enhancements --- .t/ci-test.sh | 56 ++++++++++++++++++++++++------------------------- .travis.yml | 21 ++++++++++++++----- guides/macos.md | 4 ++-- 3 files changed, 45 insertions(+), 36 deletions(-) diff --git a/.t/ci-test.sh b/.t/ci-test.sh index e82f11dd..bc777004 100644 --- a/.t/ci-test.sh +++ b/.t/ci-test.sh @@ -173,33 +173,31 @@ else exit 1 fi -if [ -z "$ci_malwareexpert_serial_key" ]; then - echo "Was /var/lib/clamav/malware.expert.hdb added ?" - if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then - echo .. OK - else - echo .. ERROR - exit 1 - fi - echo "Was /var/lib/clamav/malware.expert.fp added ?" - if [ -e "/var/lib/clamav/malware.expert.fp" ] ; then - echo .. OK - else - echo .. ERROR - exit 1 - fi - echo "Was /var/lib/clamav/malware.expert.ldb added ?" - if [ -e "/var/lib/clamav/malware.expert.ldb" ] ; then - echo .. OK - else - echo .. ERROR - exit 1 - fi - echo "Was /var/lib/clamav/malware.expert.ndb added ?" - if [ -e "/var/lib/clamav/malware.expert.ndb" ] ; then - echo .. OK - else - echo .. ERROR - exit 1 - fi +echo "Was /var/lib/clamav/malware.expert.hdb added ?" +if [ -e "/var/lib/clamav/malware.expert.hdb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "Was /var/lib/clamav/malware.expert.fp added ?" +if [ -e "/var/lib/clamav/malware.expert.fp" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "Was /var/lib/clamav/malware.expert.ldb added ?" +if [ -e "/var/lib/clamav/malware.expert.ldb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "Was /var/lib/clamav/malware.expert.ndb added ?" +if [ -e "/var/lib/clamav/malware.expert.ndb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 fi diff --git a/.travis.yml b/.travis.yml index 5419b6ba..ea426d9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -83,8 +83,19 @@ matrix: packages: - gtar clamav install: - - mkdir -p /usr/local/bin - - mkdir -p /etc/clamav-unofficial-sigs - - cp -f clamav-unofficial-sigs.sh /usr/local/bin/clamav-unofficial-sigs.sh - - cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf - - cp -f config/os/os.macos.conf /etc/clamav-unofficial-sigs/os.conf + - sudo mkdir -p /usr/local/bin + - sudo mkdir -p /etc/clamav-unofficial-sigs + - sudo cp -f clamav-unofficial-sigs.sh /usr/local/bin/clamav-unofficial-sigs.sh + - sudo chmod 755 /usr/local/bin/clamav-unofficial-sigs.sh + - sudo cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf + - sudo cp -f config/os/os.macos.conf /etc/clamav-unofficial-sigs/os.conf + + script: + - sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf + - sudo clamav-unofficial-sigs.sh --force + #- sudo sh -e .t/ci-clamav-download-default-databases.sh + #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh + #- sudo sh -e .t/ci-clamav-install-default.sh + #- sudo sh -e .t/ci-clamav-install-default-database.sh + #- sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf + #- sudo sh -e .t/ci-test.sh diff --git a/guides/macos.md b/guides/macos.md index a5a770a9..f871c24e 100644 --- a/guides/macos.md +++ b/guides/macos.md @@ -26,9 +26,9 @@ https://www.clamav.net/documents/installation-on-macos-mac-os-x ``` -# Step 2 Install gtar (gnu tar) +# Step 2 Install gtar (gnu-tar) ``` -brew install gtar +brew install gnu-tar ``` # Step 3 Install clamav From 27b281bebed99de812f69a2c2acd1dd0d7d3520c Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 16:26:26 +0200 Subject: [PATCH 084/101] detect sed command --- .travis.yml | 4 +- clamav-unofficial-sigs.sh | 95 ++++++++++++++++++++++----------------- 2 files changed, 55 insertions(+), 44 deletions(-) diff --git a/.travis.yml b/.travis.yml index ea426d9b..55974862 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,7 +81,7 @@ matrix: addons: homebrew: packages: - - gtar clamav + - gnu-tar gnu-sed clamav install: - sudo mkdir -p /usr/local/bin - sudo mkdir -p /etc/clamav-unofficial-sigs @@ -92,7 +92,7 @@ matrix: script: - sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf - - sudo clamav-unofficial-sigs.sh --force + - sudo /usr/local/bin/clamav-unofficial-sigs.sh --force #- sudo sh -e .t/ci-clamav-download-default-databases.sh #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh #- sudo sh -e .t/ci-clamav-install-default.sh diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index fda3fa84..96861c3c 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -1012,7 +1012,7 @@ function make_signature_database_from_ascii_file() { target type is used and full file scanning is enabled (see ClamAV signatures.pdf for details). - Line numbering will be done automatically by the script. - " | command sed 's/^ //g' + " | command $sed_bin 's/^ //g' echo -n "Do you wish to continue? " if xshok_prompt_confirm ; then @@ -1026,7 +1026,7 @@ function make_signature_database_from_ascii_file() { echo -n "Enter signature prefix: " read -r prefix - path_file="$(echo "$source" | cut -d "." -f -1 | command sed 's/$/.ndb/')" + path_file="$(echo "$source" | cut -d "." -f -1 | command $sed_bin 's/$/.ndb/')" db_file="$(basename "$path_file")" rm -f "$path_file" total="$(wc -l "$source" | cut -d " " -f 1)" @@ -1035,11 +1035,11 @@ function make_signature_database_from_ascii_file() { while read -r line ; do line_prefix="$(echo "$line" | awk -F ":" '{print $1}')" if [ "$line_prefix" == "-" ] ; then - echo "$line" | cut -d ":" -f 2- | perl -pe 's/(.)/sprintf("%02lx", ord $1)/eg' | command sed "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" + echo "$line" | cut -d ":" -f 2- | perl -pe 's/(.)/sprintf("%02lx", ord $1)/eg' | command $sed_bin "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" elif [ "$line_prefix" == "=" ] ; then - echo "$line" | cut -d ":" -f 2- | perl -pe 's/(\{[^}]*\}|\([^)]*\)|\*)|(.)/defined $1 ? $1 : sprintf("%02lx", ord $2)/eg' | command sed "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" + echo "$line" | cut -d ":" -f 2- | perl -pe 's/(\{[^}]*\}|\([^)]*\)|\*)|(.)/defined $1 ? $1 : sprintf("%02lx", ord $2)/eg' | command $sed_bin "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" else - echo "$line" | perl -pe 's/(.)/sprintf("%02lx", ord $1)/eg' | command sed "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" + echo "$line" | perl -pe 's/(.)/sprintf("%02lx", ord $1)/eg' | command $sed_bin "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" fi xshok_pretty_echo_and_log "Hexadecimal encoding ${source_file} line: ${line_num} of ${total}" line_num="$((line_num + 1))" @@ -1636,8 +1636,18 @@ if [ -x /usr/gnu/bin/grep ] ; then else grep_bin="$(command -v grep 2> /dev/null)" fi -# Detect support for tar or gtar +if [ -z "$grep_bin" ] ; then + xshok_pretty_echo_and_log "ERROR: grep command is missing" + exit 1 + fi +# Detect supprot for sed +sed_bin="$(command -v sed 2> /dev/null)" +if [ -z "$grep_bin" ] ; then + xshok_pretty_echo_and_log "ERROR: sed command is missing" + exit 1 +fi +# Detect support for tar or gtar if [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then tar_executable="gtar" else @@ -1647,6 +1657,7 @@ if [ -z "$tar_bin" ]; then tar_bin="$(command -v "$tar_executable" 2> /dev/null)" else xshok_pretty_echo_and_log "ERROR: gtar (gnu tar) is missing" + exit 1 fi # Detect support for curl if [ -z "$curl_bin" ]; then @@ -1782,28 +1793,28 @@ for config_file in "${config_files[@]}" ; do if [ "$(uname -s)" == "SunOS" ] ; then # Solaris FIXES only, i had issues with running with a single command.. - clean_config="$(command sed -e '/^#.*/d' "$config_file")" # Comment line - #clean_config="$(echo "$clean_config" | sed -e 's/#[[:space:]].*//')" # Comment line (duplicated) + clean_config="$(command $sed_bin -e '/^#.*/d' "$config_file")" # Comment line + #clean_config="$(echo "$clean_config" | $sed_bin -e 's/#[[:space:]].*//')" # Comment line (duplicated) clean_config=${clean_config//\#*/} # Comment line (duplicated) # shellcheck disable=SC2001 - clean_config="$(echo "$clean_config" | sed -e '/^[[:blank:]]*#/d;s/#.*//')" # Comments at end of line - #clean_config="$(echo "$clean_config" | sed -e 's/^[ \t]*//;s/[ \t]*$//')" # trailing and leading whitespace + clean_config="$(echo "$clean_config" | $sed_bin -e '/^[[:blank:]]*#/d;s/#.*//')" # Comments at end of line + #clean_config="$(echo "$clean_config" | $sed_bin -e 's/^[ \t]*//;s/[ \t]*$//')" # trailing and leading whitespace clean_config="$(echo "$clean_config" | xargs)" # shellcheck disable=SC2001 - clean_config="$(echo "$clean_config" | sed -e '/^\s*$/d')" # Blank lines + clean_config="$(echo "$clean_config" | $sed_bin -e '/^\s*$/d')" # Blank lines elif [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then # MacOS / OSX / BSD fixes, had issues with running with a single command and with SunOS work around.. # shellcheck disable=SC2001 - clean_config="$(command sed -e '/^#.*/d' "$config_file")" # Comment line + clean_config="$(command $sed_bin -e '/^#.*/d' "$config_file")" # Comment line # shellcheck disable=SC2001 - clean_config="$(echo "$clean_config" | sed -e 's/#[[:space:]].*//')" # Comment line (duplicated) + clean_config="$(echo "$clean_config" | $sed_bin -e 's/#[[:space:]].*//')" # Comment line (duplicated) # shellcheck disable=SC2001 - clean_config="$(echo "$clean_config" | sed -e '/^[[:blank:]]*#/d;s/#.*//')" # Comments at end of line - #clean_config="$(echo "$clean_config" | sed -e 's/^[ \t]*//;s/[ \t]*$//')" # trailing and leading whitespace + clean_config="$(echo "$clean_config" | $sed_bin -e '/^[[:blank:]]*#/d;s/#.*//')" # Comments at end of line + #clean_config="$(echo "$clean_config" | $sed_bin -e 's/^[ \t]*//;s/[ \t]*$//')" # trailing and leading whitespace #clean_config="$(echo "$clean_config" | xargs)" # shellcheck disable=SC2001 - clean_config="$(echo "$clean_config" | sed -e '/^\s*$/d')" # Blank lines + clean_config="$(echo "$clean_config" | $sed_bin -e '/^\s*$/d')" # Blank lines else # Delete lines beginning with # @@ -1812,7 +1823,7 @@ for config_file in "${config_files[@]}" ; do # Delete both trailing and leading whitespace # Delete all trailing whitespace # Delete all empty lines - clean_config="$(command sed -e '/^#.*/d' -e 's/[[:space:]]#.*//' -e 's/#[[:space:]].*//' -e 's/^[ \t]*//;s/[ \t]*$//' -e '/^\s*$/d' "$config_file")" + clean_config="$(command $sed_bin -e '/^#.*/d' -e 's/[[:space:]]#.*//' -e 's/#[[:space:]].*//' -e 's/^[ \t]*//;s/[ \t]*$//' -e '/^\s*$/d' "$config_file")" fi @@ -1840,7 +1851,7 @@ for config_file in "${config_files[@]}" ; do # Config loading for i in "${clean_config[@]}" ; do - eval "$(echo "${i}" | command sed -e 's/[[:space:]]*$//' 2> /dev/null)" + eval "$(echo "${i}" | command $sed_bin -e 's/[[:space:]]*$//' 2> /dev/null)" done fi done @@ -1887,58 +1898,58 @@ shopt -s extglob; work_dir="${work_dir%%+(/)}" # Allow overriding of all the individual workdirs, this is mainly to aid package maintainers if [ -z "$work_dir_sanesecurity" ] ; then - work_dir_sanesecurity="$(echo "${work_dir}/${sanesecurity_dir}" | sed 's:/*$::')" + work_dir_sanesecurity="$(echo "${work_dir}/${sanesecurity_dir}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_sanesecurity="${work_dir_sanesecurity%%+(/)}" fi if [ -z "$work_dir_securiteinfo" ] ; then - work_dir_securiteinfo="$(echo "${work_dir}/${securiteinfo_dir}" | sed 's:/*$::')" + work_dir_securiteinfo="$(echo "${work_dir}/${securiteinfo_dir}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_securiteinfo="${work_dir_securiteinfo%%+(/)}" fi if [ -z "$work_dir_linuxmalwaredetect" ] ; then - work_dir_linuxmalwaredetect="$(echo "${work_dir}/${linuxmalwaredetect_dir}" | sed 's:/*$::')" + work_dir_linuxmalwaredetect="$(echo "${work_dir}/${linuxmalwaredetect_dir}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_malwarepatrol="${work_dir_malwarepatrol%%+(/)}" fi if [ -z "$work_dir_malwareexpert" ] ; then - work_dir_malwareexpert="$(echo "${work_dir}/${malwareexpert_dir}" | sed 's:/*$::')" + work_dir_malwareexpert="$(echo "${work_dir}/${malwareexpert_dir}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_malwareexpert="${work_dir_malwareexpert%%+(/)}" fi if [ -z "$work_dir_malwarepatrol" ] ; then - work_dir_malwarepatrol="$(echo "${work_dir}/${malwarepatrol_dir}" | sed 's:/*$::')" + work_dir_malwarepatrol="$(echo "${work_dir}/${malwarepatrol_dir}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_malwarepatrol="${work_dir_malwarepatrol%%+(/)}" fi if [ -z "$work_dir_urlhaust" ] ; then - work_dir_urlhaus="$(echo "${work_dir}/${urlhaus_dir}" | sed 's:/*$::')" + work_dir_urlhaus="$(echo "${work_dir}/${urlhaus_dir}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_urlhaus="${work_dir_urlhaus%%+(/)}" fi if [ -z "$work_dir_yararulesproject" ] ; then - work_dir_yararulesproject="$(echo "${work_dir}/${yararulesproject_dir}" | sed 's:/*$::')" + work_dir_yararulesproject="$(echo "${work_dir}/${yararulesproject_dir}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_yararulesproject="${work_dir_yararulesproject%%+(/)}" fi if [ -z "$work_dir_add" ] ; then - work_dir_add="$(echo "${work_dir}/${add_dir}" | sed 's:/*$::')" + work_dir_add="$(echo "${work_dir}/${add_dir}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_add="${work_dir_add%%+(/)}" fi if [ -z "$work_dir_work_configs" ] ; then - work_dir_work_configs="$(echo "${work_dir}/${work_dir_configs}" | sed 's:/*$::')" + work_dir_work_configs="$(echo "${work_dir}/${work_dir_configs}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_work_configs="${work_dir_work_configs%%+(/)}" fi if [ -z "${work_dir_gpg}" ] ; then - work_dir_gpg="$(echo "${work_dir}/${gpg_dir}" | sed 's:/*$::')" + work_dir_gpg="$(echo "${work_dir}/${gpg_dir}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_gpg="${work_dir_gpg%%+(/)}" fi if [ -z "$work_dir_pid" ] ; then - work_dir_pid="$(echo "${work_dir}/${pid_dir}" | sed 's:/*$::')" + work_dir_pid="$(echo "${work_dir}/${pid_dir}" | $sed_bin 's:/*$::')" else shopt -s extglob; work_dir_pid="${work_dir_pid%%+(/)}" fi @@ -2535,7 +2546,7 @@ if [ -n "$ham_dir" ] && [ -d "$work_dir" ] && [ ! -d "$test_dir" ] ; then xshok_mkdir_ownership "$test_dir" cp -f -p "$work_dir"/*/*.ndb "$test_dir" cp -f -p "$work_dir"/*/*.db "$test_dir" - $clamscan_bin --infected --no-summary -d "$test_dir" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' >> "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "$test_dir" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' >> "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/*.ndb" | cut -d "*" -f 2 | sort | uniq > "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/*.db" | cut -d "=" -f 2 | awk '{ printf("=%s\n", $1);}' | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" cd "$test_dir" || exit @@ -2814,21 +2825,21 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then # Add fallback to host if dig returns no records or dig is not used if [ ${#sanesecurity_mirror_ips} -lt 1 ] ; then # shellcheck disable=SC2086 - sanesecurity_mirror_ips="$($host_bin $host_proxy -t A "$sanesecurity_url" | sed -n '/has address/{s/.*address \([^ ]*\).*/\1/;p;}')" + sanesecurity_mirror_ips="$($host_bin $host_proxy -t A "$sanesecurity_url" | $sed_bin -n '/has address/{s/.*address \([^ ]*\).*/\1/;p;}')" fi if [ ${#sanesecurity_mirror_ips} -ge 1 ] ; then for sanesecurity_mirror_ip in $sanesecurity_mirror_ips ; do if [ -n "$dig_bin" ] ; then # shellcheck disable=SC2086 - sanesecurity_mirror_name="$($dig_bin $dig_proxy +short -x "$sanesecurity_mirror_ip" | command sed 's/\.$//')" + sanesecurity_mirror_name="$($dig_bin $dig_proxy +short -x "$sanesecurity_mirror_ip" | command $sed_bin 's/\.$//')" else sanesecurity_mirror_name="" fi # Add fallback to host if dig returns no records or dig is not used if [ -z "$sanesecurity_mirror_name" ] ; then # shellcheck disable=SC2086 - sanesecurity_mirror_name="$($host_bin $host_proxy "$sanesecurity_mirror_ip" | sed -n '/name pointer/{s/.*pointer \([^ ]*\).*\.$/\1/;p;}')" + sanesecurity_mirror_name="$($host_bin $host_proxy "$sanesecurity_mirror_ip" | $sed_bin -n '/name pointer/{s/.*pointer \([^ ]*\).*\.$/\1/;p;}')" fi sanesecurity_mirror_site_info="$sanesecurity_mirror_name $sanesecurity_mirror_ip" xshok_pretty_echo_and_log "Sanesecurity mirror site used: ${sanesecurity_mirror_site_info}" @@ -2883,7 +2894,7 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_sanesecurity}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex-tmp" mv -f "${work_dir_work_configs}/whitelist.hex-tmp" "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" @@ -3024,7 +3035,7 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_securiteinfo}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" @@ -3184,7 +3195,7 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_linuxmalwaredetect}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" @@ -3327,7 +3338,7 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_malwareexpert}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" @@ -3471,7 +3482,7 @@ if [ "$malwarepatrol_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_malwarepatrol}/${malwarepatrol_db}" > "${test_dir}/${malwarepatrol_db}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${malwarepatrol_db}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${malwarepatrol_db}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${malwarepatrol_db}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${malwarepatrol_db}" > "${test_dir}/${malwarepatrol_db}-tmp" mv -f "${test_dir}/${malwarepatrol_db}-tmp" "${test_dir}/${malwarepatrol_db}" @@ -3600,7 +3611,7 @@ if [ "$urlhaus_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_urlhaus}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" @@ -3737,7 +3748,7 @@ if [ "$yararulesproject_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_yararulesproject}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" @@ -3842,7 +3853,7 @@ if [ "$additional_enabled" == "yes" ] ; then # fi #cleanup any leading and trailing whitespace. - db_url="$(echo -e "$db_url" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + db_url="$(echo -e "$db_url" | $sed_bin -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" db_file="$(basename "$db_url")" @@ -3895,7 +3906,7 @@ if [ "$additional_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_add}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command sed 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" if [[ "${work_dir_add}/${db_file}" == *.db ]] ; then $grep_bin -h -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" | cut -d "=" -f 2 | awk '{ printf("=%s\n", $1);}' |sort | uniq >> "${work_dir_work_configs}/whitelist.hex-tmp" mv -f "${work_dir_work_configs}/whitelist.hex-tmp" "${work_dir_work_configs}/whitelist.hex" From 279cad14b3e77312a3f7851e2652e1ac136dbea8 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 16:35:46 +0200 Subject: [PATCH 085/101] more macos fixes --- .travis.yml | 8 ++++---- guides/macos.md | 22 +++++++--------------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55974862..ec94ec04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -84,14 +84,14 @@ matrix: - gnu-tar gnu-sed clamav install: - sudo mkdir -p /usr/local/bin - - sudo mkdir -p /etc/clamav-unofficial-sigs + - sudo mkdir -p /usr/local/etc/clamav-unofficial-sigs - sudo cp -f clamav-unofficial-sigs.sh /usr/local/bin/clamav-unofficial-sigs.sh - sudo chmod 755 /usr/local/bin/clamav-unofficial-sigs.sh - - sudo cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf - - sudo cp -f config/os/os.macos.conf /etc/clamav-unofficial-sigs/os.conf + - sudo cp -f config/master.conf /usr/local/etc/clamav-unofficial-sigs/master.conf + - sudo cp -f config/os/os.macos.conf /usr/local/etc/clamav-unofficial-sigs/os.conf script: - - sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf + - sudo cp -f .t/tests/user.conf /usr/local/etc/clamav-unofficial-sigs/user.conf - sudo /usr/local/bin/clamav-unofficial-sigs.sh --force #- sudo sh -e .t/ci-clamav-download-default-databases.sh #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh diff --git a/guides/macos.md b/guides/macos.md index f871c24e..cc73b3c7 100644 --- a/guides/macos.md +++ b/guides/macos.md @@ -8,14 +8,6 @@ clamav-unofficial-sigs.sh --upgrade clamav-unofficial-sigs.sh --force ``` -# UPGRADE INSTRUCTIONS (version 6.1 and below) -``` -wget https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/clamav-unofficial-sigs.sh -O /usr/local/bin/clamav-unofficial-sigs.sh && chmod 755 /usr/local/sbin/clamav-unofficial-sigs.sh -wget https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/master.conf -O /etc/clamav-unofficial-sigs/master.conf -clamav-unofficial-sigs.sh --force -``` - - ## Notes: https://www.clamav.net/documents/installation-on-macos-mac-os-x @@ -26,9 +18,9 @@ https://www.clamav.net/documents/installation-on-macos-mac-os-x ``` -# Step 2 Install gtar (gnu-tar) +# Step 2 Install dependencies : gtar (gnu-tar) sed (gnu-sed) ``` -brew install gnu-tar +brew install gnu-tar gnu-tar ``` # Step 3 Install clamav @@ -41,17 +33,17 @@ brew install clamav sudo su curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/clamav-unofficial-sigs.sh --output /usr/local/bin/clamav-unofficial-sigs.sh chmod 755 /usr/local/bin/clamav-unofficial-sigs.sh -mkdir -p /etc/clamav-unofficial-sigs -curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/master.conf --output /etc/clamav-unofficial-sigs/master.conf -curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/os/os.macosx.conf --output /etc/clamav-unofficial-sigs/os.conf -curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/user.conf --output /etc/clamav-unofficial-sigs/user.conf +mkdir -p /usr/local/etc/clamav-unofficial-sigs +curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/master.conf --output /usr/local/etc/clamav-unofficial-sigs/master.conf +curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/os/os.macosx.conf --output /usr/local/etc/clamav-unofficial-sigs/os.conf +curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/user.conf --output /usr/local/etc/clamav-unofficial-sigs/user.conf exit ``` # Step 5 set your user options ``` -sudo pico /etc/clamav-unofficial-sigs/user.conf +sudo pico /usr/local/etc/clamav-unofficial-sigs/user.conf ``` # Step 6 From 694d1f9eb7fe95ea9eca1791db239129bd6d81a1 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 17:08:06 +0200 Subject: [PATCH 086/101] shellcheck fixes --- README.md | 8 +++++++- clamav-unofficial-sigs.sh | 38 +++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 4364f6dc..bd27799b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,12 @@ ClamAV Unofficial Signatures Updater ## Description -The clamav-unofficial-sigs script provides a simple way to download, test, and update third-party signature databases provided by Sanesecurity, FOXHOLE, OITC, Scamnailer, BOFHLAND, CRDF, Porcupine, Securiteinfo, MalwarePatrol, Yara-Rules Project, urlhaus, etc. The script will also generate and install cron, logrotate, and man files. +The clamav-unofficial-sigs script provides a simple way to download, test, and update third-party signature databases provided by Sanesecurity, FOXHOLE, OITC, Scamnailer, BOFHLAND, CRDF, Porcupine, Securiteinfo, MalwarePatrol, Yara-Rules Project, urlhaus, MalwareExpert etc. The script will also generate and install cron, logrotate, and man files. + +### Automated Testing and Linting +* Travis-CI +* Linting with markdownlint-cli and shellcheck +* Testing with Ubuntu Focal and MacOS / OSX ### Checkout some of our other solutions: @@ -221,6 +226,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] * Added detection of tar executable (use gtar on mac and bsd) * Config os.macosx.conf renamed to os.macos.conf * Fix: set ownership of last-version-check.txt +- More automated linting and testing (markdown and macos / osx) via travis-ci * Incremented the config to version 93 ### Version 7.1 ( Not Released) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 96861c3c..151912fe 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -1012,7 +1012,7 @@ function make_signature_database_from_ascii_file() { target type is used and full file scanning is enabled (see ClamAV signatures.pdf for details). - Line numbering will be done automatically by the script. - " | command $sed_bin 's/^ //g' + " | command "$sed_bin" 's/^ //g' echo -n "Do you wish to continue? " if xshok_prompt_confirm ; then @@ -1026,7 +1026,7 @@ function make_signature_database_from_ascii_file() { echo -n "Enter signature prefix: " read -r prefix - path_file="$(echo "$source" | cut -d "." -f -1 | command $sed_bin 's/$/.ndb/')" + path_file="$(echo "$source" | cut -d "." -f -1 | command "$sed_bin" 's/$/.ndb/')" db_file="$(basename "$path_file")" rm -f "$path_file" total="$(wc -l "$source" | cut -d " " -f 1)" @@ -1035,11 +1035,11 @@ function make_signature_database_from_ascii_file() { while read -r line ; do line_prefix="$(echo "$line" | awk -F ":" '{print $1}')" if [ "$line_prefix" == "-" ] ; then - echo "$line" | cut -d ":" -f 2- | perl -pe 's/(.)/sprintf("%02lx", ord $1)/eg' | command $sed_bin "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" + echo "$line" | cut -d ":" -f 2- | perl -pe 's/(.)/sprintf("%02lx", ord $1)/eg' | command "$sed_bin" "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" elif [ "$line_prefix" == "=" ] ; then - echo "$line" | cut -d ":" -f 2- | perl -pe 's/(\{[^}]*\}|\([^)]*\)|\*)|(.)/defined $1 ? $1 : sprintf("%02lx", ord $2)/eg' | command $sed_bin "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" + echo "$line" | cut -d ":" -f 2- | perl -pe 's/(\{[^}]*\}|\([^)]*\)|\*)|(.)/defined $1 ? $1 : sprintf("%02lx", ord $2)/eg' | command "$sed_bin" "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" else - echo "$line" | perl -pe 's/(.)/sprintf("%02lx", ord $1)/eg' | command $sed_bin "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" + echo "$line" | perl -pe 's/(.)/sprintf("%02lx", ord $1)/eg' | command "$sed_bin" "s/^/$prefix\\.$line_num:4:\\*:/" >> "$path_file" fi xshok_pretty_echo_and_log "Hexadecimal encoding ${source_file} line: ${line_num} of ${total}" line_num="$((line_num + 1))" @@ -1793,7 +1793,7 @@ for config_file in "${config_files[@]}" ; do if [ "$(uname -s)" == "SunOS" ] ; then # Solaris FIXES only, i had issues with running with a single command.. - clean_config="$(command $sed_bin -e '/^#.*/d' "$config_file")" # Comment line + clean_config="$(command "$sed_bin" -e '/^#.*/d' "$config_file")" # Comment line #clean_config="$(echo "$clean_config" | $sed_bin -e 's/#[[:space:]].*//')" # Comment line (duplicated) clean_config=${clean_config//\#*/} # Comment line (duplicated) # shellcheck disable=SC2001 @@ -1806,7 +1806,7 @@ for config_file in "${config_files[@]}" ; do elif [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then # MacOS / OSX / BSD fixes, had issues with running with a single command and with SunOS work around.. # shellcheck disable=SC2001 - clean_config="$(command $sed_bin -e '/^#.*/d' "$config_file")" # Comment line + clean_config="$(command "$sed_bin" -e '/^#.*/d' "$config_file")" # Comment line # shellcheck disable=SC2001 clean_config="$(echo "$clean_config" | $sed_bin -e 's/#[[:space:]].*//')" # Comment line (duplicated) # shellcheck disable=SC2001 @@ -1823,7 +1823,7 @@ for config_file in "${config_files[@]}" ; do # Delete both trailing and leading whitespace # Delete all trailing whitespace # Delete all empty lines - clean_config="$(command $sed_bin -e '/^#.*/d' -e 's/[[:space:]]#.*//' -e 's/#[[:space:]].*//' -e 's/^[ \t]*//;s/[ \t]*$//' -e '/^\s*$/d' "$config_file")" + clean_config="$(command "$sed_bin" -e '/^#.*/d' -e 's/[[:space:]]#.*//' -e 's/#[[:space:]].*//' -e 's/^[ \t]*//;s/[ \t]*$//' -e '/^\s*$/d' "$config_file")" fi @@ -1851,7 +1851,7 @@ for config_file in "${config_files[@]}" ; do # Config loading for i in "${clean_config[@]}" ; do - eval "$(echo "${i}" | command $sed_bin -e 's/[[:space:]]*$//' 2> /dev/null)" + eval "$(echo "${i}" | command "$sed_bin" -e 's/[[:space:]]*$//' 2> /dev/null)" done fi done @@ -2546,7 +2546,7 @@ if [ -n "$ham_dir" ] && [ -d "$work_dir" ] && [ ! -d "$test_dir" ] ; then xshok_mkdir_ownership "$test_dir" cp -f -p "$work_dir"/*/*.ndb "$test_dir" cp -f -p "$work_dir"/*/*.db "$test_dir" - $clamscan_bin --infected --no-summary -d "$test_dir" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' >> "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "$test_dir" "$ham_dir"/* | command "$sed_bin" 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' >> "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/*.ndb" | cut -d "*" -f 2 | sort | uniq > "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/*.db" | cut -d "=" -f 2 | awk '{ printf("=%s\n", $1);}' | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" cd "$test_dir" || exit @@ -2832,7 +2832,7 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then for sanesecurity_mirror_ip in $sanesecurity_mirror_ips ; do if [ -n "$dig_bin" ] ; then # shellcheck disable=SC2086 - sanesecurity_mirror_name="$($dig_bin $dig_proxy +short -x "$sanesecurity_mirror_ip" | command $sed_bin 's/\.$//')" + sanesecurity_mirror_name="$($dig_bin $dig_proxy +short -x "$sanesecurity_mirror_ip" | command "$sed_bin" 's/\.$//')" else sanesecurity_mirror_name="" fi @@ -2894,7 +2894,7 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_sanesecurity}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command "$sed_bin" 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex-tmp" mv -f "${work_dir_work_configs}/whitelist.hex-tmp" "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" @@ -3035,7 +3035,7 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_securiteinfo}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command "$sed_bin" 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" @@ -3195,7 +3195,7 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_linuxmalwaredetect}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command "$sed_bin" 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" @@ -3338,7 +3338,7 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_malwareexpert}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command "$sed_bin" 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" @@ -3482,7 +3482,7 @@ if [ "$malwarepatrol_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_malwarepatrol}/${malwarepatrol_db}" > "${test_dir}/${malwarepatrol_db}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${malwarepatrol_db}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${malwarepatrol_db}" "$ham_dir"/* | command "$sed_bin" 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${malwarepatrol_db}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${malwarepatrol_db}" > "${test_dir}/${malwarepatrol_db}-tmp" mv -f "${test_dir}/${malwarepatrol_db}-tmp" "${test_dir}/${malwarepatrol_db}" @@ -3611,7 +3611,7 @@ if [ "$urlhaus_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_urlhaus}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command "$sed_bin" 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" @@ -3748,7 +3748,7 @@ if [ "$yararulesproject_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_yararulesproject}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command "$sed_bin" 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" @@ -3906,7 +3906,7 @@ if [ "$additional_enabled" == "yes" ] ; then fi else $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_add}/${db_file}" > "${test_dir}/${db_file}" - $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command $sed_bin 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command "$sed_bin" 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" if [[ "${work_dir_add}/${db_file}" == *.db ]] ; then $grep_bin -h -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" | cut -d "=" -f 2 | awk '{ printf("=%s\n", $1);}' |sort | uniq >> "${work_dir_work_configs}/whitelist.hex-tmp" mv -f "${work_dir_work_configs}/whitelist.hex-tmp" "${work_dir_work_configs}/whitelist.hex" From f30528d5b970e925fe3c5a9ddd22e41d1be491e1 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 20:30:07 +0200 Subject: [PATCH 087/101] Updated macOS installation guide for Big Sur (OSX 11) --- .travis.yml | 4 +- README.md | 8 ++- clamav-unofficial-sigs.sh | 2 +- config/os/os.macos.conf | 10 +-- guides/macos.md | 140 +++++++++++++++++++++++++++++++++++--- 5 files changed, 145 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index ec94ec04..00d09c9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,7 +73,7 @@ matrix: addons: code_climate: repo_token: $ci_codeclimate_repo_token -#### LOGIC AND CLAMAV TESTING : MACOS / OSX +#### LOGIC AND CLAMAV TESTING : macOS / OSX - language: shell os: osx #Enable modern build platform, not container based. @@ -92,7 +92,7 @@ matrix: script: - sudo cp -f .t/tests/user.conf /usr/local/etc/clamav-unofficial-sigs/user.conf - - sudo /usr/local/bin/clamav-unofficial-sigs.sh --force + #- sudo /usr/local/bin/clamav-unofficial-sigs.sh --force #- sudo sh -e .t/ci-clamav-download-default-databases.sh #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh #- sudo sh -e .t/ci-clamav-install-default.sh diff --git a/README.md b/README.md index bd27799b..77b38828 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,10 @@ ClamAV Unofficial Signatures Updater The clamav-unofficial-sigs script provides a simple way to download, test, and update third-party signature databases provided by Sanesecurity, FOXHOLE, OITC, Scamnailer, BOFHLAND, CRDF, Porcupine, Securiteinfo, MalwarePatrol, Yara-Rules Project, urlhaus, MalwareExpert etc. The script will also generate and install cron, logrotate, and man files. ### Automated Testing and Linting + * Travis-CI * Linting with markdownlint-cli and shellcheck -* Testing with Ubuntu Focal and MacOS / OSX +* Testing with Ubuntu Focal and macOS / OSX ### Checkout some of our other solutions: @@ -226,7 +227,8 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] * Added detection of tar executable (use gtar on mac and bsd) * Config os.macosx.conf renamed to os.macos.conf * Fix: set ownership of last-version-check.txt -- More automated linting and testing (markdown and macos / osx) via travis-ci +* More automated linting and testing (markdown and macOS / osx) via travis-ci +* Updated macOS installation guide for Big Sur (OSX 11) * Incremented the config to version 93 ### Version 7.1 ( Not Released) @@ -366,7 +368,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] * eXtremeSHOK.com Maintenance * PGP is now optional and no longer a requirement and pgp support is auto-detected -* Full support for MacOS / OS X and added clamav install guide +* Full support for macOS / OS X and added clamav install guide * Full support for pfSense and added clamav install guide * Added os configs for Zimbra and Debian 8 with systemd * Much better error messages with possible solutions given diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 151912fe..b00cf818 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -1804,7 +1804,7 @@ for config_file in "${config_files[@]}" ; do clean_config="$(echo "$clean_config" | $sed_bin -e '/^\s*$/d')" # Blank lines elif [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then - # MacOS / OSX / BSD fixes, had issues with running with a single command and with SunOS work around.. + # macOS / OSX / BSD fixes, had issues with running with a single command and with SunOS work around.. # shellcheck disable=SC2001 clean_config="$(command "$sed_bin" -e '/^#.*/d' "$config_file")" # Comment line # shellcheck disable=SC2001 diff --git a/config/os/os.macos.conf b/config/os/os.macos.conf index 2333618c..8c1c8903 100644 --- a/config/os/os.macos.conf +++ b/config/os/os.macos.conf @@ -27,15 +27,15 @@ clam_user="clamav" # On some systems the clamgroup is "virusgroup" clam_group="clamav" -clam_dbs="/var/lib/clamav" +clam_dbs="/usr/local/var/clamav/db" -clamd_pid="/var/run/clamd.pid" +clamd_pid="/usr/local/var/clamav/run/clamd.pid" -clamscan_bin="/usr/local/bin/clamscan" +clamscan_bin="/usr/local/bin/clamdscan" -work_dir="/var/db/clamav-unofficial-sigs" +work_dir="/usr/local/var/clamav-unofficial-sigs" -log_file_path="/var/log" +log_file_path="/usr/local/var/clamav/log" clamd_restart_opt="/usr/local/bin/clamdscan --reload" diff --git a/guides/macos.md b/guides/macos.md index cc73b3c7..38b91aea 100644 --- a/guides/macos.md +++ b/guides/macos.md @@ -9,7 +9,7 @@ clamav-unofficial-sigs.sh --force ``` ## Notes: -https://www.clamav.net/documents/installation-on-macos-mac-os-x +Tested on macOS Big Sur (OSX 11) ## Install Requirements # Step 1 Install Homebrew @@ -28,7 +28,130 @@ brew install gnu-tar gnu-tar brew install clamav ``` -# Step 4 +# Step 4 Configure clamav +``` +# Create clamav user and group +sudo dscl . create /Groups/clamav +sudo dscl . create /Groups/clamav RealName "Clam Antivirus Group" +sudo dscl . create /Groups/clamav gid 799 +sudo dscl . create /Users/clamav +sudo dscl . create /Users/clamav RealName "Clam Antivirus User" +sudo dscl . create /Users/clamav UserShell /bin/false +sudo dscl . create /Users/clamav UniqueID 599 +sudo dscl . create /Users/clamav PrimaryGroupID 799 + +# Create the dirs +sudo mkdir -p /usr/local/var/clamav/run +sudo mkdir -p /usr/local/var/clamav/log +sudo mkdir -p /usr/local/var/clamav/db +sudo mkdir -p "/Library/LaunchDaemons" + +# Generate the configs +cp "/usr/local/etc/clamav/clamd.conf.sample" "/usr/local/etc/clamav/clamd.conf" +sed -e "s|# Example config file|# Config file|" \ + -e "s|^Example$|# Example|" \ + -e "s|^#MaxDirectoryRecursion 20$|MaxDirectoryRecursion 25|" \ + -e "s|^#LogFile .*|LogFile /usr/local/var/clamav/log/clamd.log|" \ + -e "s|^#PidFile .*|PidFile /usr/local/var/clamav/run/clamd.pid|" \ + -e "s|^#DatabaseDirectory .*|DatabaseDirectory /usr/local/var/clamav/db|" \ + -e "s|^#LocalSocket .*|LocalSocket /usr/local/var/clamav/run/clamd.socket|" \ + -i -n "/usr/local/etc/clamav/clamd.conf" + +cp "/usr/local/etc/clamav/freshclam.conf.sample" "/usr/local/etc/clamav/freshclam.conf" +sed -e "s|# Example config file|# Config file|" \ + -e "s|^Example$|# Example|" \ + -e "s|^#DatabaseDirectory .*|DatabaseDirectory /usr/local/var/clamav/db|" \ + -e "s|^#UpdateLogFile .*|UpdateLogFile /usr/local/var/clamav/log/freshclam.log|" \ + -e "s|^#PidFile .*|PidFile /usr/local/var/clamav/run/freshclam.pid|" \ + -e "s|^#NotifyClamd .*|NotifyClamd /usr/local/etc/clamav/clamd.conf|" \ + -i -n "/usr/local/etc/clamav/freshclam.conf" + +# Fix permissions +sudo chown -R clamav:clamav /usr/local/var/clamav + +sudo tee "/Library/LaunchDaemons/clamav.clamd.plist" << EOF > /dev/null + + + + + Label + clamav.clamd + ProgramArguments + + /usr/local/sbin/clamd + --foreground + + KeepAlive + + StandardErrorPath + /usr/local/var/clamav/log/clamd.error.log + + +EOF + +sudo tee "/Library/LaunchDaemons/clamav.freshclam.plist" << EOF > /dev/null + + + + + Label + ${FRESHCLAM_DAEMON_NAME} + ProgramArguments + + /usr/local/bin/freshclam + --daemon + --foreground + + KeepAlive + + RunAtLoad + + StandardErrorPath + /usr/local/var/clamav/log/freshclam.error.log + StartInterval + 86400 + + +EOF + +sudo tee "/Library/LaunchDaemons/clamav.clamdscan.plist" << EOF > /dev/null + + + + + Label + ${CLAMDSCAN_DAEMON_NAME} + ProgramArguments + + /usr/local/bin/clamdscan + --log=/usr/local/var/clamav/log/clamdscan.log + -m + / + + KeepAlive + + RunAtLoad + + StartCalendarInterval + + Hour + 1 + Minute + 45 + + StandardErrorPath + /usr/local/var/clamav/log/clamdscan.error.log + + +EOF + +sudo chown root:wheel "/Library/LaunchDaemons/clamav.clamd.plist" "/Library/LaunchDaemons/clamav.freshclam.plist" "/Library/LaunchDaemons/clamav.clamdscan.plist" +sudo chmod 0644 "/Library/LaunchDaemons/clamav.clamd.plist" "/Library/LaunchDaemons/clamav.freshclam.plist" "/Library/LaunchDaemons/clamav.clamdscan.plist" +sudo launchctl load "/Library/LaunchDaemons/clamav.clamd.plist" "/Library/LaunchDaemons/clamav.freshclam.plist" "/Library/LaunchDaemons/clamav.clamdscan.plist" + +``` + +# Step 5 ``` sudo su curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/clamav-unofficial-sigs.sh --output /usr/local/bin/clamav-unofficial-sigs.sh @@ -40,23 +163,22 @@ curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master exit ``` -# Step 5 +# Step 6 set your user options ``` sudo pico /usr/local/etc/clamav-unofficial-sigs/user.conf ``` -# Step 6 +# Step 7 Console (shell) ``` clamav-unofficial-sigs.sh --force ``` -# Step 7 +# Step 8 launchd helper Script (replaces cron) ``` -sudo su -cat < /Library/LaunchDaemons/com.clamav-unofficial-sigs.plist +sudo tee "/Library/LaunchDaemons/clamav.clamav-unofficial-sigs.plist" << EOF > /dev/null @@ -72,5 +194,7 @@ cat < /Library/LaunchDaemons/com.clamav-unofficial-sigs.plist EOF -exit +sudo chown root:wheel "/Library/LaunchDaemons/clamav.clamav-unofficial-sigs.plist" +sudo chmod 0644 "/Library/LaunchDaemons/clamav.clamav-unofficial-sigs.plist" +sudo launchctl load "/Library/LaunchDaemons/clamav.clamav-unofficial-sigs.plist" ``` From d64d3945ab0ca0cba66bb4d361b5d39b257f033a Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 21:42:23 +0200 Subject: [PATCH 088/101] more macos travis enhancements --- .t/ci-clamav-install-macos-clamav.sh | 81 +++++++++++++++++++ ...h => ci-clamav-install-macos-databases.sh} | 10 ++- ... => ci-clamav-install-ubuntu-databases.sh} | 0 .travis.yml | 7 +- guides/macos.md | 10 ++- 5 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 .t/ci-clamav-install-macos-clamav.sh rename .t/{ci-clamav-install-default.sh => ci-clamav-install-macos-databases.sh} (59%) rename .t/{ci-clamav-install-default-database.sh => ci-clamav-install-ubuntu-databases.sh} (100%) diff --git a/.t/ci-clamav-install-macos-clamav.sh b/.t/ci-clamav-install-macos-clamav.sh new file mode 100644 index 00000000..7ae8b32d --- /dev/null +++ b/.t/ci-clamav-install-macos-clamav.sh @@ -0,0 +1,81 @@ +#!/bin/sh +################### +# This is property of eXtremeSHOK.com +# You are free to use, modify and distribute, however you may not remove this notice. +# Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com +# License: BSD (Berkeley Software Distribution) +################## +export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/usr/local/musl/bin:$HOME/bin + +pwd + +echo "Installing default Clamav" + +# Create clamav user and group +sudo dscl . create /Groups/clamav +sudo dscl . create /Groups/clamav RealName "Clam Antivirus Group" +sudo dscl . create /Groups/clamav gid 799 +sudo dscl . create /Users/clamav +sudo dscl . create /Users/clamav RealName "Clam Antivirus User" +sudo dscl . create /Users/clamav UserShell /bin/false +sudo dscl . create /Users/clamav UniqueID 599 +sudo dscl . create /Users/clamav PrimaryGroupID 799 + +# Create the dirs +sudo mkdir -p /usr/local/var/clamav/run +sudo mkdir -p /usr/local/var/clamav/log +sudo mkdir -p /usr/local/var/clamav/db +sudo mkdir -p "/Library/LaunchDaemons" + +# Generate the configs +cp "/usr/local/etc/clamav/clamd.conf.sample" "/usr/local/etc/clamav/clamd.conf" +sed -e "s|# Example config file|# Config file|" \ + -e "s|^Example$|# Example|" \ + -e "s|^#MaxDirectoryRecursion 20$|MaxDirectoryRecursion 25|" \ + -e "s|^#LogFile .*|LogFile /usr/local/var/clamav/log/clamd.log|" \ + -e "s|^#PidFile .*|PidFile /usr/local/var/clamav/run/clamd.pid|" \ + -e "s|^#DatabaseDirectory .*|DatabaseDirectory /usr/local/var/clamav/db|" \ + -e "s|^#LocalSocket .*|LocalSocket /usr/local/var/clamav/run/clamd.socket|" \ + -e "s|^#FixStaleSocket|FixStaleSocket|" \" + -i -n "/usr/local/etc/clamav/clamd.conf" + +cp "/usr/local/etc/clamav/freshclam.conf.sample" "/usr/local/etc/clamav/freshclam.conf" +sed -e "s|# Example config file|# Config file|" \ + -e "s|^Example$|# Example|" \ + -e "s|^#DatabaseDirectory .*|DatabaseDirectory /usr/local/var/clamav/db|" \ + -e "s|^#UpdateLogFile .*|UpdateLogFile /usr/local/var/clamav/log/freshclam.log|" \ + -e "s|^#PidFile .*|PidFile /usr/local/var/clamav/run/freshclam.pid|" \ + -e "s|^#NotifyClamd .*|NotifyClamd /usr/local/etc/clamav/clamd.conf|" \ + -i -n "/usr/local/etc/clamav/freshclam.conf" + +# Fix permissions +sudo chown -R clamav:clamav /usr/local/var/clamav + +# Clamd socket +sudo touch /usr/local/var/clamav/run/clamd.socket +sudo chown clamav:clamav /usr/local/var/clamav/run/clamd.socket + +sudo tee "/Library/LaunchDaemons/clamav.clamd.plist" << EOF > /dev/null + + + + + Label + clamav.clamd + ProgramArguments + + /usr/local/sbin/clamd + --foreground + + KeepAlive + + StandardErrorPath + /usr/local/var/clamav/log/clamd.error.log + + +EOF + + +sudo chown root:wheel "/Library/LaunchDaemons/clamav.clamd.plist" +sudo chmod 0644 "/Library/LaunchDaemons/clamav.clamd.plist" +sudo launchctl load "/Library/LaunchDaemons/clamav.clamd.plist" diff --git a/.t/ci-clamav-install-default.sh b/.t/ci-clamav-install-macos-databases.sh similarity index 59% rename from .t/ci-clamav-install-default.sh rename to .t/ci-clamav-install-macos-databases.sh index d85ec1e0..f71fc281 100644 --- a/.t/ci-clamav-install-default.sh +++ b/.t/ci-clamav-install-macos-databases.sh @@ -9,9 +9,15 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/usr/lo pwd -echo "Installing default Clamav" +echo "Installing latest clamav databases" -apt-get install -y clamav-base clamav-freshclam clamav clamav-daemon -qq +mkdir -p /var/lib/clamav +cp -f bytecode.cvd /usr/local/var/clamav/db/bytecode.cvd +cp -f daily.cvd /usr/local/var/clamav/db/daily.cvd +cp -f main.cvd /usr/local/var/clamav/db/main.cvd +chown -R clamav:clamav /usr/local/var/clamav/db/ + +launchctl kickstart -k system/clamav.clamd if [ "$?" -eq "0" ] ; then echo .. OK else diff --git a/.t/ci-clamav-install-default-database.sh b/.t/ci-clamav-install-ubuntu-databases.sh similarity index 100% rename from .t/ci-clamav-install-default-database.sh rename to .t/ci-clamav-install-ubuntu-databases.sh diff --git a/.travis.yml b/.travis.yml index 00d09c9b..01c628ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,6 +45,7 @@ matrix: before_install: - sudo apt-get update -qq - sudo apt-get install -y ca-certificates curl wget rsync p7zip-full -qq + - sudo apt-get install -y clamav-base clamav-freshclam clamav clamav-daemon -qq #- pip install bashate # - "mkdir -p tmp/cache/" # - "ls -laFh tmp/cache/clamav-dbs" @@ -65,8 +66,7 @@ matrix: script: - sudo sh -e .t/ci-clamav-download-default-databases.sh #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh - - sudo sh -e .t/ci-clamav-install-default.sh - - sudo sh -e .t/ci-clamav-install-default-database.sh + - sudo sh -e .t/ci-clamav-install-ubuntu-databases.sh - sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf - sudo sh -e .t/ci-test.sh @@ -91,6 +91,9 @@ matrix: - sudo cp -f config/os/os.macos.conf /usr/local/etc/clamav-unofficial-sigs/os.conf script: + - sudo sh -e .t/ci-clamav-download-default-databases.sh + - sudo sh -e .t/ci-clamav-install-macos.sh + - sudo sh -e .t/ci-clamav-install-macos-databases.sh - sudo cp -f .t/tests/user.conf /usr/local/etc/clamav-unofficial-sigs/user.conf #- sudo /usr/local/bin/clamav-unofficial-sigs.sh --force #- sudo sh -e .t/ci-clamav-download-default-databases.sh diff --git a/guides/macos.md b/guides/macos.md index 38b91aea..b37ae303 100644 --- a/guides/macos.md +++ b/guides/macos.md @@ -44,7 +44,7 @@ sudo dscl . create /Users/clamav PrimaryGroupID 799 sudo mkdir -p /usr/local/var/clamav/run sudo mkdir -p /usr/local/var/clamav/log sudo mkdir -p /usr/local/var/clamav/db -sudo mkdir -p "/Library/LaunchDaemons" +sudo mkdir -p "/Library/LaunchDaemons" # Generate the configs cp "/usr/local/etc/clamav/clamd.conf.sample" "/usr/local/etc/clamav/clamd.conf" @@ -55,6 +55,7 @@ sed -e "s|# Example config file|# Config file|" \ -e "s|^#PidFile .*|PidFile /usr/local/var/clamav/run/clamd.pid|" \ -e "s|^#DatabaseDirectory .*|DatabaseDirectory /usr/local/var/clamav/db|" \ -e "s|^#LocalSocket .*|LocalSocket /usr/local/var/clamav/run/clamd.socket|" \ + -e "s|^#FixStaleSocket|FixStaleSocket|" \" -i -n "/usr/local/etc/clamav/clamd.conf" cp "/usr/local/etc/clamav/freshclam.conf.sample" "/usr/local/etc/clamav/freshclam.conf" @@ -69,6 +70,10 @@ sed -e "s|# Example config file|# Config file|" \ # Fix permissions sudo chown -R clamav:clamav /usr/local/var/clamav +# Clamd socket +sudo touch /usr/local/var/clamav/run/clamd.socket +sudo chown clamav:clamav /usr/local/var/clamav/run/clamd.socket + sudo tee "/Library/LaunchDaemons/clamav.clamd.plist" << EOF > /dev/null @@ -158,9 +163,8 @@ curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master chmod 755 /usr/local/bin/clamav-unofficial-sigs.sh mkdir -p /usr/local/etc/clamav-unofficial-sigs curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/master.conf --output /usr/local/etc/clamav-unofficial-sigs/master.conf -curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/os/os.macosx.conf --output /usr/local/etc/clamav-unofficial-sigs/os.conf +curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/os/os.macos.conf --output /usr/local/etc/clamav-unofficial-sigs/os.conf curl https://raw.githubusercontent.com/extremeshok/clamav-unofficial-sigs/master/config/user.conf --output /usr/local/etc/clamav-unofficial-sigs/user.conf -exit ``` # Step 6 From 802b89036f30829c5c995d32c554276a06c48385 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 22:28:56 +0200 Subject: [PATCH 089/101] will it run on macos ? --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 01c628ec..3ef7ac35 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,10 +92,10 @@ matrix: script: - sudo sh -e .t/ci-clamav-download-default-databases.sh - - sudo sh -e .t/ci-clamav-install-macos.sh + - sudo sh -e .t/ci-clamav-install-macos-clamav.sh - sudo sh -e .t/ci-clamav-install-macos-databases.sh - sudo cp -f .t/tests/user.conf /usr/local/etc/clamav-unofficial-sigs/user.conf - #- sudo /usr/local/bin/clamav-unofficial-sigs.sh --force + - sudo /usr/local/bin/clamav-unofficial-sigs.sh --force #- sudo sh -e .t/ci-clamav-download-default-databases.sh #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh #- sudo sh -e .t/ci-clamav-install-default.sh From 8fe5ac3c0a81531e67e2ccb81af1cd146349995e Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 22:40:36 +0200 Subject: [PATCH 090/101] more macos fixes for travis --- .t/ci-clamav-install-macos-clamav.sh | 52 ++++++++++++------------- .t/ci-clamav-install-macos-databases.sh | 3 +- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/.t/ci-clamav-install-macos-clamav.sh b/.t/ci-clamav-install-macos-clamav.sh index 7ae8b32d..0363152d 100644 --- a/.t/ci-clamav-install-macos-clamav.sh +++ b/.t/ci-clamav-install-macos-clamav.sh @@ -12,22 +12,28 @@ pwd echo "Installing default Clamav" # Create clamav user and group -sudo dscl . create /Groups/clamav -sudo dscl . create /Groups/clamav RealName "Clam Antivirus Group" -sudo dscl . create /Groups/clamav gid 799 -sudo dscl . create /Users/clamav -sudo dscl . create /Users/clamav RealName "Clam Antivirus User" -sudo dscl . create /Users/clamav UserShell /bin/false -sudo dscl . create /Users/clamav UniqueID 599 -sudo dscl . create /Users/clamav PrimaryGroupID 799 +dscl . create /Groups/clamav +dscl . create /Groups/clamav RealName "Clam Antivirus Group" +dscl . create /Groups/clamav gid 799 +dscl . create /Users/clamav +dscl . create /Users/clamav RealName "Clam Antivirus User" +dscl . create /Users/clamav UserShell /bin/false +dscl . create /Users/clamav UniqueID 599 +dscl . create /Users/clamav PrimaryGroupID 799 # Create the dirs -sudo mkdir -p /usr/local/var/clamav/run -sudo mkdir -p /usr/local/var/clamav/log -sudo mkdir -p /usr/local/var/clamav/db -sudo mkdir -p "/Library/LaunchDaemons" +mkdir -p /usr/local/var/clamav/run +mkdir -p /usr/local/var/clamav/log +mkdir -p /usr/local/var/clamav/db +mkdir -p /Library/LaunchDaemons + +ls -laFh /usr/local/etc/clamav/ # Generate the configs +if [ ! -f "/usr/local/etc/clamav/clamd.conf.sample" ] ; then + echo "Missing: /usr/local/etc/clamav/clamd.conf" + exit 1 +fi cp "/usr/local/etc/clamav/clamd.conf.sample" "/usr/local/etc/clamav/clamd.conf" sed -e "s|# Example config file|# Config file|" \ -e "s|^Example$|# Example|" \ @@ -39,23 +45,14 @@ sed -e "s|# Example config file|# Config file|" \ -e "s|^#FixStaleSocket|FixStaleSocket|" \" -i -n "/usr/local/etc/clamav/clamd.conf" -cp "/usr/local/etc/clamav/freshclam.conf.sample" "/usr/local/etc/clamav/freshclam.conf" -sed -e "s|# Example config file|# Config file|" \ - -e "s|^Example$|# Example|" \ - -e "s|^#DatabaseDirectory .*|DatabaseDirectory /usr/local/var/clamav/db|" \ - -e "s|^#UpdateLogFile .*|UpdateLogFile /usr/local/var/clamav/log/freshclam.log|" \ - -e "s|^#PidFile .*|PidFile /usr/local/var/clamav/run/freshclam.pid|" \ - -e "s|^#NotifyClamd .*|NotifyClamd /usr/local/etc/clamav/clamd.conf|" \ - -i -n "/usr/local/etc/clamav/freshclam.conf" - # Fix permissions -sudo chown -R clamav:clamav /usr/local/var/clamav +chown -R clamav:clamav /usr/local/var/clamav # Clamd socket -sudo touch /usr/local/var/clamav/run/clamd.socket -sudo chown clamav:clamav /usr/local/var/clamav/run/clamd.socket +touch /usr/local/var/clamav/run/clamd.socket +chown clamav:clamav /usr/local/var/clamav/run/clamd.socket -sudo tee "/Library/LaunchDaemons/clamav.clamd.plist" << EOF > /dev/null +tee "/Library/LaunchDaemons/clamav.clamd.plist" << EOF > /dev/null @@ -76,6 +73,5 @@ sudo tee "/Library/LaunchDaemons/clamav.clamd.plist" << EOF > /dev/null EOF -sudo chown root:wheel "/Library/LaunchDaemons/clamav.clamd.plist" -sudo chmod 0644 "/Library/LaunchDaemons/clamav.clamd.plist" -sudo launchctl load "/Library/LaunchDaemons/clamav.clamd.plist" +chown root:wheel "/Library/LaunchDaemons/clamav.clamd.plist" +chmod 0644 "/Library/LaunchDaemons/clamav.clamd.plist" diff --git a/.t/ci-clamav-install-macos-databases.sh b/.t/ci-clamav-install-macos-databases.sh index f71fc281..c0b5296a 100644 --- a/.t/ci-clamav-install-macos-databases.sh +++ b/.t/ci-clamav-install-macos-databases.sh @@ -17,7 +17,8 @@ cp -f daily.cvd /usr/local/var/clamav/db/daily.cvd cp -f main.cvd /usr/local/var/clamav/db/main.cvd chown -R clamav:clamav /usr/local/var/clamav/db/ -launchctl kickstart -k system/clamav.clamd +#launchctl kickstart -k system/clamav.clamd +launchctl load "/Library/LaunchDaemons/clamav.clamd.plist" if [ "$?" -eq "0" ] ; then echo .. OK else From ca63465ae8fc49108774854f96802efa937ba3d0 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 22:46:18 +0200 Subject: [PATCH 091/101] fix homebrew install under travis-ci --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3ef7ac35..02233d93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,7 +81,10 @@ matrix: addons: homebrew: packages: - - gnu-tar gnu-sed clamav + - gnu-tar + - gnu-sed + - clamav + update: true install: - sudo mkdir -p /usr/local/bin - sudo mkdir -p /usr/local/etc/clamav-unofficial-sigs From a3142d194f854fbdb4041c539e69091323efa3c7 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 23:09:34 +0200 Subject: [PATCH 092/101] detect for gsed on macos and bsd fix travis updating homebrew --- .travis.yml | 2 +- clamav-unofficial-sigs.sh | 17 +++++++++++------ config/os/os.macos.conf | 6 ++++-- config/os/os.openbsd.conf | 2 ++ guides/macos.md | 2 +- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 02233d93..a590bf73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -84,7 +84,7 @@ matrix: - gnu-tar - gnu-sed - clamav - update: true + update: false install: - sudo mkdir -p /usr/local/bin - sudo mkdir -p /usr/local/etc/clamav-unofficial-sigs diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index b00cf818..39cd4f69 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -1640,13 +1640,18 @@ if [ -z "$grep_bin" ] ; then xshok_pretty_echo_and_log "ERROR: grep command is missing" exit 1 fi -# Detect supprot for sed -sed_bin="$(command -v sed 2> /dev/null)" -if [ -z "$grep_bin" ] ; then - xshok_pretty_echo_and_log "ERROR: sed command is missing" - exit 1 +# Detect support for sed or gsed +if [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then + sed_executable="gsed" +else + sed_executable="sed" +fi +if [ -z "$sed_bin" ]; then + sed_bin="$(command -v "$sed_executable" 2> /dev/null)" + else + xshok_pretty_echo_and_log "ERROR: gsed (gnu sed) is missing" + exit 1 fi - # Detect support for tar or gtar if [ "$(uname -s)" == "Darwin" ] || [ "$(uname -s)" == "OpenBSD" ] || [ "$(uname -s)" == "NetBSD" ] || [ "$(uname -s)" == "FreeBSD" ] ; then tar_executable="gtar" diff --git a/config/os/os.macos.conf b/config/os/os.macos.conf index 8c1c8903..8abd5b7d 100644 --- a/config/os/os.macos.conf +++ b/config/os/os.macos.conf @@ -20,6 +20,8 @@ # Mac OS and OS X with clamav installed via homebrew +# Requires gnu-sed (gsed) and gnu-tar (gtar) + # Follow the installation Instructions: see the guide in the guides folder clam_user="clamav" @@ -31,13 +33,13 @@ clam_dbs="/usr/local/var/clamav/db" clamd_pid="/usr/local/var/clamav/run/clamd.pid" -clamscan_bin="/usr/local/bin/clamdscan" +clamscan_bin="/usr/local/bin/clamscan" work_dir="/usr/local/var/clamav-unofficial-sigs" log_file_path="/usr/local/var/clamav/log" -clamd_restart_opt="/usr/local/bin/clamdscan --reload" +clamd_restart_opt="launchctl kickstart -k system/clamav.clamd" #clamd_socket="/tmp/clamd.socket" diff --git a/config/os/os.openbsd.conf b/config/os/os.openbsd.conf index 9f311879..8cbad62b 100644 --- a/config/os/os.openbsd.conf +++ b/config/os/os.openbsd.conf @@ -18,6 +18,8 @@ # Rename to os.conf to enable this file ################################################################################ +# Requires gnu-sed (gsed) and gnu-tar (gtar) + # OpenBSD clam_user="_clamav" diff --git a/guides/macos.md b/guides/macos.md index b37ae303..1cb06fa1 100644 --- a/guides/macos.md +++ b/guides/macos.md @@ -20,7 +20,7 @@ Tested on macOS Big Sur (OSX 11) # Step 2 Install dependencies : gtar (gnu-tar) sed (gnu-sed) ``` -brew install gnu-tar gnu-tar +brew install gnu-tar gnu-sed ``` # Step 3 Install clamav From 3edf03c80b4e2b65b2f3d197852b60feb41ce115 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 23:23:37 +0200 Subject: [PATCH 093/101] cleanup travis --- .t/ci-test-macos.sh | 87 ++++++++++++++++++++++++++++ .t/{ci-test.sh => ci-test-ubuntu.sh} | 0 .travis.yml | 33 ++--------- 3 files changed, 91 insertions(+), 29 deletions(-) create mode 100644 .t/ci-test-macos.sh rename .t/{ci-test.sh => ci-test-ubuntu.sh} (100%) diff --git a/.t/ci-test-macos.sh b/.t/ci-test-macos.sh new file mode 100644 index 00000000..ba2b613d --- /dev/null +++ b/.t/ci-test-macos.sh @@ -0,0 +1,87 @@ +#!/bin/sh +################### +# This is property of eXtremeSHOK.com +# You are free to use, modify and distribute, however you may not remove this notice. +# Copyright (c) Adrian Jon Kriel :: admin@extremeshok.com +# License: BSD (Berkeley Software Distribution) +################## +export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/usr/local/musl/bin:$HOME/bin + +pwd + +echo "running script verbose default curl" +bash /usr/local/bin/clamav-unofficial-sigs.sh --verbose +if [ "$?" -eq "0" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi + +echo "check signature placed correctly" +if [ -e "/usr/local/var/clamav/db/sanesecurity.ftm" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +# +# echo "check database integrity test" +# bash clamav-unofficial-sigs.sh --test-database sanesecurity.ftm +# if [ "$?" -eq "0" ] ; then +# echo .. OK +# else +# echo .. ERROR +# exit 1 +# fi +# +# echo "check gpg verify test" +# bash clamav-unofficial-sigs.sh --gpg-verify scam.ndb +# if [ "$?" -eq "0" ] ; then +# echo .. OK +# else +# echo .. ERROR +# exit 1 +# fi + +# echo "check clamav-daemon service will start" +# service clamav-daemon stop +# service clamav-daemon start +# if [ "$?" -eq "0" ] ; then +# echo .. OK +# else +# echo .. ERROR +# exit 1 +# f + +echo "===== HIGH /var/lib/clamav/ =====" +ls -laFh /var/lib/clamav/ +echo "================" + +echo "running script verbose with LOW ratings" +cp -f .t/tests/user_low.conf /usr/local/etc/clamav-unofficial-sigs/user.conf +bash /usr/local/bin/clamav-unofficial-sigs.sh --verbose +if [ "$?" -eq "0" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "===== LOW /var/lib/clamav/ =====" +ls -laFh /var/lib/clamav/ +echo "================" + +echo "Was /var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb removed ?" +if [ ! -e "/var/lib/clamav-unofficial-sigs/dbs-ss/jurlbl.ndb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi +echo "Was /var/lib/clamav/phish.ndb removed ?" +if [ ! -e "/var/lib/clamav/phish.ndb" ] ; then + echo .. OK +else + echo .. ERROR + exit 1 +fi diff --git a/.t/ci-test.sh b/.t/ci-test-ubuntu.sh similarity index 100% rename from .t/ci-test.sh rename to .t/ci-test-ubuntu.sh diff --git a/.travis.yml b/.travis.yml index a590bf73..01fb9cb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ matrix: #### MARKDOWN-LINT - language: node_js os: linux - #Enable modern build platform, not container based. dist: focal before_install: - nvm install 14.6.0 @@ -21,7 +20,6 @@ matrix: #### SHELLCHECK - language: shell os: linux - #Enable modern build platform, not container based. dist: focal install: - sudo apt-get install -y shellcheck -qq @@ -30,9 +28,7 @@ matrix: #### LOGIC AND CLAMAV TESTING : UBUNTU LINUX - language: shell os: linux - #Enable modern build platform, not container based. dist: focal - # Required travis ci environment variables #ci_malwareexpert_serial_key=[secure] #ci_securiteinfo_authorisation_signature=[secure] @@ -41,50 +37,34 @@ matrix: #ci_malwarepatrol_list=[secure] #ci_malwarepatrol_free=[secure] #ci_codeclimate_repo_token=[secure] - before_install: - sudo apt-get update -qq - sudo apt-get install -y ca-certificates curl wget rsync p7zip-full -qq - sudo apt-get install -y clamav-base clamav-freshclam clamav clamav-daemon -qq - #- pip install bashate - # - "mkdir -p tmp/cache/" - # - "ls -laFh tmp/cache/clamav-dbs" - # - "rsync -rlptDv tmp/cache/ /var/lib/clamav/" - # - "freshclam" - # - "ls -laFh /var/lib/clamav/" - # - "rsync -rlptDav /var/lib/clamav/ tmp/cache/" - # - "chown -R travis:travis tmp/cache" - # - "chmod -R 777 tmp/cache/" - # - "ls -laFh tmp/cache/clamav-dbs/" - install: - sudo mkdir -p /etc/clamav-unofficial-sigs - sudo cp -f config/master.conf /etc/clamav-unofficial-sigs/master.conf - sudo cp -f config/os/os.ubuntu.conf /etc/clamav-unofficial-sigs/os.conf - sudo cp -f clamav-unofficial-sigs.sh /usr/sbin/clamav-unofficial-sigs - script: - sudo sh -e .t/ci-clamav-download-default-databases.sh #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh - sudo sh -e .t/ci-clamav-install-ubuntu-databases.sh - sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf - - sudo sh -e .t/ci-test.sh - + - sudo sh -e .t/ci-test-ubuntu.sh addons: code_climate: repo_token: $ci_codeclimate_repo_token #### LOGIC AND CLAMAV TESTING : macOS / OSX - language: shell os: osx - #Enable modern build platform, not container based. - dist: focal addons: homebrew: packages: - gnu-tar - gnu-sed - clamav - update: false + update: true install: - sudo mkdir -p /usr/local/bin - sudo mkdir -p /usr/local/etc/clamav-unofficial-sigs @@ -95,13 +75,8 @@ matrix: script: - sudo sh -e .t/ci-clamav-download-default-databases.sh + #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh - sudo sh -e .t/ci-clamav-install-macos-clamav.sh - sudo sh -e .t/ci-clamav-install-macos-databases.sh - sudo cp -f .t/tests/user.conf /usr/local/etc/clamav-unofficial-sigs/user.conf - - sudo /usr/local/bin/clamav-unofficial-sigs.sh --force - #- sudo sh -e .t/ci-clamav-download-default-databases.sh - #- sudo sh -e .t/ci-clamav-download-default-databases-git.sh - #- sudo sh -e .t/ci-clamav-install-default.sh - #- sudo sh -e .t/ci-clamav-install-default-database.sh - #- sudo cp -f .t/tests/user.conf /etc/clamav-unofficial-sigs/user.conf - #- sudo sh -e .t/ci-test.sh + - sudo sh -e .t/ci-test-macos.sh From b33c260f7e00e39b19e1ea1822da1c1fb647a071 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Sun, 6 Dec 2020 23:41:31 +0200 Subject: [PATCH 094/101] apply caching to homebrew --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 01fb9cb8..37c48c0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,8 +56,13 @@ matrix: code_climate: repo_token: $ci_codeclimate_repo_token #### LOGIC AND CLAMAV TESTING : macOS / OSX - - language: shell - os: osx + - os: osx + osx_image: xcode12 + before_cache: + - brew cleanup + cache: + directories: + - $HOME/Library/Caches/Homebrew addons: homebrew: packages: From 0fabaf9fe0e81cac773c0d12c477ee12f8bbdd84 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Mon, 7 Dec 2020 00:04:19 +0200 Subject: [PATCH 095/101] addons homebrew is broken on the osx image --- .travis.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37c48c0b..6a96df59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,13 +63,15 @@ matrix: cache: directories: - $HOME/Library/Caches/Homebrew - addons: - homebrew: - packages: - - gnu-tar - - gnu-sed - - clamav - update: true + # addons: + # homebrew: + # packages: + # - gnu-tar + # - gnu-sed + # - clamav + before_install: + - homebrew update + - homebrew install gnu-tar gnu-sed clamav install: - sudo mkdir -p /usr/local/bin - sudo mkdir -p /usr/local/etc/clamav-unofficial-sigs From 329b833299a8e82682e9184be5d089be3bf237e5 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Mon, 7 Dec 2020 01:02:45 +0200 Subject: [PATCH 096/101] prepare for release --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 77b38828..8759e851 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] ## Change Log -### Version 7.2 ( XX December 2020: Under Development) +### Version 7.2 (07 December 2020) * Database rating downgrades are now supported, eg, changing from HIGH to LOW will remove the HIGH and MEDIUM rated databases. * Disabled databases are automatically removed @@ -230,8 +230,9 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] * More automated linting and testing (markdown and macOS / osx) via travis-ci * Updated macOS installation guide for Big Sur (OSX 11) * Incremented the config to version 93 +* Thank you @dandanio @jkellerer @msapiro @shawniverson -### Version 7.1 ( Not Released) +### Version 7.1 (Not Released) * Enforce HTTPS validation by default * Updated sanesecurity publickey.gpg url to use SSL @@ -244,7 +245,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] * Disable cron MAILTO * BSD read config fix * Incremented the config to version 92 -* Thank you @dandanio @Mrothyr @m0urs @msapiro @orlitzky @SlothOfAnarchy @jkellerer @RobbieTheK +* Thank you @dandanio @jkellerer @m0urs @Mrothyr @msapiro @orlitzky @RobbieTheK @SlothOfAnarchy ### Version 7.0.1 (25 January 2020) From 926886f9f2afe91b9e78aa6d0610e27313641461 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Mon, 7 Dec 2020 09:58:09 +0200 Subject: [PATCH 097/101] Added interServer databases (free) --- README.md | 11 +- clamav-unofficial-sigs.sh | 215 ++++++++++++++++++++++++++++++++++++-- config/master.conf | 33 +++++- config/user.conf | 1 + 4 files changed, 243 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 8759e851..f090b70b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ClamAV Unofficial Signatures Updater ## Description -The clamav-unofficial-sigs script provides a simple way to download, test, and update third-party signature databases provided by Sanesecurity, FOXHOLE, OITC, Scamnailer, BOFHLAND, CRDF, Porcupine, Securiteinfo, MalwarePatrol, Yara-Rules Project, urlhaus, MalwareExpert etc. The script will also generate and install cron, logrotate, and man files. +The clamav-unofficial-sigs script provides a simple way to download, test, and update third-party signature databases provided by Sanesecurity, FOXHOLE, OITC, Scamnailer, BOFHLAND, CRDF, Porcupine, Securiteinfo, MalwarePatrol, Yara-Rules Project, urlhaus, MalwareExpert, interServer etc. The script will also generate and install cron, logrotate, and man files. ### Automated Testing and Linting @@ -96,6 +96,10 @@ Usage of free Yara-Rules Project: Current limitations of clamav support: +### interServer free database support (as of December 2020) + +Usage of interServer: + ### malware.expert non-free database support (as of December 2020) Usage of Malware Expert: @@ -104,7 +108,7 @@ Usage of Malware Expert: 1. You will receive an email containing your serial key 1. Enter the serial key into the config malwareexpert_serial_key: replacing YOUR-SERIAL-KEY with your serial key from the email -### MalwarePatrol Free/Delayed list support (as of May 2015) +### MalwarePatrol free/delayed list support (as of May 2015) Usage of MalwarePatrol 2015 free clamav signatures: @@ -222,6 +226,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] * Disabled databases are automatically removed * Disable databases by setting the rating to "DISABLED" eg. securiteinfo_dbs_rating="DISABLED" will disable all securiteinfo databases * Added Malware Expert databases (non-free) +* Added interServer databases (free) * Reworked securiteinfo premium databases (non-free) * Added malwarepatrol_db to specify the exact database name (default: malwarepatrol.db) * Added detection of tar executable (use gtar on mac and bsd) @@ -229,7 +234,7 @@ Usage: clamav-unofficial-sigs.sh [OPTION] [PATH|FILE] * Fix: set ownership of last-version-check.txt * More automated linting and testing (markdown and macOS / osx) via travis-ci * Updated macOS installation guide for Big Sur (OSX 11) -* Incremented the config to version 93 +* Incremented the config to version 94 * Thank you @dandanio @jkellerer @msapiro @shawniverson ### Version 7.1 (Not Released) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 39cd4f69..8d676c6d 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -916,7 +916,7 @@ function gpg_verify_specific_sanesecurity_database_file() { # databasefile else xshok_pretty_echo_and_log "File ${db_file} cannot be found or is not a Sanesecurity database file." xshok_pretty_echo_and_log "Only the following Sanesecurity and OITC databases can be GPG signature tested:" - ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "${work_dir_sanesecurity}" + ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" --ignore "*.fp" "${work_dir_sanesecurity}" fi else xshok_pretty_echo_and_log "ERROR: Missing value for option" @@ -1163,25 +1163,28 @@ function clamscan_integrity_test_specific_database_file() { # databasefile xshok_pretty_echo_and_log "Here is a list of third-party databases that can be clamscan integrity tested:" xshok_pretty_echo_and_log "=== Sanesecurity ===" - ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_sanesecurity" + ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" --ignore "*.fp" "$work_dir_sanesecurity" xshok_pretty_echo_and_log "=== SecuriteInfo ===" - ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_securiteinfo" + ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" --ignore "*.fp" "$work_dir_securiteinfo" xshok_pretty_echo_and_log "=== MalwarePatrol ===" - ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_malwarepatrol" + ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" --ignore "*.fp" "$work_dir_malwarepatrol" xshok_pretty_echo_and_log "=== Linux Malware Detect ===" - ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_linuxmalwaredetect" + ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" --ignore "*.fp" "$work_dir_linuxmalwaredetect" + + xshok_pretty_echo_and_log "=== interServer Detect ===" + ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" --ignore "*.fp" "$work_dir_interserver" xshok_pretty_echo_and_log "=== Malware Expert Detect ===" - ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_malwareexpert" + ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" --ignore "*.fp" "$work_dir_malwareexpert" xshok_pretty_echo_and_log "=== Linux Malware Detect ===" - ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_yararulesproject" + ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" --ignore "*.fp" "$work_dir_yararulesproject" xshok_pretty_echo_and_log "=== User Defined Databases ===" - ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" "$work_dir_add" + ls --ignore "*.sig" --ignore "*.md5" --ignore "*.ign2" --ignore "*.fp" "$work_dir_add" xshok_pretty_echo_and_log "Check the file name and try again..." fi @@ -1529,7 +1532,7 @@ EOF # Script Info script_version="7.2" script_version_date="2020-12-02" -minimum_required_config_version="93" +minimum_required_config_version="94" minimum_yara_clamav_version="0.100" # Discover script: name, full_path and path @@ -1917,6 +1920,11 @@ if [ -z "$work_dir_linuxmalwaredetect" ] ; then else shopt -s extglob; work_dir_malwarepatrol="${work_dir_malwarepatrol%%+(/)}" fi +if [ -z "$work_dir_interserver" ] ; then + work_dir_interserver="$(echo "${work_dir}/${interserver_dir}" | $sed_bin 's:/*$::')" +else + shopt -s extglob; work_dir_interserver="${work_dir_interserver%%+(/)}" +fi if [ -z "$work_dir_malwareexpert" ] ; then work_dir_malwareexpert="$(echo "${work_dir}/${malwareexpert_dir}" | $sed_bin 's:/*$::')" else @@ -2071,6 +2079,7 @@ if [ "$force_updates" == "yes" ] ; then securiteinfo_update_hours="0" securiteinfo_premium_update_hours="0" linuxmalwaredetect_update_hours="0" + interserver_update_hours="0" malwareexpert_update_hours="0" malwarepatrol_update_hours="0" yararulesproject_update_hours="0" @@ -2217,6 +2226,9 @@ if [ "$default_dbs_rating" == "DISABLE" ] ; then if [ "$linuxmalwaredetect_dbs_rating" != "LOW" ] && [ "$linuxmalwaredetect_dbs_rating" != "MEDIUM" ] && [ "$linuxmalwaredetect_dbs_rating" != "HIGH" ]; then linuxmalwaredetect_enabled="no" fi + if [ "$interserver_dbs_rating" != "LOW" ] && [ "$interserver_dbs_rating" != "MEDIUM" ] && [ "$interserver_dbs_rating" != "HIGH" ]; then + interserver_enabled="no" + fi if [ "$malwareexpert_dbs_rating" != "LOW" ] && [ "$malwareexpert_dbs_rating" != "MEDIUM" ] && [ "$malwareexpert_dbs_rating" != "HIGH" ]; then malwareexpert_enabled="no" fi @@ -2236,6 +2248,9 @@ else if [ "$linuxmalwaredetect_dbs_rating" == "DISABLE" ] ; then linuxmalwaredetect_enabled="no" fi + if [ "$interserver_dbs_rating" == "DISABLE" ] ; then + interserver_enabled="no" + fi if [ "$malwareexpert_dbs_rating" == "DISABLE" ] ; then malwareexpert_enabled="no" fi @@ -2354,6 +2369,28 @@ if [ -n "$temp_remove_db" ] ; then read -r -a linuxmalwaredetect_remove_dbs <<< "$temp_remove_db" fi ############################################################################################ +if [ "$interserver_enabled" == "yes" ] ; then + if [ -n "$interserver_dbs" ] ; then + if [ -n "$interserver_dbs_rating" ] ; then + temp_db="$(xshok_database "$interserver_dbs_rating" "${interserver_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$interserver_dbs_rating" "${interserver_dbs[@]}")" + else + temp_db="$(xshok_database "$default_dbs_rating" "${interserver_dbs[@]}")" + temp_remove_db="$(xshok_remove_database "$default_dbs_rating" "${interserver_dbs[@]}")" + fi + interserver_dbs=( ) + if [ -n "$temp_db" ] ; then + read -r -a interserver_dbs <<< "$temp_db" + fi + fi +else + temp_remove_db="$(xshok_remove_database "DISABLED" "${interserver_dbs[@]}")" +fi +interserver_remove_dbs=( ) +if [ -n "$temp_remove_db" ] ; then + read -r -a interserver_remove_dbs <<< "$temp_remove_db" +fi +############################################################################################ if [ "$malwareexpert_enabled" == "yes" ] ; then if [ -n "$malwareexpert_dbs" ] ; then if [ -n "$malwareexpert_dbs_rating" ] ; then @@ -2487,6 +2524,18 @@ if [ -n "${linuxmalwaredetect_remove_dbs[0]}" ] ; then fi done fi +if [ -n "${interserver_remove_dbs[0]}" ] ; then + for db_file in "${interserver_remove_dbs[@]}" ; do + if [ -f "${work_dir_interserver}/${db_file}" ] ; then + echo "Removing unused file: ${work_dir_interserver}/${db_file}" + rm -f "${work_dir_interserver}/${db_file}" + fi + if [ -f "${clam_dbs}/${db_file}" ] ; then + echo "Removing unused file: ${clam_dbs}/${db_file}" + rm -f "${clam_dbs}/${db_file}" + fi + done +fi if [ -n "${malwareexpert_remove_dbs[0]}" ] ; then for db_file in "${malwareexpert_remove_dbs[@]}" ; do if [ -f "${work_dir_malwareexpert}/${db_file}" ] ; then @@ -2585,6 +2634,7 @@ xshok_mkdir_ownership "$work_dir" xshok_mkdir_ownership "$work_dir_securiteinfo" xshok_mkdir_ownership "$work_dir_malwarepatrol" xshok_mkdir_ownership "$work_dir_linuxmalwaredetect" +xshok_mkdir_ownership "$work_dir_interserver" xshok_mkdir_ownership "$work_dir_malwareexpert" xshok_mkdir_ownership "$work_dir_sanesecurity" xshok_mkdir_ownership "$work_dir_yararulesproject" @@ -2703,6 +2753,14 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then done fi fi +if [ "$interserver_enabled" == "yes" ] ; then + if [ -n "${interserver_dbs[0]}" ] ; then + for db in "${interserver_dbs[@]}" ; do + echo "${work_dir_interserver}/${db}" >> "${current_tmp}" + clamav_files + done + fi +fi if [ "$malwareexpert_enabled" == "yes" ] ; then if [ -n "${malwareexpert_dbs[0]}" ] ; then for db in "${malwareexpert_dbs[@]}" ; do @@ -3274,6 +3332,145 @@ else fi fi fi +############################################################################################################################################## +# Check for updated interServer database files every set number of hours as defined in the "USER CONFIGURATION" section of this script # +############################################################################################################################################## +if [ "$interserver_enabled" == "yes" ] ; then + if [ -n "${interserver_dbs[0]}" ] ; then + if [ ${#interserver_dbs} -lt 1 ] ; then + xshok_pretty_echo_and_log "Failed interserver_dbs config is invalid or not defined - SKIPPING" + else + rm -f "${work_dir_interserver}/*.gz" + if [ -r "${work_dir_work_configs}/last-is-update.txt" ] ; then + last_interserver_update="$(cat "${work_dir_work_configs}/last-is-update.txt")" + else + last_interserver_update="0" + fi + db_file="" + loop="" + if [ "$interserver_premium" == "yes" ] ; then + update_interval="$((interserver_premium_update_hours * 3600))" + else + update_interval="$((interserver_update_hours * 3600))" + fi + time_interval="$((current_time - last_interserver_update))" + if [ "$time_interval" -ge "$((update_interval - 600))" ] ; then + echo "$current_time" > "${work_dir_work_configs}/last-is-update.txt" + xshok_pretty_echo_and_log "interserver Database File Updates" "=" + xshok_pretty_echo_and_log "Checking for interserver updates..." + interserver_updates="0" + for db_file in "${interserver_dbs[@]}" ; do + if [ "$loop" == "1" ] ; then + xshok_pretty_echo_and_log "---" + fi + xshok_pretty_echo_and_log "Checking for updated interServer database file: ${db_file}" + interserver_db_update="0" + xshok_file_download "${work_dir_interserver}/${db_file}" "${interserver_url}/${db_file}" + ret="$?" + if [ "$ret" -eq 0 ] ; then + loop="1" + if ! cmp -s "${work_dir_interserver}/${db_file}" "${clam_dbs}/${db_file}" ; then + db_ext="${db_file#*.}" + + xshok_pretty_echo_and_log "Testing updated interServer database file: ${db_file}" + if [ -z "$ham_dir" ] || [ "$db_ext" != "ndb" ] ; then + if $clamscan_bin --quiet -d "${work_dir_interserver}/${db_file}" "${work_dir_work_configs}/scan-test.txt" 2>&10 ; then + xshok_pretty_echo_and_log "Clamscan reports interServer ${db_file} database integrity tested good" + true + else + xshok_pretty_echo_and_log "Clamscan reports interServer ${db_file} database integrity tested BAD" + if [ "$remove_bad_database" == "yes" ] ; then + if rm -f "${work_dir_interserver}/${db_file}" ; then + xshok_pretty_echo_and_log "Removed invalid database: ${work_dir_interserver}/${db_file}" + fi + fi + false + fi && (test "$keep_db_backup" = "yes" && cp -f -p "${clam_dbs}/${db_file}" "${clam_dbs}/${db}_file-bak" 2>/dev/null ; true) && if $rsync_bin -pcqt "${work_dir_interserver}/${db_file}" "$clam_dbs" 2>&13 ; then + perms chown -f "${clam_user}:${clam_group}" "${clam_dbs}/${db_file}" + if [ "$selinux_fixes" == "yes" ] ; then + restorecon "${clam_dbs}/${db_file}" + fi + xshok_pretty_echo_and_log "Successfully updated interServer production database file: ${db_file}" + interserver_updates=1 + interserver_db_update=1 + do_clamd_reload=1 + else + xshok_pretty_echo_and_log "Failed to successfully update interServer production database file: ${db_file} - SKIPPING" + fi + else + $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${work_dir_interserver}/${db_file}" > "${test_dir}/${db_file}" + $clamscan_bin --infected --no-summary -d "${test_dir}/${db_file}" "$ham_dir"/* | command "$sed_bin" 's/\.UNOFFICIAL FOUND//' | awk '{print $NF}' > "${work_dir_work_configs}/whitelist.txt" + $grep_bin -h -f "${work_dir_work_configs}/whitelist.txt" "${test_dir}/${db_file}" | cut -d "*" -f 2 | sort | uniq >> "${work_dir_work_configs}/whitelist.hex" + $grep_bin -h -v -f "${work_dir_work_configs}/whitelist.hex" "${test_dir}/${db_file}" > "${test_dir}/${db_file}-tmp" + mv -f "${test_dir}/${db_file}-tmp" "${test_dir}/${db_file}" + if $clamscan_bin --quiet -d "${test_dir}/${db_file}" "${work_dir_work_configs}/scan-test.txt" 2>&10 ; then + xshok_pretty_echo_and_log "Clamscan reports interServer ${db_file} database integrity tested good" + true + else + xshok_pretty_echo_and_log "Clamscan reports interServer ${db_file} database integrity tested BAD" + rm -f "${work_dir_interserver}/${db_file}" + if [ "$remove_bad_database" == "yes" ] ; then + if rm -f "${work_dir_interserver}/${db_file}" ; then + xshok_pretty_echo_and_log "Removed invalid database: ${work_dir_interserver}/${db_file}" + fi + fi + false + fi && (test "$keep_db_backup" = "yes" && cp -f -p "${clam_dbs}/${db_file}" "${clam_dbs}/${db}_file-bak" 2>/dev/null ; true) && if $rsync_bin -pcqt "${test_dir}/${db_file}" "$clam_dbs" 2>&13 ; then + perms chown -f "${clam_user}:${clam_group}" "${clam_dbs}/${db_file}" + if [ "$selinux_fixes" == "yes" ] ; then + restorecon "${clam_dbs}/${db_file}" + fi + xshok_pretty_echo_and_log "Successfully updated interServer production database file: ${db_file}" + interserver_updates=1 + interserver_db_update=1 + do_clamd_reload=1 + else + xshok_pretty_echo_and_log "Failed to successfully update interServer production database file: ${db_file} - SKIPPING" + fi + fi + fi + else + xshok_pretty_echo_and_log "Failed connection to ${interserver_url} - SKIPPED interServer ${db_file} update" + fi + if [ "$interserver_db_update" != "1" ] ; then + xshok_pretty_echo_and_log "No updated interServer ${db_file} database file found" "-" + fi + done + if [ "$interserver_updates" != "1" ] ; then + xshok_pretty_echo_and_log "No interServer database file updates found" "-" + fi + else + xshok_pretty_echo_and_log "interServer Database File Updates" "=" + if [ "$interserver_premium" == "yes" ] ; then + xshok_draw_time_remaining "$((update_interval - time_interval))" "$interserver_premium_update_hours" "interserver" + else + xshok_draw_time_remaining "$((update_interval - time_interval))" "$interserver_update_hours" "interserver" + fi + fi + fi + fi +else + if [ -n "$interserver_dbs" ] ; then + if [ "$remove_disabled_databases" == "yes" ] ; then + xshok_pretty_echo_and_log "Removing disabled interServer Database files" + for db_file in "${interserver_dbs[@]}" ; do + if echo "$db_file" | $grep_bin -q "|" ; then + db_file="${db_file%|*}" + fi + if [ -r "${work_dir_interserver}/${db_file}" ] ; then + xshok_pretty_echo_and_log "Removing ${work_dir_interserver}/${db_file}" + rm -f "${work_dir_interserver}/${db_file}" + do_clamd_reload=1 + fi + if [ -r "${clam_dbs}/${db_file}" ] ; then + xshok_pretty_echo_and_log "Removing ${clam_dbs}/${db_file}" + rm -f "${clam_dbs}/${db_file}" + do_clamd_reload=1 + fi + done + fi + fi +fi ############################################################################################################################################## # Check for updated Malware Expert database files every set number of hours as defined in the "USER CONFIGURATION" section of this script # diff --git a/config/master.conf b/config/master.conf index 0c69e4c9..76e4b5b3 100644 --- a/config/master.conf +++ b/config/master.conf @@ -116,13 +116,14 @@ securiteinfo_premium="no" # ======================== # Since the database files are dynamically created, non default values can cause banning, change with caution additional_update_hours="4" # Default is 4 hours (6 downloads daily). +interserver_update_hours="1" # Default is 2 hours (12 downloads daily). linuxmalwaredetect_update_hours="6" # Default is 6 hours (4 downloads daily). malwareexpert_update_hours="2" # Default is 2 hours (12 downloads daily). malwarepatrol_update_hours="24" # Default is 24 hours (1 downloads daily). sanesecurity_update_hours="2" # Default is 2 hours (12 downloads daily). securiteinfo_premium_update_hours="1" # Default is 1 hours (24 downloads daily). securiteinfo_update_hours="4" # Default is 4 hours (6 downloads daily). -urlhaus_update_hours="0" # Default is 0 hours (Update constantly). +urlhaus_update_hours="1" # Default is 1 hours (24 downloads daily). yararulesproject_update_hours="24" # Default is 24 hours (1 downloads daily). # ======================== @@ -130,6 +131,7 @@ yararulesproject_update_hours="24" # Default is 24 hours (1 downloads daily). # ======================== # Set to no to disable an entire database, if the database is empty it will also be disabled. additional_enabled="yes" # Additional Databases +interserver_enabled="yes" # interServer linuxmalwaredetect_enabled="yes" # Linux Malware Detect malwareexpert_enabled="yes" # Malware Expert malwarepatrol_enabled="yes" # Malware Patrol @@ -329,6 +331,23 @@ rfxn.hdb|LOW # MD5 Malware detection signatures rfxn.yara|LOW # Yara Malware detection signatures ) #END LINUXMALWAREDETECT DATABASES +# ======================== +# interServer Database(s) +# ======================== +# Add or remove database file names between quote marks as needed. To +# disable any Malware Expert database downloads, remove the appropriate +# lines below. +declare -a interserver_dbs=( +## REQUIRED, Do NOT disable +whitelist.fp|REQUIRED # found to be false positive malware +# LOW +interserver256.hdb|LOW # 100% known malware sha256 format +# MEDIUM +interservertopline.db|MEDIUM # inserts into files, manual cleaning HEX +# HIGH +shell.ldb|HIGH # 99.9% known malware using logical signatures +) #END Malware Expert DATABASES + # ======================== # Malware Expert Database(s) # ======================== @@ -336,10 +355,11 @@ rfxn.yara|LOW # Yara Malware detection signatures # disable any Malware Expert database downloads, remove the appropriate # lines below. declare -a malwareexpert_dbs=( +## REQUIRED, Do NOT disable +malware.expert.fp|REQUIRED # found to be false positive malware # LOW -malware.expert.hdb|MEDIUM # statics MD5 pattern for files +malware.expert.hdb|LOW # statics MD5 pattern for files # MEDIUM -malware.expert.fp|MEDIUM # found to be false positive malware malware.expert.ldb|MEDIUM # which use multi-words search for malware in files malware.expert.ndb|MEDIUM # Generic Hex pattern PHP malware, which can cause false positive alarms ) #END Malware Expert DATABASES @@ -607,9 +627,10 @@ downloader_tries="5" # Sub-directory names: add_dir="dbs-add" # User defined databases sub-directory gpg_dir="gpg-key" # Sanesecurity GPG Key sub-directory +interserver_dir="dbs-is" # interServer sub-directory linuxmalwaredetect_dir="dbs-lmd" # Linux Malware Detect sub-directory +malwareexpert_dir="dbs-me" # Malware Expert sub-directory malwarepatrol_dir="dbs-mbl" # MalwarePatrol sub-directory -malwareexpert_dir="dbs-me" # Malware Expertt sub-directory pid_dir="pid" # User defined pid sub-directory sanesecurity_dir="dbs-ss" # Sanesecurity sub-directory securiteinfo_dir="dbs-si" # SecuriteInfo sub-directory @@ -679,6 +700,7 @@ selinux_fixes="no" # Default is "no" ignore ssl errors and warnings # the automatic detection and generation of the values when not set, this is mainly to aid package maintainers #work_dir_add="" #default: uses work_dir/add_dir #work_dir_gpg="" #default: uses work_dir/gpg_dir +#work_dir_interserver="" #default: uses work_dir/interserver_dir #work_dir_linuxmalwaredetect="" #default: uses work_dir/linuxmalwaredetect_dir #work_dir_malwareexpert="" #default: uses work_dir/malwareexpert_dir #work_dir_malwarepatrol="" #default: uses work_dir/malwarepatrol_dir @@ -696,6 +718,7 @@ user_configuration_complete="no" # ======================== # DO NOT EDIT ! # Database provider URLs +interserver_url="https://sigs.interserver.net" linuxmalwaredetect_sigpack_url="https://cdn.rfxn.com/downloads/maldet-sigpack.tgz" linuxmalwaredetect_version_url="https://cdn.rfxn.com/downloads/maldet.sigs.ver" malwareexpert_url="https://signatures.malware.expert" @@ -708,7 +731,7 @@ yararulesproject_url="https://raw.githubusercontent.com/Yara-Rules/rules/master" # ======================== # DO NOT EDIT ! -config_version="93" +config_version="94" ################################################################################ # diff --git a/config/user.conf b/config/user.conf index d7663d6e..8e957456 100644 --- a/config/user.conf +++ b/config/user.conf @@ -44,6 +44,7 @@ # Per Database # These ratings will override the global rating for the specific database # valid rating: LOW, MEDIUM, HIGH, DISABLE +#interserver_dbs_rating="HIGH" #linuxmalwaredetect_dbs_rating="HIGH" #malwareexpert_dbs_rating="HIGH" #sanesecurity_dbs_rating="HIGH" From 9fc736f8da784d2878f383b8c4cb4977f73e3c7d Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Mon, 7 Dec 2020 10:08:45 +0200 Subject: [PATCH 098/101] antidebug_antivm/antidebug_antivm.yar DISABLED # (core dumped) --- config/master.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/master.conf b/config/master.conf index 76e4b5b3..5666301b 100644 --- a/config/master.conf +++ b/config/master.conf @@ -389,7 +389,7 @@ declare -a yararulesproject_dbs=( # use subdir/file # LOW # Anti debug and anti virtualization techniques used by malware -antidebug_antivm/antidebug_antivm.yar|LOW +antidebug_antivm/antidebug_antivm.yar|DISABLED # (core dumped) # Aimed toward the detection and existence of Exploit Kits. exploit_kits/EK_Angler.yar|DISABLED # duplicated in rxfn.yara exploit_kits/EK_Blackhole.yar|DISABLED # duplicated in rxfn.yara From 48d6bbc41c9fee2d76f3cabefe398c3c34a5e8f7 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Mon, 7 Dec 2020 10:12:42 +0200 Subject: [PATCH 099/101] minor fixes to docs --- README.md | 2 +- guides/macos.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f090b70b..60201ef3 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Debian, Ubuntu, Raspbian, CentOS (RHEL and clones), OpenBSD, FreeBSD, OpenSUSE, * CentOS: * Ubuntu: * Debian: -* Mac OSX: +* macOS: * pFsense: ### UPGRADE INSTRUCTIONS (version 7.0 +) diff --git a/guides/macos.md b/guides/macos.md index 1cb06fa1..05b236d5 100644 --- a/guides/macos.md +++ b/guides/macos.md @@ -1,4 +1,4 @@ -# Basic guide to Installing and Updating on Mac OS 10.12+ and OS X +# Basic guide to Installing and Updating on macOS and OSX Press Command+Space and type Terminal and press enter/return key. Run all the following in the Terminal app: From e6ca4150b79dd99377c385b7f114a8edb12fa3ae Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Mon, 7 Dec 2020 10:30:05 +0200 Subject: [PATCH 100/101] fix broken rules --- config/master.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/master.conf b/config/master.conf index 5666301b..be6de532 100644 --- a/config/master.conf +++ b/config/master.conf @@ -430,9 +430,9 @@ cve_rules/CVE-2018-4878.yar|MEDIUM # Identification of malicious e-mails. email/bank_rule.yar|MEDIUM email/EMAIL_Cryptowall.yar|MEDIUM -email/Email_fake_it_maintenance_bulletin|MEDIUM -email/Email_generic_phishing|MEDIUM -email/Email_quota_limit_warning|MEDIUM +email/Email_fake_it_maintenance_bulletin.yar|MEDIUM +email/Email_generic_phishing.yar|MEDIUM +email/Email_quota_limit_warning.yar|MEDIUM email/email_Ukraine_BE_powerattack.yar|MEDIUM email/scam.yar|MEDIUM # Detect well-known software packers, that can be used by malware to hide itself. From 335b1feba6aa6a12dd8670bfdb36d79396752fca Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Mon, 7 Dec 2020 10:32:47 +0200 Subject: [PATCH 101/101] tidy messages --- clamav-unofficial-sigs.sh | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/clamav-unofficial-sigs.sh b/clamav-unofficial-sigs.sh index 8d676c6d..13b47a9d 100644 --- a/clamav-unofficial-sigs.sh +++ b/clamav-unofficial-sigs.sh @@ -2985,7 +2985,7 @@ if [ "$sanesecurity_enabled" == "yes" ] ; then fi done if [ ! "$sanesecurity_update" == "1" ] ; then - xshok_pretty_echo_and_log "No Sanesecurity database file updates found" "-" + xshok_pretty_echo_and_log "No Sanesecurity database file updates" "-" break else break @@ -3132,11 +3132,11 @@ if [ "$securiteinfo_enabled" == "yes" ] ; then xshok_pretty_echo_and_log "Failed connection to ${securiteinfo_url} - SKIPPED SecuriteInfo ${db_file} update" fi if [ "$securiteinfo_db_update" != "1" ] ; then - xshok_pretty_echo_and_log "No updated SecuriteInfo ${db_file} database file found" "-" + xshok_pretty_echo_and_log "No updated SecuriteInfo ${db_file} database file" "-" fi done if [ "$securiteinfo_updates" != "1" ] ; then - xshok_pretty_echo_and_log "No SecuriteInfo database file updates found" "-" + xshok_pretty_echo_and_log "No SecuriteInfo database file updates" "-" fi else xshok_pretty_echo_and_log "SecuriteInfo Database File Updates" "=" @@ -3294,7 +3294,7 @@ if [ "$linuxmalwaredetect_enabled" == "yes" ] ; then xshok_pretty_echo_and_log "WARNING: Failed connection to ${linuxmalwaredetect_sigpack_url} - SKIPPED LinuxMalwareDetect update" fi else - xshok_pretty_echo_and_log "No LinuxMalwareDetect database file updates found" "-" + xshok_pretty_echo_and_log "No LinuxMalwareDetect database file updates" "-" fi else xshok_pretty_echo_and_log "LinuxMalwareDetect Database File Updates" "=" @@ -3433,11 +3433,11 @@ if [ "$interserver_enabled" == "yes" ] ; then xshok_pretty_echo_and_log "Failed connection to ${interserver_url} - SKIPPED interServer ${db_file} update" fi if [ "$interserver_db_update" != "1" ] ; then - xshok_pretty_echo_and_log "No updated interServer ${db_file} database file found" "-" + xshok_pretty_echo_and_log "No updated interServer ${db_file} database file" "-" fi done if [ "$interserver_updates" != "1" ] ; then - xshok_pretty_echo_and_log "No interServer database file updates found" "-" + xshok_pretty_echo_and_log "No interServer database file updates" "-" fi else xshok_pretty_echo_and_log "interServer Database File Updates" "=" @@ -3574,11 +3574,11 @@ if [ "$malwareexpert_enabled" == "yes" ] ; then xshok_pretty_echo_and_log "Failed connection to ${malwareexpert_url} - SKIPPED Malware Expert ${db_file} update" fi if [ "$malwareexpert_db_update" != "1" ] ; then - xshok_pretty_echo_and_log "No updated Malware Expert ${db_file} database file found" "-" + xshok_pretty_echo_and_log "No updated Malware Expert ${db_file} database file" "-" fi done if [ "$malwareexpert_updates" != "1" ] ; then - xshok_pretty_echo_and_log "No Malware Expert database file updates found" "-" + xshok_pretty_echo_and_log "No Malware Expert database file updates" "-" fi else xshok_pretty_echo_and_log "Malware Expert Database File Updates" "=" @@ -3718,10 +3718,10 @@ if [ "$malwarepatrol_enabled" == "yes" ] ; then xshok_pretty_echo_and_log "Failed connection to ${malwarepatrol_url} - SKIPPED MalwarePatrol ${malwarepatrol_db} update" fi if [ "$malwarepatrol_db_update" != "1" ] ; then - xshok_pretty_echo_and_log "No updated MalwarePatrol ${malwarepatrol_db} database file found" "-" + xshok_pretty_echo_and_log "No updated MalwarePatrol ${malwarepatrol_db} database file" "-" fi if [ "$malwarepatrol_updates" != "1" ] ; then - xshok_pretty_echo_and_log "No MalwarePatrol database file updates found" "-" + xshok_pretty_echo_and_log "No MalwarePatrol database file updates" "-" fi else xshok_pretty_echo_and_log "MalwarePatrol Database File Updates" "=" @@ -3847,11 +3847,11 @@ if [ "$urlhaus_enabled" == "yes" ] ; then xshok_pretty_echo_and_log "WARNING: Failed connection to $urlhaus_url - SKIPPED urlhaus ${db_file} update" fi if [ "$urlhaus_db_update" != "1" ] ; then - xshok_pretty_echo_and_log "No updated urlhaus ${db_file} database file found" + xshok_pretty_echo_and_log "No updated urlhaus ${db_file} database file" fi done if [ "$urlhaus_updates" != "1" ] ; then - xshok_pretty_echo_and_log "No urlhaus database file updates found" "-" + xshok_pretty_echo_and_log "No urlhaus database file updates" "-" fi else @@ -3984,11 +3984,11 @@ if [ "$yararulesproject_enabled" == "yes" ] ; then xshok_pretty_echo_and_log "WARNING: Failed connection to $yararulesproject_url - SKIPPED yararulesproject ${db_file} update" fi if [ "$yararulesproject_db_update" != "1" ] ; then - xshok_pretty_echo_and_log "No updated yararulesproject ${db_file} database file found" + xshok_pretty_echo_and_log "No updated yararulesproject ${db_file} database file" fi done if [ "$yararulesproject_updates" != "1" ] ; then - xshok_pretty_echo_and_log "No yararulesproject database file updates found" "-" + xshok_pretty_echo_and_log "No yararulesproject database file updates" "-" fi else @@ -4147,11 +4147,11 @@ if [ "$additional_enabled" == "yes" ] ; then xshok_pretty_echo_and_log "WARNING: Failed connection to ${db_url} - SKIPPED additional ${db_file} update" fi if [ "$additional_db_update" != "1" ] ; then - xshok_pretty_echo_and_log "No updated additional ${db_file} database file found" + xshok_pretty_echo_and_log "No updated additional ${db_file} database file" fi done if [ "$additional_updates" != "1" ] ; then - xshok_pretty_echo_and_log "No additional database file updates found" "-" + xshok_pretty_echo_and_log "No additional database file updates" "-" fi else xshok_pretty_echo_and_log "Additional Database File Updates" "="