diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index cb49439ba5..fe28112c8d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.2 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6676d5a1cd..f68b756ab9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,7 @@ jobs: debian_11, ubuntu_20, ubuntu_22, + ubuntu_23, centos_8, centos_9, fedora_36, @@ -67,7 +68,7 @@ jobs: uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.6.1 with: python-version: "3.10" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index fa1cebbb7f..6f747855d9 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -230,10 +230,8 @@ initialize_debug() { # This is a function for visually displaying the current test that is being run. # Accepts one variable: the name of what is being diagnosed -# Colors do not show in the dashboard, but the icons do: [i], [✓], and [✗] echo_current_diagnostic() { # Colors are used for visually distinguishing each test in the output - # These colors do not show in the GUI, but the formatting will log_write "\\n${COL_PURPLE}*** [ DIAGNOSING ]:${COL_NC} ${1}" } diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 12295fbcfc..1d3b0a29a0 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -69,31 +69,16 @@ if [[ -n "${str:-}" ]]; then exit 1 fi -# Scan an array of files for matching strings -scanList(){ - # Escape full stops - local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" list_type="${3:-}" - - # Prevent grep from printing file path - cd "$piholeDir" || exit 1 - - # Prevent grep -i matching slowly: https://bit.ly/2xFXtUX - export LC_CTYPE=C - - # /dev/null forces filename to be printed when only one list has been generated - case "${list_type}" in - "exact" ) grep -i -E -l "(^|(?/dev/null;; - # Iterate through each regexp and check whether it matches the domainQuery - # If it does, print the matching regexp and continue looping - # Input 1 - regexps | Input 2 - domainQuery - "regex" ) - for list in ${lists}; do - if [[ "${domain}" =~ ${list} ]]; then - printf "%b\n" "${list}"; - fi - done;; - * ) grep -i "${esc_domain}" "${lists}" /dev/null 2>/dev/null;; - esac +# Scan a domain again a list of RegEX +scanRegExList(){ + local domain="${1}" list="${2}" + + for entry in ${list}; do + if [[ "${domain}" =~ ${entry} ]]; then + printf "%b\n" "${entry}"; + fi + done + } scanDatabaseTable() { @@ -188,7 +173,7 @@ scanRegexDatabaseTable() { # Split regexps over a new line str_regexList=$(printf '%s\n' "${regexList[@]}") # Check domain against regexps - mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex") + mapfile -t regexMatches < <(scanRegExList "${domain}" "${str_regexList}") # If there were regex matches if [[ "${#regexMatches[@]}" -ne 0 ]]; then # Split matching regexps over a new line diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 5ccdf733f6..7680590f81 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -22,12 +22,14 @@ readonly dnscustomcnamefile="/etc/dnsmasq.d/05-pihole-custom-cname.conf" readonly gravityDBfile="/etc/pihole/gravity.db" -# Source install script for ${setupVars}, ${PI_HOLE_BIN_DIR} and valid_ip() -readonly PI_HOLE_FILES_DIR="/etc/.pihole" -# shellcheck disable=SC2034 # used in basic-install to source the script without running it -SKIP_INSTALL="true" -source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" +readonly setupVars="/etc/pihole/setupVars.conf" +readonly PI_HOLE_BIN_DIR="/usr/local/bin" + +# Root of the web server +readonly webroot="/var/www/html" + +# Source utils script utilsfile="/opt/pihole/utils.sh" source "${utilsfile}" @@ -98,6 +100,47 @@ HashPassword() { echo "${return}" } +# Check an IP address to see if it is a valid one +valid_ip() { + # Local, named variables + local ip=${1} + local stat=1 + + # Regex matching one IPv4 component, i.e. an integer from 0 to 255. + # See https://tools.ietf.org/html/rfc1340 + local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|0)"; + # Regex matching an optional port (starting with '#') range of 1-65536 + local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; + # Build a full IPv4 regex from the above subexpressions + local regex="^${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}${portelem}$" + + # Evaluate the regex, and return the result + [[ $ip =~ ${regex} ]] + + stat=$? + return "${stat}" +} + +valid_ip6() { + local ip=${1} + local stat=1 + + # Regex matching one IPv6 element, i.e. a hex value from 0000 to FFFF + local ipv6elem="[0-9a-fA-F]{1,4}" + # Regex matching an IPv6 CIDR, i.e. 1 to 128 + local v6cidr="(\\/([1-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])){0,1}" + # Regex matching an optional port (starting with '#') range of 1-65536 + local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; + # Build a full IPv6 regex from the above subexpressions + local regex="^(((${ipv6elem}))*((:${ipv6elem}))*::((${ipv6elem}))*((:${ipv6elem}))*|((${ipv6elem}))((:${ipv6elem})){7})${v6cidr}${portelem}$" + + # Evaluate the regex, and return the result + [[ ${ip} =~ ${regex} ]] + + stat=$? + return "${stat}" +} + SetWebPassword() { if [ "${SUDO_USER}" == "www-data" ]; then echo "Security measure: user www-data is not allowed to change webUI password!" @@ -613,7 +656,6 @@ Teleporter() { host="${host//./_}" filename="pi-hole-${host:-noname}-teleporter_${datetimestamp}.tar.gz" fi - # webroot is sourced from basic-install above php "${webroot}/admin/scripts/pi-hole/php/teleporter.php" > "${filename}" } @@ -622,7 +664,7 @@ checkDomain() local domain validDomain # Convert to lowercase domain="${1,,}" - validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^((-|_)*[a-z0-9]((-|_)*[a-z0-9)*(-|_)*)(\\.(-|_)*([a-z0-9]((-|_)*[a-z0-9])*))*$" <<< "${domain}") # Valid chars check validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label echo "${validDomain}" } diff --git a/advanced/Templates/gravity_copy.sql b/advanced/Templates/gravity_copy.sql index 3bea731d1b..ed11b61a78 100644 --- a/advanced/Templates/gravity_copy.sql +++ b/advanced/Templates/gravity_copy.sql @@ -19,8 +19,6 @@ INSERT OR REPLACE INTO adlist SELECT * FROM OLD.adlist; DELETE FROM OLD.adlist_by_group WHERE adlist_id NOT IN (SELECT id FROM OLD.adlist); INSERT OR REPLACE INTO adlist_by_group SELECT * FROM OLD.adlist_by_group; -INSERT OR REPLACE INTO info SELECT * FROM OLD.info; - INSERT OR REPLACE INTO client SELECT * FROM OLD.client; DELETE FROM OLD.client_by_group WHERE client_id NOT IN (SELECT id FROM OLD.client); INSERT OR REPLACE INTO client_by_group SELECT * FROM OLD.client_by_group; diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 8afc777007..a8af830879 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -355,12 +355,17 @@ package_manager_detect() { PKG_MANAGER="zypper" fi - # These variable names match the ones for apt-get. See above for an explanation of what they are for. - PKG_INSTALL=("${PKG_MANAGER}" install -y) - OS_CHECK_DEPS=(grep bind-utils) - LIGHTTPD_USER="lighttpd" - LIGHTTPD_GROUP="lighttpd" - LIGHTTPD_CFG="lighttpd.conf.fedora" + # These variable names match the ones for apt-get. See above for an explanation of what they are for. + PKG_INSTALL=("${PKG_MANAGER}" install -y) + # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting. + PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" + OS_CHECK_DEPS=(grep bind-utils) + INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates) + PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) + PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl) + LIGHTTPD_USER="lighttpd" + LIGHTTPD_GROUP="lighttpd" + LIGHTTPD_CFG="lighttpd.conf.fedora" if [ "${PKG_MANAGER}" = "dnf" ] || [ "${PKG_MANAGER}" = "yum" ] ; then # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting. @@ -2376,6 +2381,9 @@ get_binary_name() { # set the binary to be used l_binary="pihole-FTL-linux-x86_64" fi + elif [[ "${machine}" == "riscv64" ]]; then + printf "%b %b Detected riscv64 processor\\n" "${OVER}" "${TICK}" + l_binary="pihole-FTL-riscv64-linux-gnu" else # Something else - we try to use 32bit executable and warn the user if [[ ! "${machine}" == "i686" ]]; then diff --git a/gravity.sh b/gravity.sh index 5cb6885060..fe2097dd25 100755 --- a/gravity.sh +++ b/gravity.sh @@ -129,7 +129,7 @@ gravity_swap_databases() { echo -e "${OVER} ${TICK} ${str}" if $oldAvail; then - echo -e " ${TICK} The old database remains available." + echo -e " ${TICK} The old database remains available" fi } @@ -145,18 +145,6 @@ update_gravity_timestamp() { return 0 } -# Update timestamp when the gravity table was last updated successfully -set_abp_info() { - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('abp_domains',${abp_domains});" - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to update ABP domain status in database ${gravityDBfile}\\n ${output}" - return 1 - fi - return 0 -} - # Import domains from file and store them in the specified database table database_table_from_file() { # Define locals @@ -165,7 +153,10 @@ database_table_from_file() { src="${2}" backup_path="${piholeDir}/migration_backup" backup_file="${backup_path}/$(basename "${2}")" - tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".gravity")" + # Create a temporary file. We don't use '--suffix' here because not all + # implementations of mktemp support it, e.g. on Alpine + tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}")" + mv "${tmpFile}" "${tmpFile%.*}.gravity" local timestamp timestamp="$(date --utc +'%s')" @@ -236,17 +227,6 @@ database_table_from_file() { echo -e " ${CROSS} Unable to remove ${tmpFile}" } -# Update timestamp of last update of this list. We store this in the "old" database as all values in the new database will later be overwritten -database_adlist_updated() { - output=$( { printf ".timeout 30000\\nUPDATE adlist SET date_updated = (cast(strftime('%%s', 'now') as int)) WHERE id = %i;\\n" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to update timestamp of adlist with ID ${1} in database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" - fi -} - # Check if a column with name ${2} exists in gravity table with name ${1} gravity_column_exists() { output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) @@ -264,7 +244,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_domains}" "${num_non_domains}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -438,7 +418,24 @@ gravity_DownloadBlocklists() { echo -e "${OVER} ${TICK} ${str}" fi - target="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".gravity")" + str="Creating new gravity databases" + echo -ne " ${INFO} ${str}..." + + # Gravity copying SQL script + copyGravity="$(cat "${gravityDBcopy}")" + if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then + # Replace default gravity script location by custom location + copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" + fi + + output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" + return 1 + fi + echo -e "${OVER} ${TICK} ${str}" # Use compression to reduce the amount of data that is transferred # between the Pi-hole and the ad list provider. Use this feature @@ -480,114 +477,9 @@ gravity_DownloadBlocklists() { echo "" done - str="Creating new gravity databases" - echo -ne " ${INFO} ${str}..." - - # Gravity copying SQL script - copyGravity="$(cat "${gravityDBcopy}")" - if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then - # Replace default gravity script location by custom location - copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" - fi - - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" - return 1 - fi - echo -e "${OVER} ${TICK} ${str}" - - str="Storing downloaded domains in new gravity database" - echo -ne " ${INFO} ${str}..." - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" gravity\\n" "${target}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to fill gravity table in database ${gravityTEMPfile}\\n ${output}" - gravity_Cleanup "error" - else - echo -e "${OVER} ${TICK} ${str}" - fi - - if [[ "${status}" -eq 0 && -n "${output}" ]]; then - echo -e " Encountered non-critical SQL warnings. Please check the suitability of the lists you're using!\\n\\n SQL warnings:" - local warning file line lineno - while IFS= read -r line; do - echo " - ${line}" - warning="$(grep -oh "^[^:]*:[0-9]*" <<< "${line}")" - file="${warning%:*}" - lineno="${warning#*:}" - if [[ -n "${file}" && -n "${lineno}" ]]; then - echo -n " Line contains: " - awk "NR==${lineno}" < "${file}" - fi - done <<< "${output}" - echo "" - fi - - rm "${target}" > /dev/null 2>&1 || \ - echo -e " ${CROSS} Unable to remove ${target}" - gravity_Blackbody=true } - -# global variable to indicate if we found ABP style domains during the gravity run -# is saved in gravtiy's info table to signal FTL if such domains are available -abp_domains=0 -parseList() { - local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains valid_domain_pattern abp_domain_pattern - - # define valid domain patterns - # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already - # adapted from https://stackoverflow.com/a/30007882 - - TLD_pattern="[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" - subdomain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)" - - valid_domain_pattern="${subdomain_pattern}+${TLD_pattern}" - - # supported ABP style: ||subdomain.domain.tld^ - # ${subdomain_pattern} is optional for ABP style, allowing TLD blocking: ||tld^ - abp_domain_pattern="\|\|${subdomain_pattern}*${TLD_pattern}\^" - - # A list of items of common local hostnames not to report as unusable - # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files - # but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output - false_positives="localhost|localhost.localdomain|local|broadcasthost|localhost|ip6-localhost|ip6-loopback|lo0 localhost|ip6-localnet|ip6-mcastprefix|ip6-allnodes|ip6-allrouters|ip6-allhosts" - - # Extract valid domains from source file and append ,${adlistID} to each line and save count to variable for display. - num_domains=$(grep -E "^(${valid_domain_pattern}|${abp_domain_pattern})$" "${src}" | tee >(sed "s/$/,${adlistID}/" >> "${target}") | wc -l) - - # Check if the source file contained AdBlock Plus style domains, if so we set the global variable and inform the user - if grep -E "^${abp_domain_pattern}$" -m 1 -q "${src}"; then - echo " ${INFO} List contained AdBlock Plus style domains" - abp_domains=1 - fi - - # For completeness, we will get a count of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains) - invalid_domains="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".ph-non-domains")" - - num_non_domains=$(grep -Ev "^(${valid_domain_pattern}|${abp_domain_pattern}|${false_positives})$" "${src}" | tee "${invalid_domains}" | wc -l) - - # If there are unusable lines, we display some information about them. This is not error or major cause for concern. - if [[ "${num_non_domains}" -ne 0 ]]; then - type="domains" - if [[ "${abp_domains}" -ne 0 ]]; then - type="patterns" - fi - echo " ${INFO} Imported ${num_domains} ${type}, ignoring ${num_non_domains} non-domain entries" - echo " Sample of non-domain entries:" - invalid_lines=$(head -n 5 "${invalid_domains}") - echo "${invalid_lines}" | awk '{print " - " $0}' - else - echo " ${INFO} Imported ${num_domains} domains" - fi - rm "${invalid_domains}" -} - compareLists() { local adlistID="${1}" target="${2}" @@ -598,7 +490,6 @@ compareLists() { sha1sum "${target}" > "${target}.sha1" echo " ${INFO} List has been updated" database_adlist_status "${adlistID}" "1" - database_adlist_updated "${adlistID}" else echo " ${INFO} List stayed unchanged" database_adlist_status "${adlistID}" "2" @@ -608,7 +499,6 @@ compareLists() { sha1sum "${target}" > "${target}.sha1" # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" - database_adlist_updated "${adlistID}" fi } @@ -618,7 +508,9 @@ gravity_DownloadBlocklistFromUrl() { local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM - listCurlBuffer=$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".phgpb") + # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine + listCurlBuffer="$(mktemp -p "${GRAVITY_TMPDIR}")" + mv "${listCurlBuffer}" "${listCurlBuffer%.*}.phgpb" # Determine if $saveLocation has read permission if [[ -r "${saveLocation}" && $url != "file"* ]]; then @@ -706,9 +598,8 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - parseList "${adlistID}" "${saveLocation}" "${target}" + pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "2" - database_adlist_number "${adlistID}" done="true" # Check if $listCurlBuffer is a non-zero length file elif [[ -s "${listCurlBuffer}" ]]; then @@ -717,12 +608,9 @@ gravity_DownloadBlocklistFromUrl() { # Remove curl buffer file after its use rm "${listCurlBuffer}" # Add domains to database table file - parseList "${adlistID}" "${saveLocation}" "${target}" + pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" - # Update gravity database table (status and updated timestamp are set in - # compareLists) - database_adlist_number "${adlistID}" done="true" else # Fall back to previously cached list if $listCurlBuffer is empty @@ -736,15 +624,12 @@ gravity_DownloadBlocklistFromUrl() { if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file - parseList "${adlistID}" "${saveLocation}" "${target}" - database_adlist_number "${adlistID}" + pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" # Manually reset these two numbers because we do not call parseList here - num_domains=0 - num_non_domains=0 - database_adlist_number "${adlistID}" + database_adlist_number "${adlistID}" 0 0 database_adlist_status "${adlistID}" "4" fi fi @@ -762,25 +647,21 @@ gravity_ParseFileIntoDomains() { tr '[:upper:]' '[:lower:]' < "${src}" > "${destination}" # 2) Remove carriage returns - sed -i 's/\r$//' "${destination}" - - # 3a) Remove comments (text starting with "#", include possible spaces before the hash sign) - sed -i 's/\s*#.*//g' "${destination}" - - # 3b) Remove lines starting with ! (ABP Comments) - sed -i 's/\s*!.*//g' "${destination}" - - # 3c) Remove lines starting with [ (ABP Header) - sed -i 's/\s*\[.*//g' "${destination}" - - # 4) Remove lines containing "/" - sed -i -r '/(\/).*$/d' "${destination}" - - # 5) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) - sed -i -r 's/^.*\s+//g' "${destination}" - - # 6) Remove empty lines - sed -i '/^$/d' "${destination}" + # 3) Remove lines starting with ! (ABP Comments) + # 4) Remove lines starting with [ (ABP Header) + # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter + # 6) Remove comments (text starting with "#", include possible spaces before the hash sign) + # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) + # 8) Remove empty lines + + sed -i -r \ + -e 's/\r$//' \ + -e 's/\s*!.*//g' \ + -e 's/\s*\[.*//g' \ + -e '/[a-z]\#[$?@]{0,1}\#/d' \ + -e 's/\s*#.*//g' \ + -e 's/^.*\s+//g' \ + -e '/^$/d' "${destination}" chmod 644 "${destination}" } @@ -791,9 +672,9 @@ gravity_Table_Count() { local str="${2}" local num num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" - if [[ "${table}" == "vw_gravity" ]]; then + if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(DISTINCT domain) FROM ${table};")" + unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else @@ -803,7 +684,9 @@ gravity_Table_Count() { # Output count of blacklisted domains and regex filters gravity_ShowCount() { - gravity_Table_Count "vw_gravity" "gravity domains" "" + # Here we use the table "gravity" instead of the view "vw_gravity" for speed. + # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains. + gravity_Table_Count "gravity" "gravity domains" "" gravity_Table_Count "vw_blacklist" "exact blacklisted domains" gravity_Table_Count "vw_regex_blacklist" "regex blacklist filters" gravity_Table_Count "vw_whitelist" "exact whitelisted domains" @@ -1020,9 +903,6 @@ fi # Update gravity timestamp update_gravity_timestamp -# Set abp_domain info field -set_abp_info - # Ensure proper permissions are set for the database chown pihole:pihole "${gravityDBfile}" chmod g+w "${piholeDir}" "${gravityDBfile}" diff --git a/test/_ubuntu_23.Dockerfile b/test/_ubuntu_23.Dockerfile new file mode 100644 index 0000000000..f9b3910b85 --- /dev/null +++ b/test/_ubuntu_23.Dockerfile @@ -0,0 +1,18 @@ +FROM buildpack-deps:lunar-scm + +ENV GITDIR /etc/.pihole +ENV SCRIPTDIR /opt/pihole + +RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole +ADD . $GITDIR +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV DEBIAN_FRONTEND=noninteractive + +RUN true && \ + chmod +x $SCRIPTDIR/* + +ENV SKIP_INSTALL true +ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net + +#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/requirements.txt b/test/requirements.txt index 8cd3ca7797..f11d1253ec 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ docker-compose == 1.29.2 pytest == 7.3.1 -pytest-xdist == 3.2.1 -pytest-testinfra == 7.0.0 -tox == 4.4.12 +pytest-xdist == 3.3.1 +pytest-testinfra == 8.1.0 +tox == 4.5.1 diff --git a/test/tox.ubuntu_23.ini b/test/tox.ubuntu_23.ini new file mode 100644 index 0000000000..767ed9ef56 --- /dev/null +++ b/test/tox.ubuntu_23.ini @@ -0,0 +1,8 @@ +[tox] +envlist = py3 + +[testenv:py3] +allowlist_externals = docker +deps = -rrequirements.txt +commands = docker buildx build --load --progress plain -f _ubuntu_23.Dockerfile -t pytest_pihole:test_container ../ + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py