postwhite

#!/bin/sh
#
###################################################################
# Postwhite - Automatic Postcreen Whitelist / Blacklist Generator #
# https://github.com/stevejenkins/postwhite                       #
# By Steve Jenkins (https://www.stevejenkins.com/)                #
###################################################################

version="3.4"
lastupdated="14 April 2018"

# Usage: 1) Place entire /postwhite directory in /usr/local/bin
#	 2) Move postwhite.conf to /etc
#	 3) Run ./postwhite [config-file]
# Optional config file passed via command line overrides the default config file location.
#
# Requires SPF-Tools (https://github.com/jsarenik/spf-tools)
# Please update your copy of spf-tools whenever you update Postwhite
#
# Thanks to Mike Miller (mmiller@mgm51.com) for gwhitelist.sh script.
# Thanks to Jan Sarenik for SPF-Tools.
# Thanks to Jose Borges Ferreira for IPv4 normalization help.
# Thanks to Ricardo Iván Vieitez Parra for improved error reporting, normalization, conf file
#           improvements, and removal of bash-isms so that script is usable on more systems.
# Thanks to Steve Cook for Yahoo! IP scraping help.
# Thanks to all the additional contributors on GitHub!
#
# USER-DEFINABLE OPTIONS AND CUSTOM HOSTS STORED IN /etc/postwhite.conf
# CONFIGURATION FILE CAN ALSO BE PASSED FROM COMMAND LINE
#
# NO NEED TO EDIT PAST THIS LINE
#
#################################################################

# DEFAULT HOSTS
# CUSTOM HOSTS CAN BE ADDED IN /etc/postwhite.conf

# Hosts to query
webmail_hosts="aol.com fastmail.com google.com gmx.com hotmail.com icloud.com \
	inbox.com mail.com microsoft.com outlook.com secure-mailgate.com zoho.com"

social_hosts="facebook.com facebookmail.com instagram.com linkedin.com \
	pinterest.com reddit.com tumblr.com twitter.com"

commerce_hosts="amazon.com craigslist.org ebay.com paypal.com"

bulk_hosts="amazonses.com authsmtp.com constantcontact.com \
	cust-spf.exacttarget.com exacttarget.com fbmta.com fishbowl.com \
	icontact.com mailchimp.com mailgun.com mailjet.com messagelabs.com \
	messagegears.net sendgrid.com sendgrid.net sparkpost.com sparkpostmail.com"

misc_hosts="github.com zendesk.com"

permit_line_v4="%s\tpermit\n"
reject_line_v4="%s\treject\n"

permit_line_v6="${permit_line_v4}"
reject_line_v6="${reject_line_v4}"

# Abort script on error (FYI: enabling will cause script to exit silently if mailer has no valid results)
set -e

printf "Starting Postwhite v$version ($lastupdated)\n"

# Check for passed config file
config_file="/etc/postwhite.conf"
if [ -n "$1" ]; then
	config_file="$1"
fi

# Read config file options
if [ ! -s $config_file ] ; then
	printf "%s: Can't find %s. Exiting.\n" "$0" "$config_file" 1>&2
	exit 1
fi
printf "\nReading options from %s...\n" "$config_file"
. "${config_file}"

# Create temporary files
printf "\nCreating temporary files...\n"
tmpBase=$(basename "$0")

tmpPrefix="tmp"
if [ x"$enable_blacklist" = x"yes" ]; then
	tmpPrefix="tmp blktmp"
fi

for p in $tmpPrefix; do
	for i in 1 2 3 4 5; do
		t="$(mktemp -q /tmp/"${tmpBase}".XXXXXX)"
		if [ $? -ne 0 ]; then
			>&2 printf "%s: Can't create temp files, exiting...\n" "$0"
			cleanup
			exit 1
		fi
		eval ${p}${i}="$t"
	done
done

# Create IPv4 normalize function
ip2int() {
	for i in 1 2 3 4 5; do
		eval ip2int_${i}="$(echo "$1" | cut -s -d. -f$i)"
	done
	ip2int_valid_range="$((ip2int_1 | ip2int_2 | ip2int_3 | ip2int_4))"
	[ -z "$ip2int_5" ] && [ -n "$ip2int_1" ] && [ -n "$ip2int_2" ] && [ -n "$ip2int_3" ] && [ -n "$ip2int_4" ] && [ "$ip2int_valid_range" -ge 0 ] && [ "$ip2int_valid_range" -le 255 ] &&
		printf "%d" "$(((((((ip2int_1 << 8) | ip2int_2) << 8) | ip2int_3) << 8) | ip2int_4))"
}

int2ip() {
	int2ip_ui32="$1"; shift
	int2ip_ip=""
	[ -n "$int2ip_ui32" ] && [ "$int2ip_ui32" -ge 0 ] && [ "$int2ip_ui32" -le 4294967295 ] && for n in 1 2 3 4; do
		int2ip_ip="$((int2ip_ui32 & 0xff))${int2ip_ip:+.}$int2ip_ip"
		int2ip_ui32="$((int2ip_ui32 >> 8))"
	done
	printf "%s" "${int2ip_ip}"
}

network_v4() {
	network_v4_ia="$(echo "$1" | cut -d/ -f1)"
	network_v4_netmask="$(echo "$1" | cut -d/ -f2-)"
	if [ -n "$network_v4_netmask" ] && [ "$network_v4_netmask" -ge 0 ] && [ "$network_v4_netmask" -le 32 ]; then
		network_v4_addr="$(ip2int $network_v4_ia)"
		network_v4_mask="$((0xffffffff << (32 - network_v4_netmask)))"
		if [ "$network_v4_netmask" -eq 32 ]; then
			printf "%s" $(int2ip $((network_v4_addr & network_v4_mask)))
		else
			printf "%s/%s" "$(int2ip $((network_v4_addr & network_v4_mask)))" "$network_v4_netmask"
		fi
	fi
}

network_v6() {
	printf "%s" "$1"
}

normalize_ip() {
	# split by ":"
	normalize_ip_type="$( echo $1 | cut -s -d\: -f1)"
	normalize_ip_value="$( echo $1 | cut -s -d\: -f2-)"
	normalize_ip_IP=""
	if [ x"${normalize_ip_type}" = x"ip4" ] ; then
		# check if is a CIDR
		if expr "x${normalize_ip_value}" : "x.*/[0-9]*" > /dev/null; then
			normalize_ip_IP="$(network_v4 "${normalize_ip_value}")"
		else
			normalize_ip_IP="$(network_v4 "${normalize_ip_value}/32")"
		fi
	elif [ x"${normalize_ip_type}" = x"ip6" ]; then
		normalize_ip_IP="$(network_v6 "${normalize_ip_value}")"
	fi
	printf "%s" "$normalize_ip_IP"
}

# Create host query function
query_host() {
	"${spftoolspath}"/despf.sh "$1" | (grep -Ei ^ip || true ) >> "${tmp1}"
}

query_black_host() {
	"${spftoolspath}"/despf.sh "$1" | (grep -Ei ^ip || true ) >> "${blktmp1}"
}

# Create progress dots function
show_dots() {
	while ps "$1" >/dev/null; do
		printf "."
		sleep 1
	done
	printf "\n"
}

# Fix mode
fix_invalid_ip() {
	fix_invalid_ip_iptype="$( echo "$1" | cut -d\: -f1 )"
	fix_invalid_ip_ip="$(normalize_ip "$1")"
	if [ -n "$fix_invalid_ip_ip" ]; then
		if [ x"$fix_invalid_ip_iptype" = x"ip4" ]; then
			printf "$(eval printf "%s" "\${${2}_line_v4}")" "$fix_invalid_ip_ip"
		elif [ x"$fix_invalid_ip_iptype" = x"ip6" ] ; then
			printf "$(eval printf "%s" "\${${2}_line_v6}")" "$fix_invalid_ip_ip"
		fi
	fi
}

# Remove mode
remove_invalid_ip() {
	remove_invalid_ip_iptype="$( echo "$1" | cut -d\: -f1 )"
	remove_invalid_ip_origip="$( echo "$1" | cut -d\: -f2- )"
	remove_invalid_ip_ip="$(normalize_ip "$1")"
	if [ x"$remove_invalid_ip_origip" = x"$remove_invalid_ip_ip" ]; then
		if [ x"$remove_invalid_ip_iptype" = x"ip4" ]; then
			printf "$(eval printf "%s" "\${${2}_line_v4}")" "$remove_invalid_ip_ip"
		elif [ x"$remove_invalid_ip_iptype" = x"ip6" ] ; then
			printf "$(eval printf "%s" "\${${2}_line_v6}")" "$remove_invalid_ip_ip"
		fi
	fi
}

# Keep mode
keep_invalid_ip() {
	keep_invalid_ip_iptype="$( echo "$1" | cut -d\: -f1 )"
	keep_invalid_ip_ip="$( echo "$1" | cut -d\: -f2- )"
	if [ -n "$keep_invalid_ip_ip" ]; then
		if [ x"$keep_invalid_ip_iptype" = x"ip4" ]; then
			printf "$(eval printf "%s" "\${${2}_line_v4}")" "$keep_invalid_ip_ip"
		elif [ x"$keep_invalid_ip_iptype" = x"ip6" ] ; then
			printf "$(eval printf "%s" "\${${2}_line_v6}")" "$keep_invalid_ip_ip"
		fi
	fi
}

# Let's DO this!

printf "\nRecursively querying SPF records of selected whitelist mailers...\n"

# Query selected mailers

printf "\nQuerying webmail hosts...\n"

for h in ${webmail_hosts}; do
	query_host "${h}"
done

printf "\nQuerying social network hosts...\n"

for h in ${social_hosts}; do
	query_host "${h}"
done

printf "\nQuerying ecommerce hosts...\n"

for h in ${commerce_hosts}; do
	query_host "${h}"
done

printf "\nQuerying bulk mail hosts...\n"

for h in ${bulk_hosts}; do
	query_host "${h}"
done

printf "\nQuerying miscellaneous hosts...\n"

for h in ${misc_hosts}; do
	query_host "${h}"
done

printf "\nQuerying custom hosts...\n"

for h in ${custom_hosts}; do
	query_host "${h}"
done

if [ x"$include_yahoo" = x"yes" ] ; then
	printf "\nIncluding scraped Yahoo! outbound hosts...\n"

	cat "${yahoo_static_hosts}" >> "${tmp1}"
fi

if [ x"$enable_blacklist" = x"yes" ] ; then
	printf "\nQuerying blacklist hosts...\n"

	for h in ${blacklist_hosts}; do
		query_black_host "${h}"
	done
fi

# If enabled, simplify (remove) any individual IPs already included in CIDR ranges (disabled by default)
if [ x"$simplify" = x"yes" ]; then
	printf "\nSimplifying whitelist IP addresses already included in CIDR ranges. These calculations\n"
	printf "can take a LONG time if you have many mailers selected. Please be patient..."
	
	sed '/\./s/\/32//g' "${tmp1}" | sort -u | "${spftoolspath}"/simplify.sh > "${tmp2}" &
	show_dots "$!"

	if [ x"$enable_blacklist" = x"yes" ] ; then
		printf "\nSimplifying blacklist IP addresses already included in CIDR ranges. These calculations\n"
			printf "can take a LONG time if you have many mailers selected. Please be patient..."
			cat "${blktmp1}" | sort -u | "${spftoolspath}"/simplify.sh > "${blktmp2}" &
			show_dots "$!"
	fi
	
	printf "\nIP address simplification complete.\n"
else
	sed '/\./s/\/32//g' "${tmp1}" > "${tmp2}"
	if [ x"$enable_blacklist" = x"yes" ] ; then
		cat "${blktmp1}" > "${blktmp2}"
	fi
fi

# Check for invalid IPv4 CIDRs, then format the whitelist

# If enabled, fix invalid CIDRs
if [ x"$invalid_ip4" = x"fix" ] ; then
	printf "\nFixing invalid whitelist IPv4 CIDRs..."
	cat "${tmp2}" | while read ip; do
		fix_invalid_ip "$ip" "permit"
	done >> "${tmp3}" &
	show_dots "$!"

	if [ x"$enable_blacklist" = x"yes" ] ; then
		printf "\nFixing invalid blacklist IPv4 CIDRs..."
		cat "${blktmp2}" | while read ip; do
			fix_invalid_ip "$ip" "reject"
		done >> "${blktmp3}" &
			show_dots "$!"
	fi

# If enabled, remove invalid CIDRs
elif [ x"$invalid_ip4" = x"remove" ] ; then
	printf "\nRemoving invalid IPv4 CIDRs from whitelist..."
	cat "${tmp2}" | while read ip; do
		remove_invalid_ip "$ip" "permit"
	done >> "${tmp3}" &
	show_dots "$!"

	if [ x"$enable_blacklist" = x"yes" ] ; then
		printf "\nRemoving invalid IPv4 CIDRs from blacklist..."
		cat "${blktmp2}" | while read ip; do
			remove_invalid_ip "$ip" "reject"
		done >> "${blktmp3}" &
		show_dots "$!"
	fi

# If enabled, keep invalid CIDRs
elif [ x"$invalid_ip4" = x"keep" ] ; then
	printf "\nKeeping invalid whitelist IPv4 CIDRs...\n"
	cat "${tmp2}" | while read ip; do
		keep_invalid_ip "$ip" "permit"
	done >> "${tmp3}" &
	show_dots "$!"

	if [ x"$enable_blacklist" = x"yes" ] ; then
		printf "\nKeeping invalid blacklist IPv4 CIDRs...\n"
		cat "${blktmp2}" | while read ip; do
			keep_invalid_ip "$ip" "reject"
		done >> "${blktmp3}" &
		show_dots "$!"
	fi
fi

# Sort, uniq, and count final rules
# Have to do sort and uniq separately, as 'sort -u -t. -k1,1n...' removes valid rules
printf "\nSorting whitelist rules...\n"
sort -t. -k1,1n -k2,2n -k3,3n -k4,4n "${tmp3}" > "${tmp4}"
uniq "${tmp4}" >> "${tmp5}"
numrules="$(cat "${tmp5}" | wc -l)"

if [ x"$enable_blacklist" = x"yes" ] ; then
	printf "\nSorting blacklist rules...\n"
	sort -t. -k1,1n -k2,2n -k3,3n -k4,4n "${blktmp3}" > "${blktmp4}"
	uniq "${blktmp4}" >> "${blktmp5}"
	numblackrules="$(cat "${blktmp5}" | wc -l)"
fi

# Write whitelist and blacklist to Postfix directory
printf "\nWriting $numrules whitelist rules to ${postfixpath}/${whitelist}...\n"
printf "# Whitelist generated by Postwhite v$version on $(date)\n# https://github.com/stevejenkins/postwhite/\n# $numrules total rules\n" > "${postfixpath}"/"${whitelist}"
cat "${tmp5}" >> "${postfixpath}"/"${whitelist}"

if [ x"$enable_blacklist" = x"yes" ] ; then
	printf "\nWriting $numblackrules blacklist rules to ${postfixpath}/${blacklist}...\n"
	printf "# Blacklist generated by Postwhite v$version on $(date)\n# https://github.com/stevejenkins/postwhite/\n# $numblackrules total rules\n" > "${postfixpath}"/"${blacklist}"
	cat "${blktmp5}" >> "${postfixpath}"/"${blacklist}"
fi

# Remove temp files
cleanup() {
	test -e "${tmp1}" && rm "${tmp1}"
	test -e "${tmp2}" && rm "${tmp2}"
	test -e "${tmp3}" && rm "${tmp3}"
	test -e "${tmp4}" && rm "${tmp4}"
	test -e "${tmp5}" && rm "${tmp5}"
	if [ x"$enable_blacklist" = x"yes" ] ; then
		test -e "${blktmp1}" && rm "${blktmp1}"
		test -e "${blktmp2}" && rm "${blktmp2}"
		test -e "${blktmp3}" && rm "${blktmp3}"
		test -e "${blktmp4}" && rm "${blktmp4}"
		test -e "${blktmp5}" && rm "${blktmp5}"
	fi
}
cleanup

# Reload Postfix to pick up changes in whitelist
if [ x"$reload_postfix" = x"yes" ]; then
printf '\nReloading Postfix configuration to refresh rules...\n'
	${postfixbinarypath}/postfix reload
fi

printf '\nDone!\n'

exit