misspell-fixer

#!/usr/bin/env bash

function warning {
	echo "misspell-fixer: $*" >&2
}
export -f warning

function verbose {
	if [[ $opt_verbose = 1 ]]
	then
		warning "$@"
	fi
}
export -f verbose

function init_variables {
	set -f

	export LC_CTYPE=C
	export LANG=C

	export opt_debug=0
	export opt_verbose=0
	export opt_show_diff=0
	export opt_real_run=0
	export opt_backup=1
	export opt_dots=0
	export bash_arg
	
	export rule_file_base="${BASH_SOURCE[0]}"
	if [ ! -f $rule_file_base"-safe.0.sed" ]
	then
		rule_file_base=/usr/share/misspell-fixer/rules
	fi
	rules_safe0=$rule_file_base"-safe.0.sed"
	rules_safe1=$rule_file_base"-safe.1.sed"
	rules_safe2=$rule_file_base"-safe.2.sed"
	rules_safe3=$rule_file_base"-safe.3.sed"
	rules_not_so_safe=$rule_file_base"-not-so-safe.sed"
	rules_gb_to_us=$rule_file_base"-gb-to-us.sed"
	export enabled_rules="$rules_safe0"

	export cmd_part_ignore_scm="-o -iname .git -o -iname .svn -o -iname .hg -o -iname CVS"
	export cmd_part_ignore_bin="-o -iname *.gif -o -iname *.jpg -o -iname *.jpeg -o -iname *.png -o -iname *.zip -o -iname *.gz -o -iname *.bz2 -o -iname *.xz -o -iname *.rar -o -iname *.po -o -iname *.pdf -o -iname *.woff -o -iname yarn.lock  -o -iname package-lock.json  -o -iname composer.lock  -o -iname *.mo"
	export cmd_part_ignore

	export cmd_part_parallelism

	export loop_function=loop_main_replace
	export prefilter_progress_function=prefilter_progress_none

	export opt_name_filter=''
	export cmd_size="-and ( -size -1024k )"  # find will ignore files > 1MB
	export smart_sed

	export directories

	export tmpfile=.misspell-fixer.$$
}

function parse_basic_options {
	local OPTIND
	while getopts ":dvorfsibnRVDmughN:P:" opt; do
		case $opt in
			d)
				warning "-d Enable debug mode."
				opt_debug=1
				bash_arg=x
			;;
			v)
				warning "-v Enable verbose mode."
				opt_verbose=1
			;;
			o)
				warning "-o Print dots for each file scanned, comma for each file fix iteration/file."
				opt_dots=1
				prefilter_progress_function=prefilter_progress_dots
			;;
			r)
				warning "-r Enable real run. Overwrite original files!"
				opt_real_run=1
			;;
			f)
				warning "-f Enable fast mode. (Equivalent with -P4)"
				cmd_part_parallelism="-P 4"
			;;
			s)
				warning "-s Enable showing of diffs."
				opt_show_diff=1
			;;
			i)
				warning "-i Disable scm dir ignoring."
				cmd_part_ignore_scm=''
			;;
			b)
				warning "-i Disable binary ignoring."
				cmd_part_ignore_bin=''
			;;
			n)
				warning "-n Disable backups."
				opt_backup=0
			;;
			u)
				warning "-u Enable unsafe rules."
				enabled_rules="$enabled_rules $rules_not_so_safe"
			;;
			R)
				warning "-R Enable rare rules."
				enabled_rules="$enabled_rules $rules_safe1"
			;;
			V)
				warning "-V Enable very-rare rules."
				enabled_rules="$enabled_rules $rules_safe2"
			;;
			D)
				warning "-D Enable rules from lintian.debian.org / spelling."
				enabled_rules="$enabled_rules $rules_safe3"
			;;
			m)
				warning "-m Disable max-size check. Default is to ignore files > 1MB."
				cmd_size=" "
			;;
			g)
				warning "-g Enable GB to US rules."
				enabled_rules="$enabled_rules $rules_gb_to_us"
			;;
			N)
				warning "-N Enable name filter: $OPTARG"
				if [ -n "$opt_name_filter" ]; then
					opt_name_filter="$opt_name_filter -or -name $OPTARG"
				else
					opt_name_filter="-name $OPTARG"
				fi
			;;
			P)
				warning "-P Enable parallelism: $OPTARG"
				cmd_part_parallelism="-P $OPTARG"
			;;
			h)
				d="dirname ${BASH_SOURCE[0]}"
				if [[ -f "$($d)"/README.md ]]
				then
					cat "$($d)"/README.md
				else
					zcat /usr/share/doc/misspell-fixer/README.md.gz
				fi
				return 1
			;;
			\?)
				warning "Invalid option: -$OPTARG"
				return 1
			;;
			:)
				warning "Option -$OPTARG requires an argument."
				return 1
			;;
		esac
	done
	
	if [ -z "$opt_name_filter" ]; then
		opt_name_filter='-true'
	fi
	
	shift $((OPTIND-1))
	
	if [[ "$*" = "" ]]
	then
		warning "Not enough arguments. (target directory not found) => Exiting."
		return 1
	fi

	directories="$@"
	cmd_part_ignore="( -iname $tmpfile* -o -iname *.BAK $cmd_part_ignore_scm $cmd_part_ignore_bin ) -prune -o "
	warning "Target directories: $directories"

	return 0
}

function process_parameter_rules {
	if [[ 	$opt_show_diff = 1 ||\
		$opt_backup = 1 ||\
		$opt_real_run = 0 ||\
		$opt_verbose = 1 ]]
	then
		loop_function=loop_decorated_mode
	fi
	if [[ -z $cmd_part_parallelism ]]
	then
		return 0
	fi
	if [[ $opt_show_diff = 1 ]]
	then
		warning "Parallel mode cannot show diffs. Showing diffs is turned on. => Exiting."
		return 1
	fi
	return 0
}

function check_grep_version {
	local current_version=$(grep --version|head -1|sed -e 's/.* //g')
	local required_version="2.28"
	if printf '%s\n%s\n' "$required_version" "$current_version" | sort -VC
	then
		verbose "Your grep version is $current_version which is at least the optimal: $required_version."
	else
		warning "!! Your grep version is $current_version which is less than the optimal: $required_version. This may degrade misspell fixer's performance significantly! (100x) !!"
	fi
}

function check_sed_arguments {
	if sed -b 2>&1 |grep -q illegal
	then
		# OS/X
		smart_sed=0
	else
		# Everything else
		smart_sed=1
	fi
}

function preprocess_rules {
	grep -vh 'bin/sed' $enabled_rules > $tmpfile.prep.allsedrules
	grep    '\\b' $tmpfile.prep.allsedrules|sed -e 's/^s\///g' -e 's/\/.*g//g' -e 's/\\b//g'	>$tmpfile.prep.grep.rules.w
	grep -v '\\b' $tmpfile.prep.allsedrules|sed -e 's/^s\///g' -e 's/\/.*g//g'			>$tmpfile.prep.grep.rules
}

function prepare_prefilter_input_from_find {
	find $directories $cmd_part_ignore -type f -and \( $opt_name_filter \) $cmd_size -print0
}

function prepare_prefilter_input_from_cat {
	while read -r filename
	do
		printf '%s\0' "$filename"
	done <"$1"
}

function prefilter_progress_none {
	cat >/dev/null
}

function prefilter_progress_dots {
	while IFS= read -r -d '' filename
	do
		echo -n "." >&2
	done
}

function main_work {
	local input_function=$1
	local iteration=$2
	local prev_matched_files=$3
	local prev_matches=$4
	local itertmpfile=$tmpfile.$iteration

	warning "Iteration $iteration: prefiltering."
	
	$input_function $prev_matched_files|\
	tee >($prefilter_progress_function) \
		>(xargs -0 $cmd_part_parallelism -n 100 grep --text -F -noH     -f "$tmpfile.prep.grep.rules"   >"$itertmpfile.combos")|\
		  xargs -0 $cmd_part_parallelism -n 100 grep --text -F -noH -w  -f "$tmpfile.prep.grep.rules.w" >"$itertmpfile.combos.w"

	sort -u "$itertmpfile.combos" "$itertmpfile.combos.w" >"$itertmpfile.combos.all"

	if [[ $opt_verbose = 1 ]]
	then
		verbose "Results of prefiltering: (filename:line:pattern)"
		cat "$itertmpfile.combos.all" >&2
	fi

	if [[ -s $itertmpfile.combos.all ]]
	then
		grep --text -f <(cut -d ':' -f 3 "$itertmpfile.combos.all") "$tmpfile.prep.allsedrules" >"$itertmpfile.rulesmatched"
		warning "Iteration $iteration: replacing."
		cut -d ':' -f 1 "$itertmpfile.combos.all" |sort -u >"$itertmpfile.matchedfiles"
		xargs <"$itertmpfile.matchedfiles" $cmd_part_parallelism -n 1 -I '{}' $COVERAGE_WRAPPER bash -c$bash_arg "$loop_function $itertmpfile.combos.all $itertmpfile.rulesmatched '{}' 1"
		rm "$itertmpfile.rulesmatched"
		if [[ $opt_dots = 1 ]]
		then
			echo >&2
		fi
	else
		warning "Iteration $iteration: nothing to replace."
	fi
	warning "Iteration $iteration: done."
	if [[ $iteration -lt 5 && -s "$itertmpfile.combos.all" ]]
	then
		if diff --text -q "$prev_matches" "$itertmpfile.combos.all" >/dev/null
		then
			warning "Iteration $iteration: matchlist is the same as in previous iteration..."
		else
			main_work prepare_prefilter_input_from_cat $((iteration + 1)) "$itertmpfile.matchedfiles" "$itertmpfile.combos.all"
		fi
	fi
	if [[ -f $itertmpfile.matchedfiles ]]
	then
		rm "$itertmpfile.matchedfiles"
	fi
	rm "$itertmpfile.combos" "$itertmpfile.combos.w" "$itertmpfile.combos.all"
	return 0
}

function loop_main_replace {
	local findresult=$1
	local rulesmatched=$2
	local filename=$3
	local inplaceflag=$4

	if [[ $opt_dots = 1 ]]
	then
		echo -n "," >&2
	fi

	local sed=('sed')
	if [[ $smart_sed = 1 ]]
	then
		# -b is useful on windows, harmless on linux, but OS/X sed does not like it.
		sed=("${sed[@]}" '-b')
		if [[ $inplaceflag = 1 ]]
		then
			sed=("${sed[@]}" '-i')
		fi
	else
		if [[ $inplaceflag = 1 ]]
		then
			# OS/X sed needs separate arguments
			sed=("${sed[@]}" '-i' '')
		fi
	fi

	"${sed[@]}" -f <(
		IFS=$'\n'
		for patternwithline in $(grep --text "$filename" "$findresult"|cut -d ':' -f 2,3)
		do
			line=${patternwithline/:*/}
			pattern=${patternwithline/*:/}
			grep --text -e "$pattern" $rulesmatched|sed "s/^/$line/"
		done
	) "$filename"
}
export -f loop_main_replace

function loop_decorated_mode {
	local findresult=$1
	local rulesmatched=$2
	local filename=$3
	# Passed, but not used at the moment:
	#local inplace=$4

	verbose "actual file: $filename"
	local workfile=$filename.$$
	verbose "temp file: $workfile"
	if [[ -f $workfile ]]
	then
		warning "Temp file ($workfile) for ($filename) already exists. Skipping it."
		return 1
	fi
	cp -a "$filename" "$workfile"
	loop_main_replace "$findresult" "$rulesmatched" "$filename" 0 >"$workfile"
	diff=$(diff -u "$filename" "$workfile")
	if [[ $? = 0 ]]
	then
		verbose "nothing changed"
		rm "$workfile"
	else
		verbose "misspellings are fixed!"
		if [[ $opt_show_diff = 1 ]]
		then
			echo "$diff"
		fi
		if [[ $opt_real_run = 1 ]]
		then
			if [[ $opt_backup = 1 ]]
			then
				mv -n "$filename" "$workfile.BAK"
			fi
			mv "$workfile" "$filename"
		else
			rm "$workfile"
		fi
	fi
}
export -f loop_decorated_mode

if [[ "$FUNCTIONS_ONLY_FOR_COVERAGE" = "1" ]]
then
    return
fi

init_variables

parse_basic_options "$@"
retval=$?

if [[ $retval == 0 ]]
then
	process_parameter_rules
	retval=$?
fi

if [[ $retval == 0 ]]
then
	if [[ $opt_debug = 1 ]]
	then
		set -x
	fi
	check_grep_version
	check_sed_arguments
	preprocess_rules
	main_work prepare_prefilter_input_from_find 0 '' /dev/null
	retval=$?
	rm $tmpfile.prep.grep.rules $tmpfile.prep.grep.rules.w $tmpfile.prep.allsedrules
	if [[ $opt_debug = 1 ]]
	then
		set +x
	fi
fi

if [[ "$SHUNIT_VERSION" = "" ]]
then
	exit $retval
else
	return $retval
fi