White listing (#30)

README.md

@@ -65,6 +65,30 @@ File filtering options:
 * `-b` Process binary, generated files. (do not ignore `*.gif`, `*.jpg`, `*.jpeg`, `*.png`, `*.zip`, `*.gz`, `*.bz2`, `*.xz`, `*.rar`, `*.po`, `*.pdf`, `*.woff`, `yarn.lock`, `package-lock.json`, `composer.lock`, `*.mo`)
 * `-m` Disable file size checks. Default is to ignore files > 1MB. (usually csv, compressed JS, ..)
 
+White listing files/entries
+
+Misspell-fixer automatically ignores the issues matching to the patterns listed in `.misspell-fixer.ignore`.
+The format of this file follows the prefiltering's temporary result format:
+`^filename:line number:matched word`
+
+* `-W` can be used to append the found issues instead of fixing them based on the other settings.
+
+The ignore file is interpreted as a `grep` exclusion list.
+It is applied after the prefiltering step as a set of grep expression.
+So it is possible to exclude any prefixes or more specifically whole files with keeping only their file names:
+
+`^filename`
+
+Or a directory:
+
+`^directory`
+
+The entries are listed/matched with the paths based on the current invocation. Reaching the same target with different path from
+the same working directory will will not apply the white listed entries geneated from the other invocation.
+In directory `x` the white list entries generated with target `.` will not be applied for target `../x`, although they are the same.
+This can be worked around with manually editing the white list to your needs.
+(Patches are welcome for a more general solution...)
+
 ### Sample usage
 
 By default nothing important will happen

doc/misspell-fixer.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "misspell-fixer" "" "September 2018" "" "misspell-fixer"
+.TH "misspell-fixer" "" "October 2018" "" "misspell-fixer"
 .SH NAME
 misspell-fixer - misspell-fixer
 .
@@ -110,6 +110,32 @@ File filtering options:
 .
 .IP "" 0
 .
+.P
+White listing files/entries
+.
+.P
+Misspell\-fixer automatically ignores the issues matching to the patterns listed in \fB\.misspell\-fixer\.ignore\fR\. The format of this file follows the prefiltering\'s temporary result format: \fB^filename:line number:matched word\fR
+.
+.IP "\(bu" 4
+\fB\-W\fR can be used to append the found issues instead of fixing them based on the other settings\.
+.
+.IP "" 0
+.
+.P
+The ignore file is interpreted as a \fBgrep\fR exclusion list\. It is applied after the prefiltering step as a set of grep expression\. So it is possible to exclude any prefixes or more specifically whole files with keeping only their file names:
+.
+.P
+\fB^filename\fR
+.
+.P
+Or a directory:
+.
+.P
+\fB^directory\fR
+.
+.P
+The entries are listed/matched with the paths based on the current invocation\. Reaching the same target with different path from the same working directory will will not apply the white listed entries geneated from the other invocation\. In directory \fBx\fR the white list entries generated with target \fB\.\fR will not be applied for target \fB\.\./x\fR, although they are the same\. This can be worked around with manually editing the white list to your needs\. (Patches are welcome for a more general solution\.\.\.)
+.
 .SS "Sample usage"
 By default nothing important will happen
 .

misspell-fixer

@@ -26,7 +26,10 @@ function init_variables {
 	export opt_backup=1
 	export opt_dots=0
 	export bash_arg
-
+
+	export opt_whitelist_save=0
+	export opt_whitelist_filename=".misspell-fixer.ignore"
+
 	export rule_file_base="${BASH_SOURCE[0]}"
 	if [ ! -f "${rule_file_base}-safe.0.sed" ]
 	then
@@ -60,7 +63,7 @@ function init_variables {
 
 function parse_basic_options {
 	local OPTIND
-	while getopts ":dvorfsibnRVDmughN:P:" opt; do
+	while getopts ":dvorfsibnRVDmughwWN:P:" opt; do
 		case $opt in
 			d)
 				warning "-d Enable debug mode."
@@ -146,6 +149,10 @@ function parse_basic_options {
 				fi
 				return 1
 			;;
+			W)
+				warning "-W Save found misspelled file entries into $opt_whitelist_filename instead of fixing them."
+				opt_whitelist_save=1
+			;;
 			\?)
 				warning "Invalid option: -$OPTARG"
 				return 1
@@ -170,14 +177,20 @@ function parse_basic_options {
 	fi
 
 	directories=( "$@" )
-	cmd_part_ignore="( -iname $tmpfile* -o -iname *.BAK $cmd_part_ignore_scm $cmd_part_ignore_bin ) -prune -o "
+	cmd_part_ignore="( -iname $tmpfile* -o -iname $opt_whitelist_filename -o -iname *.BAK $cmd_part_ignore_scm $cmd_part_ignore_bin ) -prune -o "
 	warning "Target directories: ${directories[*]}"
 
 	return 0
 }
 
 function process_parameter_rules {
-	if [[ 	$opt_show_diff = 1 ||\
+	if [[ $opt_whitelist_save = 1 &&
+		$opt_real_run = 1 ]]
+	then
+		warning "Whitelist cannot be generated in real run mode. => Exiting."
+		return 1
+	fi
+	if [[ $opt_show_diff = 1 ||\
 		$opt_backup = 1 ||\
 		$opt_real_run = 0 ||\
 		$opt_verbose = 1 ]]
@@ -263,6 +276,13 @@ function main_work {
 
 	sort -u "$itertmpfile.combos" "$itertmpfile.combos.w" >"$itertmpfile.combos.all"
 
+	if [[ -s "$opt_whitelist_filename" ]]
+	then
+		warning "Skipping whitelisted entries based on $opt_whitelist_filename."
+		grep -vf $opt_whitelist_filename "$itertmpfile.combos.all" >"$itertmpfile.combos.all.onlynonwhitelisted"
+		mv "$itertmpfile.combos.all.onlynonwhitelisted" "$itertmpfile.combos.all"
+	fi
+
 	if [[ $opt_verbose = 1 ]]
 	then
 		verbose "Results of prefiltering: (filename:line:pattern)"
@@ -271,14 +291,24 @@ function main_work {
 
 	if [[ -s $itertmpfile.combos.all ]]
 	then
-		grep --text -f <(cut -d ':' -f 3 "$itertmpfile.combos.all") "$tmpfile.prep.allsedrules" >"$itertmpfile.rulesmatched"
-		warning "Iteration $iteration: replacing."
-		cut -d ':' -f 1 "$itertmpfile.combos.all" |sort -u >"$itertmpfile.matchedfiles"
-		xargs <"$itertmpfile.matchedfiles" $cmd_part_parallelism -n 1 -I '{}' $COVERAGE_WRAPPER bash -c$bash_arg "$loop_function $itertmpfile.combos.all $itertmpfile.rulesmatched '{}' 1"
-		rm "$itertmpfile.rulesmatched"
-		if [[ $opt_dots = 1 ]]
+		if [[ $opt_whitelist_save = 1 ]]
 		then
-			echo >&2
+			warning "Saving found misspells into $opt_whitelist_filename."
+			sed -e 's/^/^/' "$itertmpfile.combos.all" >> "$opt_whitelist_filename"
+		else
+			if [[ $opt_real_run = 0 ]]
+			then
+				warning "Real run (-r) has not been enabled. Files will not be changed. Use -r to override this."
+			fi
+			grep --text -f <(cut -d ':' -f 3 "$itertmpfile.combos.all") "$tmpfile.prep.allsedrules" >"$itertmpfile.rulesmatched"
+			warning "Iteration $iteration: replacing."
+			cut -d ':' -f 1 "$itertmpfile.combos.all" |sort -u >"$itertmpfile.matchedfiles"
+			xargs <"$itertmpfile.matchedfiles" $cmd_part_parallelism -n 1 -I '{}' $COVERAGE_WRAPPER bash -c$bash_arg "$loop_function $itertmpfile.combos.all $itertmpfile.rulesmatched '{}' 1"
+			rm "$itertmpfile.rulesmatched"
+			if [[ $opt_dots = 1 ]]
+			then
+				echo >&2
+			fi
 		fi
 	else
 		warning "Iteration $iteration: nothing to replace."

test/coverage_wrapper.sh

@@ -4,4 +4,4 @@ export FUNCTIONS_ONLY_FOR_COVERAGE=1
 
 source misspell-fixer
 
-eval $3 >>$TEST_OUTPUT 2>&1
+eval $3 >>"$TEST_OUTPUT" 2>&1

test/expected.debug.output

@@ -52,7 +52,8 @@
 ++++ echo 'misspell-fixer: Iteration X: done.'
 ++++ echo 'misspell-fixer: Iteration X: prefiltering.'
 ++++ echo 'misspell-fixer: Iteration X: replacing.'
-++++ find /tmp/misspell-fixer-test/X/work '(' -iname '.misspell-fixer.X*' -o -iname '*.BAK' -o -iname .git -o -iname .svn -o -iname .hg -o -iname CVS -o -iname '*.gif' -o -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.zip' -o -iname '*.gz' -o -iname '*.bzX' -o -iname '*.xz' -o -iname '*.rar' -o -iname '*.po' -o -iname '*.pdf' -o -iname '*.woff' -o -iname yarn.lock -o -iname package-lock.json -o -iname composer.lock -o -iname '*.mo' ')' -prune -o -type f -and '(' -true ')' -and '(' -size -Xk ')' -printX
+++++ echo 'misspell-fixer: Real run (-r) has not been enabled. Files will not be changed. Use -r to override this.'
+++++ find /tmp/misspell-fixer-test/X/work '(' -iname '.misspell-fixer.X*' -o -iname .misspell-fixer.ignore -o -iname '*.BAK' -o -iname .git -o -iname .svn -o -iname .hg -o -iname CVS -o -iname '*.gif' -o -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.png' -o -iname '*.zip' -o -iname '*.gz' -o -iname '*.bzX' -o -iname '*.xz' -o -iname '*.rar' -o -iname '*.po' -o -iname '*.pdf' -o -iname '*.woff' -o -iname yarn.lock -o -iname package-lock.json -o -iname composer.lock -o -iname '*.mo' ')' -prune -o -type f -and '(' -true ')' -and '(' -size -Xk ')' -printX
 ++++ grep '\\b' .misspell-fixer.X.prep.allsedrules
 ++++ grep --text -f /dev/fd/X .misspell-fixer.X.prep.allsedrules
 ++++ grep -q illegal
@@ -87,9 +88,11 @@
 ++++ warning 'Iteration X: done.'
 ++++ warning 'Iteration X: prefiltering.'
 ++++ warning 'Iteration X: replacing.'
+++++ warning 'Real run (-r) has not been enabled. Files will not be changed. Use -r to override this.'
 ++++ xargs -P X -n X -I '{}' bash -cx 'loop_decorated_mode .misspell-fixer.X.X.combos.all .misspell-fixer.X.X.rulesmatched '\''{}'\'' X'
 ++++ xargs -X -P X -n X grep --text -F -noH -w -f .misspell-fixer.X.prep.grep.rules.w
 ++++ [[ -f .misspell-fixer.X.X.matchedfiles ]]
+++++ [[ -s .misspell-fixer.ignore ]]
 ++++ [[ -s .misspell-fixer.X.X.combos.all ]]
 ++++ [[ -s .misspell-fixer.X.X.combos.all ]]
 ++++ [[ X -lt X ]]
@@ -98,6 +101,8 @@
 ++++ [[ X = X ]]
 ++++ [[ X = X ]]
 ++++ [[ X = X ]]
+++++ [[ X = X ]]
+++++ [[ X = X ]]
 +++++ cat
 +++++ cut -d : -f X .misspell-fixer.X.X.combos.all
 +++++ grep --version
@@ -116,6 +121,7 @@ misspell-fixer: -d Enable debug mode.
 misspell-fixer: Iteration X: done.
 misspell-fixer: Iteration X: prefiltering.
 misspell-fixer: Iteration X: replacing.
+misspell-fixer: Real run (-r) has not been enabled. Files will not be changed. Use -r to override this.
 misspell-fixer: Target directories: /tmp/misspell-fixer-test/X/work
 \ No newline at end of file
 \ No newline at end of file'

test/expected.diff.output

@@ -1,6 +1,7 @@
 misspell-fixer: -s Enable showing of diffs.
 misspell-fixer: Target directories: /tmp/misspell-fixer-test/X/work
 misspell-fixer: Iteration X: prefiltering.
+misspell-fixer: Real run (-r) has not been enabled. Files will not be changed. Use -r to override this.
 misspell-fixer: Iteration X: replacing.
 --- /tmp/misspell-fixer-test/X/work/X.txt	X-X-X X:X:X.X +X
 +++ /tmp/misspell-fixer-test/X/work/X.txt.X	X-X-X X:X:X.X +X

test/expected.dots.output

@@ -1,6 +1,7 @@
 misspell-fixer: -o Print dots for each file scanned, comma for each file fix iteration/file.
 misspell-fixer: Target directories: /tmp/misspell-fixer-test/X/work
 misspell-fixer: Iteration X: prefiltering.
-......misspell-fixer: Iteration X: replacing.
+......misspell-fixer: Real run (-r) has not been enabled. Files will not be changed. Use -r to override this.
+misspell-fixer: Iteration X: replacing.
 ,
 misspell-fixer: Iteration X: done.

test/expected.verbose.output

@@ -5,6 +5,7 @@ misspell-fixer: Results of prefiltering: (filename:line:pattern)
 /tmp/misspell-fixer-test/X/work/X.txt:X:succesful
 /tmp/misspell-fixer-test/X/work/X.txt:X:succesful
 /tmp/misspell-fixer-test/X/work/X.txt:X:lower then
+misspell-fixer: Real run (-r) has not been enabled. Files will not be changed. Use -r to override this.
 misspell-fixer: Iteration X: replacing.
 misspell-fixer: actual file: /tmp/misspell-fixer-test/X/work/X.txt
 misspell-fixer: temp file: /tmp/misspell-fixer-test/X/work/X.txt.X

test/expected.whitelist_vs_real.output

@@ -0,0 +1,4 @@
+misspell-fixer: -r Enable real run. Overwrite original files!
+misspell-fixer: -W Save found misspelled file entries into .misspell-fixer.ignore instead of fixing them.
+misspell-fixer: Target directories: /tmp/misspell-fixer-test/X/work
+misspell-fixer: Whitelist cannot be generated in real run mode. => Exiting.

test/tests.sh

@@ -226,7 +226,28 @@ testMNoChange(){
 	rm -rf $TEMP/work/nochange.txt $TEMP/expected/nochange.txt
 }
 
+testWhitelistConflictWithRealRun(){
+	runAndCompareOutput -rW whitelist_vs_real
+}
+
+testWhitelist(){
+	cp test/stubs/0.txt $TEMP/work/0.txt
+	cp test/stubs/0.txt $TEMP/expected/0.txt
+	local whitelist=".misspell-fixer.ignore"
+	assertFalse 'Whitelist should not exist' "[ -s $whitelist ]"
+	$RUN -W $TEMP/work/
+	assertTrue 'Whitelist should not exist' "[ -s $whitelist ]"
+	cp test/stubs/0.txt $TEMP/work/1.txt
+	cp test/expecteds/0.txt $TEMP/expected/1.txt
+	runAndCompare -rn
+	cp -a $TEMP/expected /tmp/xxx
+	rm $whitelist
+}
+
 suite(){
+	export TEST_OUTPUT=$TEMP/output.default
+	suite_addTest testWhitelist
+	suite_addTest testWhitelistConflictWithRealRun
 	suite_addTest testShowDiff
 	suite_addTest testErrors
 	suite_addTest testOnlyDir