Add syntax highlighting tests

README.markdown

@@ -89,6 +89,33 @@ different PATH to run a different `vim`.
 
 See [gopher.vim for an example Travis integration](https://github.com/Carpetsmoker/gopher.vim/blob/master/.travis.yml).
 
+Syntax highlighting tests
+-------------------------
+
+testing.vim includes support for testing syntax highlighting with the
+`test-syntax` script.
+
+How it works:
+
+- Write a test file for the syntax to test; for example `basic.go` with the
+  contents:
+
+      package main
+
+      import "fmt"
+
+      func main() {
+      	var msg = "Let's go!"
+      	fmt.Println(msg)
+      }
+
+- Verify that the highlighting is correct in this file.
+
+- Run `./test-syntax path/to/basic.go`; this will generate a test script
+  detailing the current syntax highlighting.
+
+- Test the file as any other `_test.vim` script.
+
 Benchmarks
 ----------
 

shared

@@ -0,0 +1,72 @@
+# vim: ft=sh
+
+# Set up tmpdir
+tmpdir=$(mktemp -d -p "${TMPDIR:-/tmp}" testing.vim.XXXXXXX)
+mkdir -p "$tmpdir"
+trap 'rm -r "$tmpdir"' EXIT
+
+# Find files.
+find_files() {
+	local target=$1
+	local pattern=$2
+
+	# Find files to test.
+	if echo "$target" | grep -q '/\.\.\.$'; then
+		find_path="${target%...}"
+		find_args="-name $pattern"
+	elif [ -e "$target" ]; then
+		find_path="${target}"
+		find_args="-maxdepth 1 -name $pattern"
+	else
+		>&2 echo "no such file or directory: $target"
+		exit 1
+	fi
+
+	set -f
+	# shellcheck disable=SC2086
+	files=$(find "$find_path" $find_args)
+	set +f
+
+	if [ "$files" = "" ]; then
+		>&2 echo "no $pattern files in $target"
+		exit 3
+	fi
+
+	if [ $(link_pack $files) -gt 0 ]; then
+		>&2 echo "could not locate the plugin directory for $files"
+		exit 2
+	fi
+
+	echo $files
+}
+
+# Set up package; find the plugin's root directory and make sure that the plugin
+# we're testing is available as a package in $tmpdir.
+link_pack() {
+	mkdir -p "$tmpdir/pack/t/start"
+
+	IFS="
+"
+	for f in "$@"; do
+		local f="${f#./}"
+		local dir=$(dirname "$f")
+
+		local pack="$(cd "$dir" && pwd)"
+		while :; do
+			if [ "$pack" = "/" ]; then
+				printf 1
+				return
+			fi
+
+			if [ -d "$pack/autoload" ] || [ -d "$pack/plugin" ] || [ -d "$pack/ftplugin" ]; then
+				ln -s "$pack" "$tmpdir/pack/t/start"
+				printf 0
+				return
+			fi
+
+			pack="$(dirname "$pack")"
+		done
+	done
+
+	printf 2
+}

syntax.vim

@@ -0,0 +1,44 @@
+fun! ListAllSyntax()
+	let l:out = []
+
+	for l:line in range(1, line('$'))
+		let l:outline = []
+		let l:prevgroup = ''
+		let l:linelen = len(getline(l:line))
+		for l:col in range(1, l:linelen)
+			let l:group = synIDattr(synID(l:line, l:col, 1), 'name')
+
+			if l:prevgroup is# '' && l:group isnot# ''
+				call add(l:outline, [l:group, l:col, -1])
+			elseif l:group isnot# l:prevgroup || (l:col is l:linelen && l:prevgroup isnot# '')
+				let l:outline[len(l:outline) - 1][2] = l:col
+			endif
+
+			let l:prevgroup = l:group
+		endfor
+
+		call add(l:out, l:outline)
+	endfor
+
+	return l:out
+endfun
+
+fun! TestSyntax(file, want) abort
+    syntax on
+    exe 'e ' . fnameescape(a:file)
+
+    let l:want = a:want
+    let l:out = ListAllSyntax()
+
+    if len(l:out) != len(l:want)
+      call Errorf("out has different line length (%d, want %d)\nout: %s",
+            \ len(l:out), len(l:want), l:out)
+      return
+    endif
+
+    for l:i in range(0, len(l:out) - 1)
+        if l:out[l:i] != l:want[l:i]
+            call Errorf("line %d wrong\nwant: %s\nout:  %s", l:i, l:want[l:i], l:out[l:i])
+        endif
+    endfor
+endfun

syntaxtest.vim

@@ -0,0 +1,26 @@
+" vint: -ProhibitSetNoCompatible
+set nocompatible nomore shellslash encoding=utf-8 shortmess+=WIF
+lang mess C
+
+filetype plugin indent on
+syntax on
+
+let s:path = fnamemodify(bufname(''), ':.')
+let s:fname = fnamemodify(bufname(''), ':t:r')
+let s:lines = ListAllSyntax()
+
+silent! exe printf('split %s/syntax.tmp', fnameescape(g:test_tmpdir))
+call append('$', [
+			\ '',
+			\ printf('fun! Test_%s() abort', s:fname), 
+			\ printf('    call TestSyntax(g:test_packdir . "/%s",', s:path),
+			\ '        \ [',
+	\ ])
+
+for s:line in s:lines
+	call append('$', printf('        \ %s,', s:line))
+endfor
+call append('$', ['    \ ])', 'endfun'])
+
+silent! write
+qa!

test

@@ -3,9 +3,7 @@
 set -euC
 testingdir=$(cd -P "$(dirname "$0")" >/dev/null && pwd)
 
-tmpdir=$(mktemp -d -p "${TMPDIR:-/tmp}" testing.vim.XXXXXXX)
-mkdir -p "$tmpdir"
-trap 'rm -r "$tmpdir"' EXIT
+. "$testingdir/shared"
 
 _usage() {
 	echo "Usage: ${0##*/} [-hvVc] [-r pattern] [-b path] [-p path] target"
@@ -93,72 +91,32 @@ testtarget="$1"
 [ -f "$tmpdir/cov-profile.txt" ] && rm "$tmpdir/cov-profile.txt"
 [ -f "$tmpdir/cov-report.txt" ]  && rm "$tmpdir/cov-report.txt"
 
-# Find files to test.
-if echo "$testtarget" | grep -q '/\.\.\.$'; then
-	find_path="${testtarget%...}"
-	find_args="-name *_test.vim"
-elif [ -e "$testtarget" ]; then
-	find_path="${testtarget}"
-	find_args="-maxdepth 1 -name *_test.vim"
-else
-	echo "no such file or directory: $testtarget"
-	exit 1
-fi
-
 vim_cmd=vim
 
 # Generate code coverage report.
 if [ "$coverage" -eq 1 ]; then
 	vim_cmd="covimerage -q run --data-file $tmpdir/cov-profile.data --report-file $tmpdir/cov-profile.txt --append $vim_cmd"
 fi
 
-# Set up package.
-mkdir -p "$tmpdir/pack/t/start"
-
-did_link=0
-set -f
-# shellcheck disable=SC2086
-files=$(find "$find_path" $find_args)
-set +f
-IFS="
-"
-
 # Run the actual tests.
-for test_file in $files; do
+test_files=$(find_files "$testtarget" '*_test.vim')
+packdir="$tmpdir/pack/t/start/$(ls "$tmpdir/pack/t/start")"
+for test_file in $test_files; do
 	IFS="$(printf ' \n')"
 
 	test_file="${test_file#./}"
 	test_dir=$(dirname "$test_file")
 
-	# Find the plugin's root directory and make sure that the plugin we're
-	# testing is available as a package.
-	if [ "$did_link" -eq 0 ]; then
-		pack="$(cd "$test_dir" && pwd)"
-		while :; do
-			if [ "$pack" = "/" ]; then
-				echo "could not locate the plugin directory for $test_file"
-				exit 2
-			fi
-
-			if [ -d "$pack/autoload" ] || [ -d "$pack/plugin" ] || [ -d "$pack/ftplugin" ]; then
-				ln -s "$pack" "$tmpdir/pack/t/start"
-				did_link=1
-				break
-			fi
-
-			pack="$(dirname "$pack")"
-		done
-	fi
-
 	$vim_cmd --noplugin -u NONE -N $vim_verbose -e \
 		+"set shm+=WAFI rtp^=$testingdir packpath=$tmpdir" \
 		+'filetype plugin indent on' \
 		+'packloadall!' \
 		+"silent e $test_file" \
 		+"let g:test_verbose=$verbose" \
-		+"let g:test_tmpdir='$tmpdir'" \
+		+"let g:test_tmpdir='$tmpdir' | let g:test_packdir='$packdir'" \
 		+"let g:test_run='$run'" \
 		+"let g:test_bench='$bench'" \
+		-S "$testingdir/syntax.vim" \
 		-S "$testingdir/runtest.vim" </dev/null || (
 		# If Vim exits with non-0 it's almost certainly a bug in the test runner;
 		# should never happen in normal usage.
@@ -172,8 +130,8 @@ for test_file in $files; do
 		#
 		# So try grepping that, maybe?
 		echo  'TEST RUNNER FAILURE; error in runtest.vim'
-		[  -f "$tmpdir/test.tmp" ] && cat "$tmpdir/test.tmp"
-		exit  5
+		[ -f "$tmpdir/test.tmp" ] && cat "$tmpdir/test.tmp"
+		exit 5
 	)
 
 	# Append logs
@@ -187,12 +145,6 @@ for test_file in $files; do
 	rm "$tmpdir/test.tmp"
 done
 
-# Loop never ran.
-if [ "$did_link" -eq 0 ]; then
-	echo "no *_test.vim files in $testtarget"
-	exit 3
-fi
-
 if [ -n "$profile" ]; then
 	covimerage xml --omit '*_test.vim' --data-file "$tmpdir/cov-profile.data"
 	mv coverage.xml "$profile"

test-syntax

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -euC
+testingdir=$(cd -P "$(dirname "$0")" >/dev/null && pwd)
+
+. "$testingdir/shared"
+
+if [ -z "${1:-}" ]; then
+	>&2 echo "Must give at least one test file"
+	exit 1
+fi
+
+if [ $(link_pack "$@") -gt 0 ]; then
+	>&2 echo "could not locate the plugin directory for $files"
+	exit 2
+fi
+
+for f in "$@"; do
+	vim --noplugin -u NONE -N -e \
+		+"set shm+=WAFI rtp^=$testingdir packpath=$tmpdir" \
+		+'filetype plugin indent on' \
+		+'packloadall!' \
+		+"silent e $f" \
+		+"let g:test_tmpdir='$tmpdir'" \
+		-S "$testingdir/syntax.vim" \
+		-S "$testingdir/syntaxtest.vim" >/dev/null </dev/null
+done
+
+printf '" This file is automatically generated by test-syntax from testing.vim'
+
+cat "$tmpdir/syntax.tmp"