Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
184 lines (163 sloc) 6.83 KB
#! /bin/csh -f
#
# Part of automation of the testing process for the Tokenizer/De-Tokenizer
# Automate Comparison of results of Auto-Execution of the various test-cases
# Updated Fri, 16 Jun 2006 at 12:12 PDT by David L. Paktor
# To Do:
# Conditional comparison of .DeTok files:
# Detect difference in .fc file, but do not display
# Display differences in .Log file
# If difference detected in .fc file, display mssg
# Otherwise, detect difference in .DeTok file and display mssg
# If either diff detected in .fc or in .DeTok file, show .DeTok diffs
# Display differences in .RomHdr
#
# Compare the source (.fth) files (for regression testing...)
# Usage:
# Run this from the parent-directory of the various Test categories.
# This script uses the same TestArgs files that were used by
# (and described in) the AutoExec script.
#
# This script needs a shell environment variable called CygTestLogs
# whose value is the path to the parent-directory that contains
# the results of Auto-Execution of the various test-cases under
# Cygwin, which is the "Gold Standard" to which the other
# platforms' results will be compared.
#
# This script accepts an optional environment variable called LogFilesDiff
# which may be set to change the number of lines by which two .Log
# files are expected to differ (due to different compilation-signatures)
# If this is not supplied in the environment, a default will be provided.
#
# This script compares only one platform's results at a time;
# the parent-directory that contains the Auto-Execution results
# for the platform to be compared should be the current directory.
#
# This script displays both operational errors and detected discrepancies
# in Standard Output and also collects the same in a log file whose
# name is AutoComp.Log.<time-and-date-stamp>
#
# A secondary output file, called AutoComp.TKdiffs.<time-and-date-stamp>
# is produced; it can be "source"d to cause a sequential tkdiff of
# the detected discrepancies
# Set the number of lines by which two .Log files are expected to differ.
# (this is due to different compilation-signature lines)
set LogFilesDiffLines = 6
# Over-ride with optional shell env't variable called LogFilesDiff
if ( $?LogFilesDiff ) then
set notallnumbers = `echo $LogFilesDiff | tr '0-9' ' '`
if ( "$notallnumbers" == "" ) then
set LogFilesDiffLines = $LogFilesDiff
endif
endif
# Preliminary error checking:
if ( ! $?CygTestLogs ) then
echo 'Please define a shell environment variable called CygTestLogs'
echo ' whose value is the path to the Cygwin test-results directory.'
exit 1
endif
set TArgFiles = `find . -name TestArgs -exec expr {} : '\./\(.*\)' \;`
if ( $#TArgFiles == 0 ) then
echo 'No TestArgs files found in subdirectories.'
echo 'Starting in wrong directory?'
exit 2
endif
set TArgReslts = `find $CygTestLogs -name TestArgs -exec expr {} : $CygTestLogs/'\(.*\)' \;`
if ( $#TArgReslts != $#TArgFiles ) then
echo 'Number of TestArgs files in CygTestLogs ('$CygTestLogs') directory'
echo ' does not match those found under current directory.'
echo 'Please correct discrepancy and try again.'
exit 4
endif
if ( "$TArgReslts" != "$TArgFiles" ) then
echo 'List of TestArgs files in CygTestLogs ('$CygTestLogs') directory'
echo ' does not match those found under current directory.'
echo 'Please correct discrepancy and try again.'
exit 8
endif
# Let's save mismatch identifiers:
set datemark = `date '+%y%m%d.%H%M%S'`
set ErrResltFil = AutoComp.Log.$datemark
set ScriptResltFil = AutoComp.TKdiffs.$datemark
# Let's show a progress-downcount
# Find the count of actual tests; leave the max number on display:
set cnt = `egrep '^[^#].*' $TArgFiles | wc -l`
echo ''
echo $cnt
foreach TArgFil ( $TArgFiles )
set dir = $TArgFil:h
set lindxmax = `cat $TArgFil | wc -l`
set lindx = 0
while ( $lindx < $lindxmax )
@ lindx++
set targline = `sed -n ${lindx}p $TArgFil`
if ( $#targline == 0 ) continue
if ( "$targline[1]" == "#" ) continue
set ttarg = $dir/$targline[1]
# One more error-check:
if ( ! -f ${ttarg}.fth ) then
echo File Not found: ${ttarg}.fth
echo Please update ${TArgFil} file, line $lindx
continue
endif
set label = `echo $targline | awk -F , '{print $2}'`
if ( "$label" != "" ) set label = .$label
# Now begins the real fun...
# Show a running down-count
echo -n X${cnt}' ' | tr X \\015
@ cnt--
# Check the extensions that are text-type files
foreach txext ( Log RomHdr fl fl.missing )
# Is the file in both?
if ( -f $CygTestLogs/${ttarg}${label}.${txext} && ! -f ${ttarg}${label}.${txext} ) then
echo Missing ${ttarg}${label}.${txext} | tee -a $ErrResltFil
else
if ( -f ${ttarg}${label}.${txext} ) then
set lim = 0
set fxext = `echo $txext | tr '.' '_'`
if ( `eval echo '$?'${fxext}FilesDiffLines` ) then
set lim = `eval echo '$'${fxext}FilesDiffLines`
endif
set logdiff = `diff {,$CygTestLogs/}${ttarg}${label}.${txext} | wc -l`
if ( $logdiff > $lim ) then
echo ${ttarg}${label}.${txext} files differ. | tee -a $ErrResltFil
diff {$CygTestLogs/,}${ttarg}${label}.${txext} | tee -a $ErrResltFil
echo 'tkdiff {$CygTestLogs/,}'${ttarg}${label}.${txext} >> $ScriptResltFil
endif
endif
endif
end
# Now, check the FCode file.
# If there's an FCode file in the reference directory ($CygTestLogs)
# there should be one in the directory under test.
if ( -f $CygTestLogs/${ttarg}${label}.fc && ! -f ${ttarg}${label}.fc ) then
echo Missing ${ttarg}${label}.fc | tee -a $ErrResltFil
else
# Is there an FCode file in the directory under test?
unset ShowDetokDiffs
if ( -f ${ttarg}${label}.fc ) then
set bindiff = `cmp {,$CygTestLogs/}${ttarg}${label}.fc`
if ( "$bindiff" != "" ) then
echo "$bindiff" | tee -a $ErrResltFil
# If the binaries are different, show the difference in the DeTok form
# If the DeTok form is missing, it's an even stranger error!
if ( ! -f $CygTestLogs/${ttarg}${label}.DeTok || ! -f ${ttarg}${label}.DeTok ) then
echo Missing DeTok file for ${ttarg}${label} | tee -a $ErrResltFil
else
set ShowDetokDiffs
endif
else
# Even if the binaries are not different,
# check for changes in in the DeTok form
set detokdiff = `diff {,$CygTestLogs/}${ttarg}${label}.DeTok | wc -l`
if ( $detokdiff > 0 ) set ShowDetokDiffs
endif
endif
if ( $?ShowDetokDiffs ) then
echo ${ttarg}${label}.DeTok files differ. | tee -a $ErrResltFil
diff {$CygTestLogs/,}${ttarg}${label}.DeTok | tee -a $ErrResltFil
echo 'tkdiff {$CygTestLogs/,}'${ttarg}${label}.DeTok >> $ScriptResltFil
endif
endif
end
end