Permalink
Browse files

Added the tmtest -f command-line option

Also fixed up some old tests that were conflicting with -e.
  • Loading branch information...
Scott Bronson Scott Bronson
Scott Bronson authored and Scott Bronson committed Jan 24, 2007
1 parent 48a67f5 commit 598793b9a0a5c8b36daa46e7caa64f1fa2d4db6f
Showing with 217 additions and 32 deletions.
  1. +7 −2 BUGS
  2. +1 −0 CHANGES
  3. +1 −2 TODO
  4. +36 −1 main.c
  5. +56 −13 test.c
  6. +1 −0 test.h
  7. +6 −0 test/00-cmdline/10-CfgDir.test
  8. +7 −0 test/00-cmdline/12-CfgEmpty.test
  9. +52 −0 test/03-results/40-TestFailures.test
  10. +50 −14 tmtest.pod
View
9 BUGS
@@ -13,8 +13,8 @@ These are bugs that tmtest inherits from Bash:
Bash3 has removed the ability to set LINENO, so you may see the
wrong line number be printed if there's an error. If you want to
see the correct line numbers, use Bash2. It's an unfortunate
-regression but since it only affects tests scripts being piped
-on stdin it won't affect many people.
+regression. I've worked around it so that it only affects tests
+piped in on stdin.
The <<-EOL operator strips ALL tabs from the front of each line.
@@ -53,3 +53,8 @@ redirect both stdout and stderr to a pipe?
Why is the unset command totally unrelated to the set command?
Why can I "exec > file" but I can't "exec | prog"?
+ I have to exec > >(prog). Seems asymmetrical.
+
+Why oh why can't I get the pid of the most recent process substitution?
+ (exec > >(tr a-z A-Z); pid=$!; echo howdy; wait $pid)
+ Except that $! is not set for process substitutions.
View
@@ -1,3 +1,4 @@
+- Added -f command-line option to print only the failed tests.
- Made REPLACE handle multiple text substitutions; could only handle 1 before.
- Got rid of RESULT and all its code. echo $? from within your test instead.
- Got rid of Cutest since it was very non-cute. Wrote zutest instead.
View
3 TODO
@@ -1,6 +1,5 @@
0.96:
-- Add a cmdline option that prints only tests that fail, and prints their
- full path.
+- Change exit value.
- Is it possible to separate STDOUT and STDERR? Maybe stderr comes first
in the testfile with each line prefixed by :. Then STDOUT. No need
for this delimiter craziness.
View
37 main.c
@@ -42,7 +42,8 @@
enum {
outmode_test,
outmode_dump,
- outmode_diff
+ outmode_diff,
+ outmode_failures_only
};
int outmode = outmode_test;
@@ -395,6 +396,29 @@ static void finish_diff(struct test *test, int diffpid)
}
+/* Prints the relative path from the original cwd to the current testfile */
+
+static void print_test_path(struct test *test)
+{
+ char result[PATH_MAX];
+
+ int keep = curpush(test->testfilename);
+ if(keep <= 0) {
+ printf("print_test_path: path is too long!\n");
+ return;
+ }
+
+ if(abs2rel(curabsolute(), orig_cwd, result, sizeof(result))) {
+ printf("%s\n", result);
+ } else {
+ printf("print_test_path: abs2rel error: %s relto %s\n",
+ curabsolute(), orig_cwd);
+ }
+
+ curpop(keep);
+}
+
+
/** Runs the named testfile.
*
* If warn_suffix is true and the ffilename doesn't end in ".test"
@@ -451,6 +475,7 @@ static int run_test(const char *name, const char *dispname, int warn_suffix)
// initialize the test mode
switch(outmode) {
case outmode_test:
+ case outmode_failures_only:
// nothing to do
break;
case outmode_dump:
@@ -561,6 +586,11 @@ static int run_test(const char *name, const char *dispname, int warn_suffix)
case outmode_test:
test_results(&test, dispname);
break;
+ case outmode_failures_only:
+ if(check_for_failure(&test, dispname)) {
+ print_test_path(&test);
+ }
+ break;
case outmode_dump:
dump_results(&test);
break;
@@ -1017,6 +1047,7 @@ static void process_args(int argc, char **argv)
{"config", 1, 0, 'c'},
{"diff", 0, 0, 'd'},
{"dump-script", 0, &dumpscript, 1},
+ {"failures-only", 0, 0, 'f'},
{"help", 0, 0, 'h'},
{"output", 0, 0, 'o'},
{"quiet", 0, 0, 'q'},
@@ -1050,6 +1081,10 @@ static void process_args(int argc, char **argv)
outmode = outmode_diff;
break;
+ case 'f':
+ outmode = outmode_failures_only;
+ break;
+
case 'h':
usage();
exit(0);
View
69 test.c
@@ -705,30 +705,24 @@ static void print_reason(struct test *test, const char *name, const char *prep)
}
-/** Checks the actual results against the expected results.
- * dispname is the name we should display for the test.
- */
-
-void test_results(struct test *test, const char *dispname)
+static void test_analyze_results(struct test *test, int *stdo, int *stde)
{
scanstate scanner;
char scanbuf[BUFSIZ];
- int stdo, stde; // true if there are differences.
+ *stdo = *stde = -1;
+
if(was_aborted(test->status)) {
- print_reason(test, "ABRT", "by");
test_failures++;
test->aborted = 1;
return;
}
if(was_disabled(test->status)) {
- print_reason(test, "dis ", "by");
return;
}
if(!was_started(test->status)) {
- print_reason(test, "ERR ", "error in");
test_failures++;
return;
}
@@ -749,14 +743,45 @@ void test_results(struct test *test, const char *dispname)
test->stderr_match = (fd_has_data(test->errfd) ? match_no : match_yes);
}
- stdo = (test->stdout_match != match_yes);
- stde = (test->stderr_match != match_yes);
+ *stdo = (test->stdout_match != match_yes);
+ *stde = (test->stderr_match != match_yes);
- if(!stdo && !stde && !test->exitsignal) {
+ if(!*stdo && !*stde && !test->exitsignal) {
test_successes++;
- printf("ok %s \n", convert_testfile_name(dispname));
} else {
test_failures++;
+ }
+}
+
+
+/** Checks the actual results against the expected results.
+ * dispname is the name we should display for the test.
+ */
+
+void test_results(struct test *test, const char *dispname)
+{
+ int stdo, stde; // true if there are differences.
+
+ test_analyze_results(test, &stdo, &stde);
+
+ if(test->aborted) {
+ print_reason(test, "ABRT", "by");
+ return;
+ }
+
+ if(was_disabled(test->status)) {
+ print_reason(test, "dis ", "by");
+ return;
+ }
+
+ if(!was_started(test->status)) {
+ print_reason(test, "ERR ", "error in");
+ return;
+ }
+
+ if(!stdo && !stde && !test->exitsignal) {
+ printf("ok %s \n", convert_testfile_name(dispname));
+ } else {
printf("FAIL %-25s ", convert_testfile_name(dispname));
if(test->exitsignal) {
printf("terminated by signal %d%s", test->exitsignal,
@@ -779,6 +804,24 @@ void test_results(struct test *test, const char *dispname)
}
+/** Like test_results() except that it returns 1 if the test failed
+ * and 0 if it was disabled or succeeded.
+ */
+
+int check_for_failure(struct test *test, const char *testpath)
+{
+ int stdo, stde; // true if there are differences.
+ int oldfailures = test_failures;
+
+ test_analyze_results(test, &stdo, &stde);
+ if(oldfailures+1 == test_failures) {
+ return 1;
+ }
+
+ return 0;
+}
+
+
/**
* Reads all the data from infd and writes it onto outfd.
*
View
1 test.h
@@ -85,6 +85,7 @@ void test_command_copy(struct test *test, FILE *fp);
void test_results(struct test *test, const char *dispname);
void dump_results(struct test *test);
void print_test_summary(struct timeval *start, struct timeval *stop);
+int check_for_failure(struct test *test, const char *testpath);
void test_init(struct test *test);
void test_free(struct test *test);
@@ -1,3 +1,9 @@
+# Test opening a directory as if it were a config file.
+
+# Our tests run with -e by default. Therefore, if we want to see
+# the test result, if it's nonzero, we need to turn that off.
+set +e
+
$tmtest --config=/tmp - <<-EOL
echo Test!
EOL
@@ -1,3 +1,10 @@
+# Test specifying nothing for config
+
+# tmtest runs -e by default, so we need to turn that off if we
+# want to be able to read the result of the command (else the
+# script just exits immediately and the echo never fires).
+set +e
+
$tmtest --config= - <<-EOL
echo Howdy
STDOUT:
@@ -0,0 +1,52 @@
+# ensures that tmtest -f works.
+#
+# We verify that it runs all tests and only prints the ones that fail.
+# We hit it with a disabled test, an aborted test, and
+
+MKDIR dir
+
+MKFILE fsuccess "$dir/success.test" <<-EOL
+ # ensure this test isn't printed
+ echo HI
+ STDOUT:
+ HI
+EOL
+
+MKFILE ffail "$dir/fail.test" <<-EOL
+ # ensure this test is printed
+ echo HI
+ STDOUT:
+ HO
+EOL
+
+MKFILE fdisabled "$dir/disabled.test" <<-EOL
+ # this test would succeed if it weren't disabled
+ DISABLED
+ echo HI
+ STDOUT:
+ HI
+EOL
+
+# We can't do this because it stops the entire test run.
+#MKFILE fabort "$dir/aborted.test" <<-EOL
+# # this test would succeed if it didn't abort
+# ABORT
+# echo HI
+# STDOUT:
+# HI
+#EOL
+
+MKDIR subdir "$dir/subdir"
+
+MKFILE subfail "$subdir/fail.test" <<-EOL
+ # make sure we show the proper path in subdirectories
+ echo HI
+ STDOUT:
+ HO
+EOL
+
+$tmtest -f $dir | REPLACE `basename "$dir"` DIR
+
+STDOUT:
+DIR/fail.test
+DIR/subdir/fail.test
Oops, something went wrong.

0 comments on commit 598793b

Please sign in to comment.