Permalink
Browse files

Implement concurrent testsuite

Use python instead of bash because it has better
multiprocessing support.
  • Loading branch information...
iliastsi committed Oct 19, 2012
1 parent 25ef351 commit 5a14efec76b314f130f0001467acf6ba8f1e5003
Showing with 184 additions and 74 deletions.
  1. +1 −1 Makefile
  2. +1 −1 README.md
  3. +182 −0 testsuite/runtests.py
  4. +0 −72 testsuite/runtests.sh
View
@@ -160,7 +160,7 @@ utest: all
-s concuerror_util test -s init stop
test: all
- @(cd testsuite && ./runtests.sh)
+ @(cd testsuite && THREADS=$(THREADS) ./runtests.py)
concuerror:
printf "\
View
@@ -19,7 +19,7 @@ Howto
* Build Concuerror : `make`
* Run Concuerror : `concuerror --help`
* Run Concuerror GUI : `concuerror --gui`
-* Run testsuite : `make test`
+* Run testsuite : `make THREADS=4 test`
* Run unit tests : `make utest`
* Dialyze : `make dialyze`
* Cleanup : `make clean`
View
@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+
+import os
+import re
+import sys
+import glob
+import subprocess
+from ctypes import c_int
+from multiprocessing import Process, Lock, Value, BoundedSemaphore
+
+
+#---------------------------------------------------------------------
+# Extract scenarios from the specified test
+
+def runTest(test):
+ global dirname
+ global results
+ # test has the format of '.*/suites/<suite_name>/src/<test_name>(.erl)?'
+ # Split the test in suite and name components using pattern matching
+ t = test.split("/")
+ suite = t[-3]
+ name = os.path.splitext(t[-1])[0]
+ if os.path.isdir(test):
+ # Our test is a multi module directory
+ dirn = test # directory
+ modn = "test" # module name
+ files = glob.glob(dirn + "/*.erl")
+ else:
+ dirn = os.path.dirname(test)
+ modn = name
+ files = [test]
+ # Create a dir to save the results
+ try:
+ os.mkdir(results + "/" + suite)
+ except OSError:
+ pass
+ # Compile it
+ os.system("erlc -W0 -o %s %s/%s.erl" % (dirn, dirn, modn))
+ # And extract scenarios from it
+ pout = subprocess.Popen(
+ ["erl -noinput -pa %s -pa %s -s scenarios extract %s -s init stop"
+ % (dirname, dirn, modn)], stdout=subprocess.PIPE, shell=True)
+ procS = []
+ for scenario in pout.stdout:
+ # scenario has the format of {<mod_name>,<func_name>,<preb>}\n
+ scen = scenario.strip("{}\n").split(",")
+ # And run the test
+ p = Process(target=runScenario,
+ args=(suite, name, modn, scen[1], scen[2], files))
+ p.start()
+ procS.append(p)
+ pout.stdout.close()
+ # Wait
+ for p in procS:
+ p.join()
+
+#---------------------------------------------------------------------
+# Run the specified scenario and print the results
+
+def runScenario(suite, name, modn, funn, preb, files):
+ global concuerror
+ global results
+ global dirname
+ global sema
+ global lock
+ global total_tests
+ global total_failed
+ sema.acquire()
+ # Run concuerror
+ os.system("%s --target %s %s --files %s --output %s/%s/%s-%s-%s.txt --preb %s --quiet"
+ % (concuerror, modn, funn, ' '.join(files), results, suite, name,
+ funn, preb, preb))
+ # Compare the results
+ a = "%s/suites/%s/results/%s-%s-%s.txt" % (dirname, suite, name, funn, preb)
+ b = "%s/%s/%s-%s-%s.txt" % (results, suite, name, funn, preb)
+ equalRes = equalResults(a, b)
+ sema.release()
+ # Print the results
+ lock.acquire()
+ total_tests.value += 1
+ if equalRes:
+ print "%-10s %-20s %-40s \033[01;32mok\033[00m" % \
+ (suite, name, "("+funn+", "+preb+")")
+ else:
+ total_failed.value += 1
+ print "%-10s %-20s %-40s \033[01;31mfailed\033[00m" % \
+ (suite, name, "("+funn+", "+preb+")")
+ lock.release()
+
+def equalResults(f1, f2):
+ try:
+ fp1 = open(f1, 'r')
+ except IOError:
+ return False
+ try:
+ fp2 = open(f2, 'r')
+ except IOError:
+ fp1.close()
+ return False
+ while True:
+ l1 = fp1.readline()
+ l2 = fp2.readline()
+ if (l1 != l2) and (not ignoreLine(l1)):
+ fp1.close(); fp2.close()
+ return False
+ if not l1:
+ fp1.close(); fp2.close()
+ return True
+
+def ignoreLine(line):
+ global ignore_matches
+ for match in ignore_matches:
+ if re.search(match, line):
+ return True
+ return False
+
+#---------------------------------------------------------------------
+# Main program
+
+# Compile some regular expressions
+match_pids = re.compile("<\d+\.\d+\.\d+>")
+match_refs = re.compile("#Ref<[\d\.]+>")
+match_file = re.compile("suites/.+/src/.*\.erl")
+ignore_matches = [match_pids, match_refs, match_file]
+
+# Get the directory of Concuerror's testsuite
+dirname = os.path.normpath(os.path.dirname(sys.argv[0]))
+concuerror = dirname + "/../concuerror"
+results = dirname + "/results"
+
+# Cleanup temp files
+os.system("find %s -name '*.beam' -exec rm {} \;" % dirname)
+os.system("rm -rf %s/*" % results)
+
+# Compile scenarios.erl
+os.system("erlc %s/scenarios.erl" % dirname)
+
+# If we have arguments we should use them as tests,
+# otherwise check them all
+if len(sys.argv) > 1:
+ tests = sys.argv[1:]
+ tests = [item.rstrip('/') for item in tests]
+else:
+ tests = glob.glob(dirname + "/suites/*/src/*")
+
+# Print header
+print "Concuerror's Testsuite\n"
+print "%-10s %-20s %-40s %s" % \
+ ("Suite", "Test", "(Function, Preemption Bound)", "Result")
+print "---------------------------------------" + \
+ "------------------------------------------"
+
+# Create share integers to count tests and
+# a lock to protect printings
+lock = Lock()
+total_tests = Value(c_int, 0, lock=False)
+total_failed = Value(c_int, 0, lock=False)
+
+# How many threads we want (default 4)
+threads = os.getenv("THREADS", "")
+if threads == "":
+ threads = "4"
+sema = BoundedSemaphore(int(threads))
+
+# For every test do
+procT = []
+for test in tests:
+ p = Process(target=runTest, args=(test,))
+ p.start()
+ procT.append(p)
+# Wait
+for p in procT:
+ p.join()
+
+# Print overview
+print "\nOVERALL SUMMARY for test run"
+print " %d total tests, which gave rise to" % len(tests)
+print " %d test cases, of which" % total_tests.value
+print " %d caused unexpected failures" % total_failed.value
+
+# Cleanup temp files
+os.system("find %s -name '*.beam' -exec rm {} \;" % dirname)
View
@@ -1,72 +0,0 @@
-#!/bin/bash
-
-concuerror="../concuerror"
-results="./results"
-prevDir=`pwd`
-
-# Cleanup temp files
-cd `dirname $0`
-find . -name '*.beam' -exec rm {} \;
-rm -rf $results/*
-
-# Compile scenarios.erl
-erlc scenarios.erl
-
-# If we have arguments we should use this as
-# tests, otherwise check them all
-if [ $# -eq 0 ]; then
- tests=(`ls -d suites/*/src/*`)
-else
- tests=("$@")
-fi
-
-# For every test do
-for test in "${tests[@]}"; do
- unset files
- unset temp
- temp=(`echo $test | sed -e 's/suites\/\(\w\+\)\/src\/\(\w\+\)\(\.erl\)\?/\1 \2/'`)
- suite="${temp[0]}"
- name="${temp[1]}"
- if [ -d $test ]; then
- # Our test is a multi module directory
- dir=$test
- mod="test"
- files=(`ls $dir/*.erl`)
- else
- # Our test is a single module file
- dir=${test%/*}
- mod=$name
- files=$test
- fi
- # Create a dir to save the results
- mkdir -p $results/$suite
- # Compile it
- erlc -W0 -o $dir $dir/$mod.erl
- # And extract scenarios from it
- erl -noinput -pa . -pa $dir -s scenarios extract $mod -s init stop | \
- while read line; do
- # Get function and preemption bound
- unset temp
- temp=(`echo $line | sed -e 's/{\w\+,\(\w\+\),\(\w\+\)}/\1 \2/'`)
- fun="${temp[0]}"
- preb="${temp[1]}"
- printf "Running test %s-%s (%s, %s).." $suite $name $fun $preb
- # And run concuerror
- $concuerror --target $mod $fun --files "${files[@]}" \
- --output $results/$suite/$name-$fun-$preb.txt \
- --preb $preb --quiet
- diff -I '<[0-9]\+\.[0-9]\+\.[0-9]\+>' \
- -I '#Ref<[0-9\.]\+>' \
- -uw suites/$suite/results/$name-$fun-$preb.txt \
- $results/$suite/$name-$fun-$preb.txt &> /dev/null
- if [ $? -eq 0 ]; then
- printf "\033[01;32mok\033[00m\n"
- else
- printf "\033[01;31mfailed\033[00m\n"
- fi
- done
-done
-
-# Cleanup temp files
-find . -name '*.beam' -exec rm {} \;
-cd $prevDir

0 comments on commit 5a14efe

Please sign in to comment.