Permalink
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 249 lines (213 sloc) 8.35 KB
#!/usr/bin/env python3
# Run this script from an empty temporary directory; it places all of
# its output in the current working directory.
#
# You probably need to run this program with the include path for
# csmith.h; for example:
# -I/usr/local/include/csmith-2.3.0
#
# You may pass -D or -U flags for the C pre-processor as well.
#
# In addition, you may pass flags for csmith. This is especially useful
# if you have the random seed from a previous run and you want to retry
# it:
# --seed <seed>
import os
import os.path
import re
import shlex
import subprocess
import sys
# How long to wait, in seconds, for various subprocesses to finish.
csmith_timeout = 90
compiler_timeout = 120
prog_timeout = 8
# Disable operations Corrode can't handle yet.
#
# NOTE: Although Corrode supports pointers, we can't use them together
# with global variables because Rust doesn't allow storing the address
# of one static variable in another. Given the choice between globals or
# pointers, we currently have to pick globals because otherwise csmith's
# generated test harness doesn't report anything meaningful.
CSMITH_FLAGS = [
'--no-arrays',
'--no-bitfields',
'--no-jumps',
'--no-packed-struct',
'--no-pointers',
'--no-unions',
'--no-volatiles',
'--no-builtins',
]
# Disable all warnings from the compilers, using `-w` for gcc/clang and
# `-A warnings` for rustc. csmith-generated programs trigger plenty of
# warnings and that's OK. We only care that the program can be compiled
# without errors.
#
# When pre-processing the C source with either the C compiler or
# Corrode, we also define macros that make `csmith.h` use fewer features
# of standard C, so Corrode has a better shot at translating it.
CFLAGS = ['-w', '-DCSMITH_MINIMAL', '-DUSE_MATH_MACROS']
RUSTFLAGS = ['-A', 'warnings']
compiler_env = os.environ.copy()
# If ccache is installed, there's no point caching the builds of these
# randomly generated C files.
compiler_env['CCACHE_DISABLE'] = '1'
def test(cfile):
try:
# Generate a new random C program. Pass --output last so the
# user can't accidentally override it.
subprocess.run(['csmith'] + CSMITH_FLAGS + ['--output', cfile],
check=True, timeout=csmith_timeout, env=compiler_env)
except subprocess.SubprocessError:
# If csmith failed, we won't find any interesting Corrode or
# Rust bugs with this program.
return
return check(cfile)
def check(cfile):
rsfile = os.path.splitext(cfile)[0] + '.rs'
cprog = './via-c'
rsprog = './via-rust'
# If csmith didn't generate any variables that it could update the
# CRC with, this program is boring and we shouldn't bother testing
# it.
with open(cfile) as f:
crc_count = sum('transparent_crc(' in line for line in f)
if not crc_count:
return
try:
# Compile with a real C compiler for reference.
subprocess.run([
'gcc',
'-o', cprog,
cfile
] + CFLAGS, check=True, timeout=compiler_timeout, env=compiler_env)
except subprocess.SubprocessError:
# If the C compiler failed, we won't find any interesting
# Corrode or Rust bugs with this program.
return
try:
# Translate this C program to Rust.
subprocess.run([
'corrode',
cfile
] + CFLAGS, check=True, timeout=compiler_timeout, env=compiler_env)
# Compile the generated Rust program.
subprocess.run([
'rustc',
'-o', rsprog,
rsfile
] + RUSTFLAGS, check=True, timeout=compiler_timeout, env=compiler_env)
except subprocess.SubprocessError as e:
# Found a compile-time bug!
return "compiling via Rust failed: {}".format(e)
# Get reference output from the version compiled with a native C
# compiler. If any of the reference output looks wrong, comparing to
# the output from the Rust version won't be interesting.
try:
ref_long = subprocess.run(
[cprog, '1'], check=True, timeout=prog_timeout,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# Long output must have one line per call to `transparent_crc`.
if ref_long.stderr or ref_long.stdout.count(b'\n') != crc_count:
return
ref_short = subprocess.run(
[cprog], check=True, timeout=prog_timeout,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# Short output must have one checksum line.
if ref_short.stderr or not re.match(b'checksum = [0-9a-fA-F]+\n$', ref_short.stdout):
return
except subprocess.SubprocessError:
# Give up if either mode of the reference version timed out or
# exited unsuccessfully.
return
# Report any differences between the reference output and the Rust
# version, for both short and long output. Prefer long-output
# differences as they allow reducing smaller test cases, but if the
# long output is the same we should still catch differences in the
# short output.
return (
result_differences([rsprog, '1'], ref_long) or
result_differences([rsprog], ref_short)
)
def result_differences(cmd, ref):
cmdstr = ' '.join(cmd)
# Get output from the Rust version. If the command fails in any way,
# return a message rather than propagating an exception.
try:
test = subprocess.run(
cmd, timeout=prog_timeout,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except subprocess.TimeoutExpired as e:
return "'{}' timeout after {} seconds".format(cmdstr, e.timeout)
if test.stderr:
return "'{}' error:\n{}".format(cmdstr, test.stderr.decode())
if test.returncode != 0:
return "'{}' failed with status {}".format(cmdstr, test.returncode)
if not test.stdout:
return "'{}' produced no output".format(cmdstr)
ref_lines = ref.stdout.splitlines()
test_lines = test.stdout.splitlines()
if len(ref_lines) != len(test_lines):
return "'{}' produced wrong output:\n{}".format(cmdstr, test.stdout.decode())
differences = [
"expected '{}', got '{}'".format(r.decode(), t.decode())
for (r,t) in zip(ref_lines, test_lines)
if r != t
]
if differences:
return "'{}' produced wrong output:\n{}".format(cmdstr, '\n'.join(differences))
# Otherwise, the test output matched the reference output. Hooray!
return None
def creduce(origfile, errorfile):
reduced = 'reduced.c'
# Preprocess the source. C-Reduce seems to be able to produce
# smaller output if the input was already pre-processed.
subprocess.run(['gcc',
'-P', '-E',
'-o', reduced, origfile
] + CFLAGS, check=True)
# Package up a recursive call to the current script into a temporary
# shell script, preserving all the options we were passed.
script = "test.sh"
extra_arg = '--reduce-check={},{}'.format(reduced,
os.path.abspath(errorfile))
with open(script, 'w') as f:
f.write('#!/bin/sh\n')
f.write(' '.join(
shlex.quote(arg)
for arg in ['exec'] + sys.argv + [extra_arg]
) + ' > /dev/null 2>&1\n')
os.chmod(script, 0o755)
# Run creduce using the generated shell script.
subprocess.run(['creduce', script, reduced], check=True)
if __name__ == "__main__":
checkpath = None
for arg in sys.argv[1:]:
# Put include-path and macro definitions in CFLAGS.
if arg.startswith(('-I', '-D', '-U')):
CFLAGS.append(arg)
# Check if we're being called recursively under creduce.
elif arg.startswith("--reduce-check="):
checkpath = arg.split('=', 1)[1].split(',', 1)
# Pass anything else through to csmith.
else:
CSMITH_FLAGS.append(arg)
if checkpath is not None:
# This input is "interesting" if `check` returned the same error
# message that we started with.
with open(checkpath[1]) as f:
expected_error = f.read()
if check(checkpath[0]) == expected_error:
exit(0)
# Otherwise it's boring and creduce should backtrack.
exit(1)
cfile = 'random.c'
result = test(cfile)
if result:
print(result)
with open('error', 'w') as f:
f.write(result)
print("reducing to a minimal test case...")
creduce(cfile, 'error')
exit(1)