Permalink
Fetching contributors…
Cannot retrieve contributors at this time
893 lines (799 sloc) 29.9 KB
# compiler.py
# Original Author: Jeff Cameron (jeff@jpcameron.com)
import collections
import errno
import fnmatch
import os
import os.path
import re
import subprocess
import sys
import util
try:
from server_info import server_info
MEMORY_LIMIT = server_info.get('memory_limit', 1500)
except ImportError:
MEMORY_LIMIT = 1500
BOT = "MyBot"
# Which file is used to override the detected language?
LANGUAGE_FILE = "LANGUAGE"
SAFEPATH = re.compile('[a-zA-Z0-9_.$-]+$')
class CD(object):
"""
A context manager that enters a given directory and restores the old one.
"""
def __init__(self, new_dir):
self.new_dir = new_dir
def __enter__(self):
self.org_dir = os.getcwd()
os.chdir(self.new_dir)
return self.new_dir
def __exit__(self, type, value, traceback):
os.chdir(self.org_dir)
def safeglob(pattern):
safepaths = []
for root, dirs, files in os.walk("."):
files = fnmatch.filter(files, pattern)
for fname in files:
if SAFEPATH.match(fname):
safepaths.append(os.path.join(root, fname))
return safepaths
def safeglob_multi(patterns):
safepaths = []
for pattern in patterns:
safepaths.extend(safeglob(pattern))
return safepaths
def nukeglob(pattern):
paths = safeglob(pattern)
for path in paths:
try:
os.unlink(path)
except OSError as e:
if e.errno != errno.ENOENT and e.errno != errno.EACCES:
raise
def _run_cmd(cmd, working_dir, timelimit):
"""
Run a compilation command in the sandbox.
:param cmd: The command to run.
:param working_dir: The directory to run the command in.
:param timelimit: The number of seconds the command is allowed to run.
:return: The value of stdout as well as any errors that occurred.
"""
absoluteWorkingDir = os.path.abspath(working_dir)
cmd = "sudo -H -iu bot_compilation bash -c \"cd "+absoluteWorkingDir+"; "+cmd+"\""
print(cmd)
process = subprocess.Popen(cmd, cwd=working_dir, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
try:
rawOut, rawErrors = process.communicate(timeout=timelimit)
outString = rawOut.decode("utf-8").strip()
out = outString.split("\n") if outString.isspace() == False and outString != "" else None
errorsString = rawErrors.decode("utf-8").strip()
errors = errorsString.split("\n") if errorsString.isspace() == False and errorsString != "" else None
except subprocess.TimeoutExpired:
out = []
errors = ["Compilation timed out with command %s" % (cmd,)]
# Clean up any processes that didn't exit cleanly
util.kill_processes_as("bot_compilation")
return out, errors, process.returncode
def check_path(path, errors):
print(path)
if not os.path.exists(path):
errors.append("Output file " + str(os.path.basename(path)) + " was not created.")
return False
else:
return True
class Compiler(object):
"""
A way to compile an uploaded bot.
"""
def compile(self, bot_dir, globs, errors, timelimit):
raise NotImplementedError
class ChmodCompiler(Compiler):
"""
A compiler that simply sets the executable flag on a file.
"""
def __init__(self, language):
self.language = language
def __str__(self):
return "ChmodCompiler: %s" % (self.language,)
def compile(self, bot_dir, globs, errors, timelimit):
with CD(bot_dir):
for f in safeglob_multi(globs):
try:
os.chmod(f, 0o755)
except Exception as e:
errors.append("Error chmoding %s - %s\n" % (f, e))
return True
class CSharpMonoCompiler(Compiler):
def compile(self, bot_dir, globs, errors, timelimit):
with CD(bot_dir):
print("Looking for *.cs files:")
cs_files = safeglob("*.cs")
print(str(len(cs_files)) + " found")
if len(cs_files) == 0:
print("No *.cs files found, nothing to do, exiting successfully.")
return True
print("Looking for *.dll files:")
dll_files = safeglob("*.dll")
print(str(len(dll_files)) + " found")
try:
cmd_args = ["mcs", "-warn:0", "-optimize+", "-pkg:dotnet", "-out:MyBot.exe"]
if len(dll_files) != 0:
cmd_args.append("-r:" + ",".join(dll_files))
cmd_args += cs_files
cmdline = " ".join(cmd_args)
cmd_out, cmd_errors, _ = _run_cmd(cmdline, bot_dir, timelimit)
if not cmd_errors:
check_path(os.path.join(bot_dir, "MyBot.exe"), cmd_errors)
if cmd_errors:
cmd_errors += cmd_out
if cmd_errors:
errors += cmd_errors
return False
except:
pass
return True
class ExternalCompiler(Compiler):
"""
A compiler that calls an external process.
"""
def __init__(self, args, separate=False, out_files=[], out_ext=None, trailing_args=[]):
self.args = args
# Arguments that must follow the filename (e.g. `-lm` for C compilers)
self.trailing_args = trailing_args
self.separate = separate
self.out_files = out_files
self.out_ext = out_ext
self.stderr_re = re.compile("WARNING: IPv4 forwarding is disabled")
def __str__(self):
return "ExternalCompiler: %s" % (' '.join(self.args),)
def compile(self, bot_dir, globs, errors, timelimit):
with CD(bot_dir):
print("GLOBS: " + ", ".join(globs))
files = safeglob_multi(globs)
if len("".join(globs)) != 0 and len(files) == 0:
# no files to compile
return True
try:
if self.separate:
for filename in files:
print("file: " + filename)
cmdline = " ".join(self.args + [filename] + self.trailing_args)
cmd_out, cmd_errors, returncode = _run_cmd(cmdline, bot_dir, timelimit)
cmd_errors = self.cmd_error_filter(cmd_out, cmd_errors, returncode)
if not cmd_errors:
for ofile in self.out_files:
check_path(os.path.join(bot_dir, ofile), cmd_errors)
if self.out_ext:
oname = os.path.splitext(filename)[0] + self.out_ext
check_path(os.path.join(bot_dir, oname), cmd_errors)
if cmd_errors:
cmd_errors += cmd_out
if cmd_errors:
errors += cmd_errors
return False
else:
cmdline = " ".join(self.args + files + self.trailing_args)
print("Files: " + " ".join(files))
cmd_out, cmd_errors, returncode = _run_cmd(cmdline, bot_dir, timelimit)
cmd_errors = self.cmd_error_filter(cmd_out, cmd_errors, returncode)
if not cmd_errors:
for ofile in self.out_files:
check_path(os.path.join(bot_dir, ofile), cmd_errors)
if self.out_ext:
for filename in files:
oname = os.path.splitext(filename)[0] + self.out_ext
check_path(os.path.join(bot_dir, oname), cmd_errors)
if cmd_errors:
cmd_errors += cmd_out
if cmd_errors:
errors += cmd_errors
return False
except:
pass
return True
def cmd_error_filter(self, cmd_out, cmd_errors, returncode):
cmd_errors = [line for line in cmd_errors if line is None or self.stderr_re.search(line) is None]
return cmd_errors
class ErrorFilterCompiler(ExternalCompiler):
def __init__(self, args, separate=False, out_files=[], out_ext=None, stdout_is_error=False, skip_stdout=0, filter_stdout=None, filter_stderr=None):
ExternalCompiler.__init__(self, args, separate, out_files, out_ext)
self.stdout_is_error = stdout_is_error
self.skip_stdout = skip_stdout;
if filter_stdout is None:
self.stdout_re = None
else:
self.stdout_re = re.compile(filter_stdout)
if filter_stderr is None:
self.stderr_re = None
else:
self.stderr_re = re.compile(filter_stderr)
def __str__(self):
return "ErrorFilterCompiler: %s" % (' '.join(self.args),)
def cmd_error_filter(self, cmd_out, cmd_errors, returncode):
cmd_errors = ExternalCompiler.cmd_error_filter(self, cmd_out, cmd_errors, returncode)
if self.skip_stdout > 0:
del cmd_out[:self.skip_stdout]
# Somehow there are None values in the output
if self.stdout_re is not None:
cmd_out = [line for line in cmd_out if
line is None or not self.stdout_re.search(line)]
if self.stderr_re is not None:
cmd_errors = [line for line in cmd_errors if
line is None or self.stderr_re.search(line) is None]
if self.stdout_is_error:
return [line for line in cmd_out if line is not None] + cmd_errors
return cmd_errors
class ReturncodeCompiler(ExternalCompiler):
"""
A compiler that returns an error if the return code is not 0.
"""
def __init__(self, args, separate=False, out_files=[], out_ext=None):
ExternalCompiler.__init__(self, args, separate, out_files, out_ext)
def __str__(self):
return "ReturncodeCompiler: %s" % (' '.join(self.args),)
def cmd_error_filter(self, cmd_out, cmd_errors, returncode):
if returncode == 0:
return []
result = [
"Error compiling with command {}".format(' '.join(self.args)),
"Process exited with return code {}".format(returncode),
"stdout was:",
]
result.extend(line for line in (cmd_out or []) if line is not None)
result.append("stderr was:")
result.extend(line for line in (cmd_errors or []) if line is not None)
return result
class TargetCompiler(Compiler):
def __init__(self, args, replacements, outflag="-o"):
self.args = args
self.replacements = replacements
self.outflag = outflag
def __str__(self):
return "TargetCompiler: %s" % (' '.join(self.args),)
def compile(self, bot_dir, globs, errors, timelimit):
with CD(bot_dir):
sources = safeglob_multi(globs)
try:
for source in sources:
head, ext = os.path.splitext(source)
if ext in self.replacements:
target = head + self.replacements[ext]
else:
errors.append("Could not determine target for source file %s." % source)
return False
cmdline = " ".join(self.args + [self.outflag, target, source])
cmd_out, cmd_errors, _ = _run_cmd(cmdline, bot_dir, timelimit)
if cmd_errors:
errors += cmd_errors
return False
except:
pass
return True
PYTHON_EXT_COMPILER = '''"from distutils.core import setup
from distutils.extension import read_setup_file
setup(ext_modules = read_setup_file('setup_exts'), script_args = ['-q', 'build_ext', '-i'])"'''
# Arrays of compiler commands to run for each language.
comp_args = {
"Ada": [
["gcc-4.4", "-O3", "-funroll-loops", "-c"],
["gnatbind"],
["gnatlink", "-o", BOT],
],
"CMake": [
["cmake", "."],
["make", "-j2"],
],
"C": [
["gcc", "-O3", "-funroll-loops", "-c"],
["gcc", "-O2", "-lm", "-o", BOT],
],
"C#/.NET Core": [
["dotnet", "restore"],
["dotnet", "build", "-c", "Release", "-o", "."],
],
"Clojure": [
["lein", "uberjar"],
],
"VB/Mono": [
["vbnc", "-out:%s.exe" % BOT],
],
"C++": [
["g++", "-O3", "-w", "-c"],
["g++", "-O2", "-lm", "-o", BOT],
],
"D": [
["dmd", "-O", "-inline", "-release", "-noboundscheck", "-version=StdLoggerDisableLogging", "-of" + BOT],
],
"Elixir": [
["yes", "|", "mix", "deps.get"],
["mix", "escript.build"],
],
"F#/.NET Core": [
["dotnet", "restore"],
["dotnet", "build", "-c", "Release", "-o", "."],
],
"Groovy": [
["groovyc"],
["jar", "cfe", BOT + ".jar", BOT],
],
"Julia": [
["JULIA_DEPOT_PATH=\$(pwd) julia -e 'using Pkg; Pkg.activate(\\\".\\\"); Pkg.instantiate()'"],
],
"Kotlin": [
["kotlinc", "-include-runtime", "-d", BOT + ".jar"],
],
"Haskell": [
["ghc", "--make", BOT + ".hs", "-O", "-v0", "-rtsopts"],
],
"Haskell/Stack": [
["stack", "build", "--allow-different-user"],
["stack", "install", "--local-bin-path", "."],
],
"Java": [
["javac", "-encoding", "UTF-8", "-J-Xmx%sm" % (MEMORY_LIMIT)],
],
"Lisp": [
['sbcl', '--dynamic-space-size', str(MEMORY_LIMIT), '--script', BOT + '.lisp'],
],
"OCaml": [
["ocamlbuild -lib unix", BOT + ".native"],
],
"Pascal": [
["fpc", "-Mdelphi", "-Si", "-O3", "-Xs", "-v0", "-o" + BOT],
],
"Python": [
["python3.6", "-c", PYTHON_EXT_COMPILER],
],
"Python (PyPy)": [
["pypy3", "-c", PYTHON_EXT_COMPILER],
],
"Rust": [
["cargo", "rustc", "--release", "-q", "--bins", "--", "-Awarnings"],
],
"Scala": [
["scalac"],
],
"Swift": [
["swift", "build", "--configuration", "release"],
["chmod", "-R", "777", ".build"],
["mv", ".build/release/MyBot", "./MyBot"],
],
}
targets = {
"C" : { ".c" : ".o" },
"C++": { ".c" : ".o", ".cpp" : ".o", ".cc" : ".o" },
}
Language = collections.namedtuple(
"Language",
['name', 'out_file', 'main_code_file', 'command',
'nukeglobs', 'compilers'])
# The actual languages supported.
languages = (
Language("Ada", BOT, "MyBot.adb",
"./MyBot",
["*.ali"],
[(["*.adb"], ExternalCompiler(comp_args["Ada"][0])),
(["MyBot.ali"], ExternalCompiler(comp_args["Ada"][1])),
(["MyBot.ali"], ExternalCompiler(comp_args["Ada"][2]))]
),
Language("C++", BOT, "CMakeLists.txt",
"./MyBot",
["*.o", BOT],
[
([], ExternalCompiler(comp_args["CMake"][0])),
([], ExternalCompiler(comp_args["CMake"][1])),
]
),
Language("C", BOT, "MyBot.c",
"./MyBot",
["*.o", BOT],
[(["*.c"], TargetCompiler(comp_args["C"][0], targets["C"])),
(["*.o"], ExternalCompiler(comp_args["C"][1], trailing_args=["-lm"]))]
),
Language("C#/.NET Core", BOT + ".dll", "MyBot.csproj",
"dotnet MyBot.dll",
[BOT + ".dll"],
[
([], ExternalCompiler(comp_args["C#/.NET Core"][0])),
([], ExternalCompiler(comp_args["C#/.NET Core"][1])),
]
),
Language("C#/Mono", "MyBot.exe", "MyBot.cs",
"mono MyBot.exe",
["MyBot.exe"],
[([], CSharpMonoCompiler())]
),
Language("VB/Mono", BOT +".exe", "MyBot.vb",
"mono MyBot.exe",
[BOT + ".exe"],
[(["*.vb"],
ExternalCompiler(comp_args["VB/Mono"][0], out_files=['MyBot.exe']))]
),
Language("C++", BOT, "MyBot.cpp",
"./MyBot",
["*.o", BOT],
[
(["*.c", "*.cpp", "*.cc"],
TargetCompiler(comp_args["C++"][0], targets["C++"])),
(["*.o"], ExternalCompiler(comp_args["C++"][1]))
]
),
Language("Clojure", "target/"+BOT +".jar", "project.clj",
"java -cp target/MyBot.jar MyBot",
[],
[([""], ErrorFilterCompiler(comp_args["Clojure"][0], filter_stderr="(Retrieving|Compiling)"))]
),
Language("CoffeeScript", BOT +".coffee", "MyBot.coffee",
"coffee MyBot.coffee",
[],
[(["*.coffee"], ChmodCompiler("CoffeeScript"))]
),
Language("D", BOT, "MyBot.d",
"./MyBot",
["*.o", BOT],
[(["*.d"], ExternalCompiler(comp_args["D"][0]))]
),
Language("Dart", BOT +".dart", "MyBot.dart",
"dart MyBot.dart",
[], [(["*.dart"], ChmodCompiler("Dart"))]
),
Language("Elixir", BOT, "mix.exs",
"./MyBot",
[],
[
([], ErrorFilterCompiler(comp_args["Elixir"][0])),
([], ErrorFilterCompiler(comp_args["Elixir"][1])),
]
),
Language("Erlang", "my_bot.beam", "my_bot.erl",
"erl -hms"+ str(MEMORY_LIMIT) +"m -smp disable -noshell -s my_bot start -s init stop",
["*.beam"],
[(["*.erl"], ExternalCompiler(["erlc"], out_ext=".beam"))]
),
Language("F#/.NET Core", BOT + ".dll", "MyBot.fsproj",
"dotnet MyBot.dll",
[BOT + ".dll"],
[
([], ExternalCompiler(comp_args["F#/.NET Core"][0])),
([], ExternalCompiler(comp_args["F#/.NET Core"][1])),
]
),
Language("Forth", BOT +".fs", "MyBot.fs",
"gforth-fast MyBot.fs -e bye",
[],
[(["*.fs"], ChmodCompiler("Forth"))]
),
Language("Go", BOT +".go", "MyBot.go",
"export GOPATH=\"$(pwd)\"; go run MyBot.go",
[],
[(["*.go"], ChmodCompiler("Go"))]
),
Language("Groovy", BOT +".jar", "MyBot.groovy",
"java -Xmx" + str(MEMORY_LIMIT) + "m -cp MyBot.jar:/usr/share/groovy/embeddable/groovy-all-1.7.5.jar MyBot",
["*.class, *.jar"],
[(["*.groovy"], ExternalCompiler(comp_args["Groovy"][0])),
(["*.class"], ExternalCompiler(comp_args["Groovy"][1]))]
),
Language("Haskell", BOT, "MyBot.hs",
"./MyBot +RTS -M" + str(MEMORY_LIMIT) + "m",
[BOT],
[([""], ExternalCompiler(comp_args["Haskell"][0]))]
),
Language("Haskell/Stack", BOT, "stack.yaml",
"./MyBot +RTS -M" + str(MEMORY_LIMIT) + "m",
[BOT],
[
([""], ReturncodeCompiler(comp_args["Haskell/Stack"][0])),
([""], ReturncodeCompiler(comp_args["Haskell/Stack"][1])),
]
),
Language("Java", BOT +".java", "MyBot.java",
"java MyBot",
["*.class", "*.jar"],
[(["*.java"], ErrorFilterCompiler(comp_args["Java"][0], filter_stderr="Note:", out_files=["MyBot.class"]))]
),
Language("JavaScript", BOT +".js", "MyBot.js",
"node MyBot.js",
[],
[(["*.js"], ChmodCompiler("JavaScript"))]
),
Language("JAR", BOT +".jar", "MyBot.jar",
"java -jar MyBot.jar",
[],
[(["*.jar"], ChmodCompiler("JAR"))]
),
Language("Julia", BOT +".jl", "MyBot.jl",
"JULIA_DEPOT_PATH=$(pwd) julia --project MyBot.jl",
[],
[
([], ExternalCompiler(comp_args["Julia"][0])),
(["*.jl"], ChmodCompiler("Julia")),
]
),
Language("Kotlin", BOT +".jar", "MyBot.kt",
"java -Xmx" + str(MEMORY_LIMIT) + "m -cp MyBot.jar MyBot",
["*.class, *.jar"],
[
(["*.kt"], ExternalCompiler(comp_args["Kotlin"][0])),
]
),
Language("Lisp", BOT, "MyBot.lisp",
"./MyBot --dynamic-space-size " + str(MEMORY_LIMIT),
[BOT],
[([""], ExternalCompiler(comp_args["Lisp"][0]))]
),
Language("Lua/LuaJIT", BOT +".lua", "MyBot.lua",
"luajit MyBot.lua",
[],
[(["*.lua"], ChmodCompiler("Lua"))]
),
Language("Lua/Lua5.3", BOT +".lua53", "MyBot.lua53",
"lua5.3 MyBot.lua53",
[],
[
(["*.lua"], ChmodCompiler("Lua")),
(["*.lua53"], ChmodCompiler("Lua")),
]
),
Language("OCaml", BOT +".native", "MyBot.ml",
"./MyBot.native",
[BOT + ".native"],
[([""], ExternalCompiler(comp_args["OCaml"][0]))]
),
Language("Octave", BOT + ".m", "MyBot.m",
"octave -qf MyBot.m",
[],
[(["*.m"], ChmodCompiler("Octave"))]
),
Language("Pascal", BOT, BOT + ".pas",
"./" + BOT,
[BOT, "*.o", "*.ppu"],
[([BOT + ".pas"], ErrorFilterCompiler(comp_args["Pascal"][0],
stdout_is_error=True, skip_stdout=2,
filter_stderr='^/usr/bin/ld: warning: link.res contains output sections; did you forget -T\?$'))]
),
Language("Perl", BOT +".pl", "MyBot.pl",
"perl MyBot.pl",
[],
[(["*.pl"], ChmodCompiler("Perl"))]
),
Language("PHP", BOT +".php", "MyBot.php",
"php MyBot.php",
[],
[(["*.php"], ChmodCompiler("PHP"))]
),
Language("Python", BOT +".py", "MyBot.py",
"python3.6 MyBot.py",
["*.pyc"],
[(["*.py"], ChmodCompiler("Python")),
(["setup_exts"], ErrorFilterCompiler(comp_args["Python"][0], separate=True, filter_stderr='-Wstrict-prototypes'))]
),
Language("Python (PyPy)", BOT +".pypy", "MyBot.pypy",
"pypy3 MyBot.pypy",
["*.pyc"],
[
(["*.py"], ChmodCompiler("Python (PyPy)")),
(["*.pypy"], ChmodCompiler("Python (PyPy)")),
(["setup_exts"], ErrorFilterCompiler(comp_args["Python (PyPy)"][0], separate=True, filter_stderr='-Wstrict-prototypes'))
]
),
Language("Racket", BOT +".rkt", "MyBot.rkt",
"racket MyBot.rkt",
[],
[(["*.rkt"], ChmodCompiler("Racket"))]
),
Language("Ruby", BOT +".rb", "MyBot.rb",
"ruby MyBot.rb",
[],
[(["*.rb"], ChmodCompiler("Ruby"))]
),
Language("Rust", "target/release/my_bot", "Cargo.toml",
"target/release/my_bot",
[],
[([""], ErrorFilterCompiler(comp_args["Rust"][0], filter_stderr="warning:"))]
),
Language("Scala", BOT +".scala", "MyBot.scala",
'scala -J-Xmx'+ str(MEMORY_LIMIT) +'m -howtorun:object MyBot',
["*.scala, *.jar"],
[(["*.java"], ExternalCompiler(comp_args["Java"][0])), (["*.scala"], ExternalCompiler(comp_args["Scala"][0]))]
),
Language("Scheme", BOT +".ss", "MyBot.ss",
"./MyBot",
[],
[(["*.ss"], ChmodCompiler("Scheme"))]
),
Language("Swift", BOT, "Package.swift",
"./MyBot",
[],
[
([""], ErrorFilterCompiler(comp_args["Swift"][0], filter_stderr="warning:")),
([""], ExternalCompiler(comp_args["Swift"][1])),
([""], ExternalCompiler(comp_args["Swift"][2])),
]
),
Language("Tcl", BOT +".tcl", "MyBot.tcl",
"tclsh8.5 MyBot.tcl",
[],
[(["*.tcl"], ChmodCompiler("Tcl"))]
),
)
def compile_function(language, bot_dir, timelimit):
with CD(bot_dir):
print("cd " + bot_dir)
for glob in language.nukeglobs:
print("nuke " + glob)
nukeglob(glob)
errors = []
for globs, compiler in language.compilers:
try:
if not compiler.compile(bot_dir, globs, errors, timelimit):
return False, errors
except Exception as exc:
raise
errors.append("Compiler %s failed with: %s"
% (compiler, exc))
return False, errors
print("joining")
compiled_bot_file = os.path.join(bot_dir, language.out_file)
print("joined")
return check_path(compiled_bot_file, errors), errors
_LANG_NOT_FOUND = """Did not find a recognized MyBot.* file.
Please add one of the following filenames to your zip file:
%s"""
def detect_language(bot_dir):
with CD(bot_dir):
detected_langs = [
lang for lang in languages if os.path.exists(lang.main_code_file)
]
# if we have cmake-based compilation, then remove any other autodetected C and C++ compilers
if any(lang.main_code_file == "CMakeLists.txt" for lang in detected_langs):
detected_langs = [lang for lang in detected_langs if
lang.main_code_file == "CMakeLists.txt" or (lang.name != "C" and lang.name != "C++")]
if len(detected_langs) > 1:
return None, ['Found multiple MyBot.* files: \n'+
'\n'.join([l.main_code_file for l in detected_langs])]
elif len(detected_langs) == 0:
return None, [_LANG_NOT_FOUND % (
'\n'.join(l.name +": "+ l.main_code_file for l in languages),)]
else:
return detected_langs[0], None
def detect_language_file(bot_dir):
with CD(bot_dir):
try:
with open(LANGUAGE_FILE, 'r') as lang_file:
print("detected %s file" % LANGUAGE_FILE)
language_name = lang_file.readline().strip()
if not language_name:
return None
else:
return language_name
except IOError:
return None
def get_run_cmd(submission_dir):
with CD(submission_dir):
if os.path.exists('run.sh'):
with open('run.sh') as f:
for line in f:
if line[0] != '#':
return line.rstrip('\r\n')
def get_run_lang(submission_dir):
with CD(submission_dir):
if os.path.exists('run.sh'):
with open('run.sh') as f:
for line in f:
if line[0] == '#':
return line[1:-1]
INSTALL_ERROR_START = "Possible errors running install.sh. stdout output as follows:"
INSTALL_ERROR_MID = "End of install.sh stdout. stderr as follows:"
INSTALL_ERROR_END = "End of install.sh output."
def truncate_errors(install_stdout, install_errors, language_detection_errors,
compile_errors, max_error_len=10*1024):
"""
Combine lists of errors into a single list under a maximum length.
"""
install_stdout = install_stdout or []
install_errors = install_errors or []
language_detection_errors = language_detection_errors or []
compile_errors = compile_errors or []
all_errors = install_stdout + install_errors + language_detection_errors + compile_errors
result = []
if sum(len(line) for line in all_errors) <= max_error_len:
if install_stdout or install_errors:
result.append(INSTALL_ERROR_START)
result.extend(install_stdout)
result.append(INSTALL_ERROR_MID)
result.extend(install_errors)
result.append(INSTALL_ERROR_END)
result.extend(language_detection_errors)
result.extend(compile_errors)
return result
def bound_errors(source, bound):
total_length = sum(len(line) for line in source)
if total_length <= bound:
return total_length, source
length = 0
current = 0
result = []
# Take 1/3 from start of errors
while current < len(source) and (
length == 0 or
length + len(source[current]) < bound // 3):
result.append(source[current])
length += len(source[current])
current += 1
if current < len(source):
result.append("...(output truncated)...")
end_errors = []
end = current
current = -1
while current >= -(len(source) - end) and (
len(end_errors) == 0 or
length + len(source[current])) < bound:
end_errors.append(source[current])
length += len(source[current])
current -= 1
result.extend(reversed(end_errors))
return length, result
remaining_length = max_error_len
if install_stdout or install_errors:
result.append(INSTALL_ERROR_START)
used, lines = bound_errors(install_stdout, 0.2 * max_error_len)
remaining_length -= used
result.extend(lines)
result.append(INSTALL_ERROR_MID)
used, lines = bound_errors(install_errors,
max(0.3 * max_error_len,
0.5 * max_error_len - used))
remaining_length -= used
result.extend(lines)
result.append(INSTALL_ERROR_END)
_, lines = bound_errors(language_detection_errors + compile_errors, remaining_length)
result.extend(lines)
return result
def compile_anything(bot_dir, installTimeLimit=600, timelimit=600, max_error_len=10*1024):
install_stdout = []
install_errors = []
if os.path.exists(os.path.join(bot_dir, "install.sh")):
install_stdout, install_errors, _ = _run_cmd(
"bash ./install.sh", bot_dir, installTimeLimit)
detected_language, language_errors = detect_language(bot_dir)
print("detected language " + str(detected_language))
if not detected_language:
return "Unknown", truncate_errors(install_stdout, install_errors,
language_errors, [], max_error_len)
print("compiling")
compiled, compile_errors = compile_function(detected_language, bot_dir, timelimit)
if not compiled or install_errors:
return (detected_language.name,
truncate_errors(install_stdout, install_errors,
language_errors, compile_errors, max_error_len))
print("done compiling")
name = detected_language.name
run_cmd = detected_language.command
run_filename = os.path.join(bot_dir, 'run.sh')
print("filename:")
print(run_filename)
try:
with open(run_filename, 'wb') as f:
f.write(('#%s\n%s\n' % (name, run_cmd)).encode('utf-8'))
print("file:")
with open(run_filename, 'r') as f:
for line in f:
print(line)
except Exception as e:
print("error")
print(e)
# allow LANGUAGE file to override language name
override_name = detect_language_file(bot_dir)
if override_name:
name = override_name
return name, None
if __name__ == "__main__":
if len(sys.argv) > 1:
workingPath = sys.argv[1]
lang, errors = compile_anything(workingPath)
print("Language:", lang)
if errors:
print("--- Errors: ---")
print("\n".join(errors))