diff --git a/.ci/buildbot/worker.py b/.ci/buildbot/worker.py new file mode 100644 index 0000000000000..88702a6cd2e81 --- /dev/null +++ b/.ci/buildbot/worker.py @@ -0,0 +1,393 @@ +import argparse +import filecmp +import os +import pathlib +import re +import shlex +import shutil +import subprocess +import sys +import traceback +from contextlib import contextmanager + +_SHQUOTE_WINDOWS_ESCAPEDCHARS = re.compile(r'(["\\])') +_SHQUOTE_WINDOWS_QUOTEDCHARS = re.compile("[ \t\n]") + + +def _shquote_windows(txt): + """shlex.quote for Windows cmd.exe""" + txt = txt.replace("%", "%%") + quoted = re.sub(_SHQUOTE_WINDOWS_ESCAPEDCHARS, r"\\\1", txt) + if len(quoted) == len(txt) and not _SHQUOTE_WINDOWS_QUOTEDCHARS.search(txt): + return txt + else: + return '"' + quoted + '"' + + +def shjoin(args): + """Convert a list of shell arguments to an appropriately quoted string.""" + if os.name in set(("nt", "os2", "ce")): + return " ".join(map(_shquote_windows, args)) + else: + return shlex.join(args) + + +def report(msg): + """Emit a message to the build log. Appears in red font. Lines surrounded by @@@ may be interpreted as meta-instructions.""" + print(msg, file=sys.stderr, flush=True) + + +def run_command(cmd, shell=False, **kwargs): + """Report which command is being run, then execute it using subprocess.check_call.""" + report(f"Running: {cmd if shell else shjoin(cmd)}") + sys.stderr.flush() + subprocess.check_call(cmd, shell=shell, **kwargs) + + +def _remove_readonly(func, path, _): + """Clear the readonly bit and reattempt the removal.""" + os.chmod(path, os.stat.S_IWRITE) + func(path) + + +def rmtree(path): + """ + Remove directory path and all its subdirectories. Includes a workaround for Windows where shutil.rmtree errors on read-only files. + + Taken from official Pythons docs + https://docs.python.org/3/library/shutil.html#rmtree-example + """ + shutil.rmtree(path, onexc=_remove_readonly) + + +def checkout(giturl, sourcepath): + """ + Use git to checkout the remote repository giturl at local directory sourcepath. + + If the repository already exists, clear all local changes and check out the latest main branch. + """ + if not os.path.exists(sourcepath): + run_command(["git", "clone", giturl, sourcepath]) + + # Reset repository state no matter what there was before + run_command(["git", "-C", sourcepath, "stash", "--all"]) + run_command(["git", "-C", sourcepath, "stash", "clear"]) + + # Fetch and checkout the newest + run_command(["git", "-C", sourcepath, "fetch", "origin"]) + run_command(["git", "-C", sourcepath, "checkout", "origin/main", "--detach"]) + + +@contextmanager +def step(step_name, halt_on_fail=False): + """Report a new build step being started. + + Use like this:: + with step("greet-step"): + report("Hello World!") + """ + # Barrier to separate stdio output for the the previous step + sys.stderr.flush() + sys.stdout.flush() + + report(f"@@@BUILD_STEP {step_name}@@@") + if halt_on_fail: + report("@@@HALT_ON_FAILURE@@@") + try: + yield + except Exception as e: + if isinstance(e, subprocess.CalledProcessError): + report(f"{shjoin(e.cmd)} exited with return code {e.returncode}.") + report("@@@STEP_FAILURE@@@") + else: + traceback.print_exc() + report("@@@STEP_EXCEPTION@@@") + if halt_on_fail: + # Do not continue with the next steps, but allow except/finally blocks to execute + raise e + + +class Worker: + """Helper class to keep context in a worker.run() environment""" + + def __init__(self, args, clean, clobber, workdir, jobs, cachefile, llvmsrcroot): + self.args = args + self.clean = clean + self.clobber = clobber + self.workdir = workdir + self.jobs = jobs + self.cachefile = cachefile + self.llvmsrcroot = llvmsrcroot + + def in_llvmsrc(self, path): + """Convert a path in the llvm-project source checkout to an absolute path""" + return os.path.join(self.llvmsrcroot, path) + + def in_workdir(self, path): + """Convert a path in the workdir to an absolute path""" + return os.path.join(self.workdir, path) + + def run_ninja( + self, targets: list = [], *, builddir, ccache_stats: bool = False, **kwargs + ): + """Run ninja in builddir. If self.jobs is set, automatically adds an -j option to set the number of parallel jobs. + + Parameters + ---------- + targets : list + List of build targets; build the default target 'all' if list is empty + builddir + Directory of the build.ninja file + ccache_stats : bool + If true, also emit ccache statistics when finishing the build + """ + cmd = ["ninja"] + if builddir is not None: + cmd += ["-C", builddir] + cmd += targets + if self.jobs: + cmd.append(f"-j{self.jobs}") + if ccache_stats: + run_command(["ccache", "-z"]) + try: + run_command(cmd, **kwargs) + finally: + # TODO: Pipe to stderr to separate from build log itself + run_command(["ccache", "-sv"]) + else: + run_command(cmd, **kwargs) + + @contextmanager + def step(self, step_name, halt_on_fail=False): + """Convenience wrapper for step()""" + with step(step_name, halt_on_fail=halt_on_fail) as s: + yield s + + def run_command(self, *args, **kwargs): + """Convenience wrapper for run_command()""" + return run_command(*args, **kwargs) + + def rmtree(self, *args, **kwargs): + """Convenience wrapper for rmtree()""" + return rmtree(*args, *kwargs) + + def checkout(self, giturl, sourcepath): + """Convenience wrapper for checkout()""" + return checkout(giturl, sourcepath) + + +def convert_bool(v): + """Convert input to bool type + + Use to convert the value of bool environment variables. Specifically, the buildbot master sets 'false' to build properties, which by default Python would interpret as true-ish. + """ + match v: + case None: + return False + case bool(b): + return b + case str(s): + return not s.strip().upper() in ["", "0", "N", "NO", "FALSE", "OFF"] + case _: + return bool(v) + + +def relative_if_possible(path, relative_to): + """Like os.path.relpath, but does not fail if path is not a parent of relative_to; keeps the original path in that case""" + path = os.path.normpath(path) + if not os.path.isabs(path): + # Path is already relative (assumed to relative_to) + return path + try: + result = os.path.relpath(path, start=relative_to) + return result if result else path + except ValueError: + return path + + +@contextmanager +def run( + scriptpath, + llvmsrcroot, + parser=None, + clobberpaths=[], + workerjobs=None, + always_clobber=False, +): + """ + Runs the boilerplate for a ScriptedBuilder buildbot. It is not necessary to + use this function (one can also all run_command() etc. directly), but allows + for some more flexibility and safety checks. Arguments passed to this + function represent the worker configuration. + + We use the term 'clean' for resetting the worker to an empty state. This + involves deleting ${prefix}/llvm.src as well as ${prefix}/build. + The term 'clobber' means deleting build artifacts, but not already + downloaded git repositories. Build artifacts including build- and + install-directories, but not source directories. Changes in the llvm.src + directory will be reset before the next build anyway. Clobber is necessary + if the build instructions change. Otherwise, we try an incremental build. + We consider 'clean' to imply 'clean_obj'. + + A buildbot worker will invoke this script using this directory structure, + where ${prefix} is a dedicated directory for this builder: + ${prefix}/llvm.src # Checkout location for the llvm-source + ${prefix}/build # cwd when launching the build script + + The build script is called with --workdir=. parameter, i.e. the build + artifacts are written into ${prefix}/build. When cleaning, the worker (NOT + the build script) will delete ${prefix}/llvm.src; Deleting any contents of + ${prefix}/build is to be done by the builder script, e.g. by this function. + The builder script can choose to not delete the complete workdir, e.g. + additional source checkouts such as the llvm-test-suite. + + The buildbot master will set the 'clean' build property and the environment + variable BUILDBOT_CLEAN when in the GUI the option "Clean source code and + build directory" is checked by the user. The 'clean_obj' build property and + the BUILDBOT_CLEAN_OBJ environment variable will be set when either the + "Clean build directory" GUI option is set, or the master detects a change + to a CMakeLists.txt or *.cmake file. + + Parameters + ---------- + scriptpath + Pass __file__ from the main builder script. + llvmsrcroot + Absolute path to the llvm-project source checkout. Since the builder + script is supposed to be a part of llvm-project itself, the builder + script can compute it from __file__. + parser + Use this argparse.ArgumentParser instead of creating a new one. Allows + adding additional command line switched in addition to the pre-defined + ones. Build script are encouraged to apply the pre-defined switches. + clobberpaths + Directories relative to workdir that need to be deleted if the build + configuration changes (due to changes of CMakeLists.txt or changes of + configuration parameters). Typically, only source checkouts are not + deleted. + workerjobs + Default number of build and test jobs; If set, expected to be the number + of jobs of the actual buildbot worker that executes this script. Can be + overridden using the --jobs parameter so in case someone needs to + reproduce this build, they can adjust the number of jobs for the + reproducer platform. Alternatively, the worker can set the + BUILDBOT_JOBS environment variable or keep ninja/llvm-lit defaults. + always_clobber + Always clobber the build artifacts, i.e. disable incremental builds. + """ + + scriptpath = os.path.abspath(scriptpath) + llvmsrcroot = os.path.abspath(llvmsrcroot) + stem = pathlib.Path(scriptpath).stem + workdir_default = f"{stem}.workdir" + + jobs_default = None + if jobs_env := os.environ.get("BUILDBOT_JOBS"): + jobs_default = int(jobs_env) + if not jobs_default: + jobs_default = workerjobs + if not jobs_default: + jobs_default = None + + parser = parser or argparse.ArgumentParser( + allow_abbrev=True, + description="When executed without arguments, builds the worker's " + f"LLVM build configuration in {os.path.abspath(workdir_default)}. " + "Some build configuration parameters can be altered using the " + "following switches:", + ) + parser.add_argument( + "--workdir", + default=workdir_default, + help="Use this dir as workdir to write the build artifact into. " + "--workdir=. uses the current directory.\nWarning: This directory " + "might be deleted", + ) + parser.add_argument( + "--cachefile", + default=relative_if_possible( + pathlib.Path(scriptpath).with_suffix(".cmake"), llvmsrcroot + ), + help="File containing the initial values for the CMakeCache.txt for " + "the llvm build.", + ) + parser.add_argument( + "--clean", + type=bool, + default=convert_bool(os.environ.get("BUILDBOT_CLEAN")), + help="Delete the entire workdir before starting the build, including " + "source directories", + ) + parser.add_argument( + "--clobber", + type=bool, + default=always_clobber + or convert_bool(os.environ.get("BUILDBOT_CLOBBER")) + or convert_bool(os.environ.get("BUILDBOT_CLEAN_OBJ")), + help="Delete build artifacts before starting the build", + ) + parser.add_argument( + "--jobs", "-j", default=jobs_default, help="Number of build- and test-jobs" + ) + args = parser.parse_args() + + workdir = os.path.abspath(args.workdir) + clean = args.clean + clobber = args.clobber + cachefile = os.path.join(llvmsrcroot, args.cachefile) + oldcwd = os.getcwd() + + prevcachepath = os.path.join(workdir, "prevcache.cmake") + if cachefile and os.path.exists(prevcachepath): + # Force clobber if cache file has changed; a new cachefile does not override entries already present in CMakeCache.txt + if not filecmp.cmp( + os.path.join(llvmsrcroot, args.cachefile), prevcachepath, shallow=False + ): + clobber = True + + # Safety check + parentdir = os.path.dirname(scriptpath) + while True: + if os.path.samefile(parentdir, workdir): + raise Exception( + f"Cannot use {args.workdir} as workdir; a '--clean' build would rmtree the llvm-project source in {parentdir} as well" + ) + newparentdir = os.path.dirname(parentdir) + if newparentdir == parentdir: + break + parentdir = newparentdir + + w = Worker( + args, + clean=clean, + clobber=clobber, + workdir=workdir, + jobs=args.jobs, + cachefile=cachefile, + llvmsrcroot=llvmsrcroot, + ) + + if clean: + # Ensure that the cwd is not the directory we are going to delete. This would not work under Windows. We will chdir to workdir later anyway. + os.chdir("/") + + with w.step(f"clean"): + rmtree(workdir) + elif clobber: + with w.step(f"clobber"): + for d in clobberpaths: + rmtree(os.path.join(workdir, d)) + os.path.unlink(prevcachepath) + + os.makedirs(workdir, exist_ok=True) + os.chdir(workdir) + + # Remember used cachefile in case it changes + if cachefile: + shutil.copy( + os.path.join(oldcwd, llvmsrcroot, args.cachefile), + os.path.join(oldcwd, prevcachepath), + ) + + os.environ["NINJA_STATUS"] = "[%p/%es :: %u->%r->%f (of %t)] " + yield w diff --git a/polly/ci/.gitignore b/polly/ci/.gitignore new file mode 100644 index 0000000000000..f2198f0f4ef8e --- /dev/null +++ b/polly/ci/.gitignore @@ -0,0 +1 @@ +/*.workdir diff --git a/polly/ci/polly-x86_64-linux-test-suite.cmake b/polly/ci/polly-x86_64-linux-test-suite.cmake new file mode 100644 index 0000000000000..f5699804f2109 --- /dev/null +++ b/polly/ci/polly-x86_64-linux-test-suite.cmake @@ -0,0 +1,11 @@ +# General settings +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") +set(CMAKE_C_COMPILER_LAUNCHER "ccache" CACHE STRING "") +set(CMAKE_CXX_COMPILER_LAUNCHER "ccache" CACHE STRING "") +set(LLVM_ENABLE_LLD ON CACHE BOOL "") + +set(LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "") +set(LLVM_ENABLE_PROJECTS "clang;polly" CACHE STRING "") +set(LLVM_TARGETS_TO_BUILD "X86" CACHE STRING "") + +set(LLVM_POLLY_LINK_INTO_TOOLS ON CACHE BOOL "") diff --git a/polly/ci/polly-x86_64-linux-test-suite.py b/polly/ci/polly-x86_64-linux-test-suite.py new file mode 100644 index 0000000000000..be60798847777 --- /dev/null +++ b/polly/ci/polly-x86_64-linux-test-suite.py @@ -0,0 +1,70 @@ +#! /usr/bin/env python3 + +import os +import sys + +# Adapt to location in source tree +llvmsrcroot = os.path.normpath(f"{__file__}/../../..") + +sys.path.insert(0, os.path.join(llvmsrcroot, ".ci/buildbot")) +import worker + +llvmbuilddir = "llvm.build" +llvminstalldir = "llvm.install" +testsuitesrcdir = "testsuite.src" +testsuitebuilddir = "testsuite.build" + +with worker.run( + __file__, + llvmsrcroot, + clobberpaths=[llvmbuilddir, testsuitebuilddir, llvminstalldir], +) as w: + with w.step("configure-llvm", halt_on_fail=True): + cmakecmd = [ + "cmake", + f"-S{w.in_llvmsrc('llvm')}", + f"-B{llvmbuilddir}", + "-GNinja", + f"-C{w.in_llvmsrc(w.cachefile)}", + f"-DCMAKE_INSTALL_PREFIX={llvminstalldir}", + ] + if w.jobs: + cmakecmd.append(f"-DLLVM_LIT_ARGS=-sv;-j{w.jobs}") + w.run_command(cmakecmd) + + with w.step("build-llvm", halt_on_fail=True): + w.run_ninja(builddir=llvmbuilddir, ccache_stats=True) + + with w.step("check-polly"): + w.run_ninja(["check-polly"], builddir=llvmbuilddir) + + with w.step("install-llvm", halt_on_fail=True): + w.run_ninja(["install"], builddir=llvmbuilddir) + + with w.step("checkout-testsuite", halt_on_fail=True): + w.checkout("https://github.com/llvm/llvm-test-suite", testsuitesrcdir) + + with w.step("configure-testsuite", halt_on_fail=True): + jobsarg = f";-j{w.jobs}" if w.jobs else "" + w.run_command( + [ + "cmake", + f"-S{testsuitesrcdir}", + f"-B{testsuitebuilddir}", + "-GNinja", + "-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_C_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang", + f"-DCMAKE_CXX_COMPILER={os.path.abspath(llvminstalldir)}/bin/clang++", + f"-DTEST_SUITE_LIT={os.path.abspath(llvmbuilddir)}/bin/llvm-lit", + f"-DTEST_SUITE_LLVM_SIZE={os.path.abspath(llvmbuilddir)}/bin/llvm-size", + "-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", + "-DTEST_SUITE_EXTRA_CXX_FLAGS=-Wno-unused-command-line-argument -mllvm -polly", + f"-DLLVM_LIT_ARGS=-sv{jobsarg};-o;report.json", + ] + ) + + with w.step("build-testsuite", halt_on_fail=True): + w.run_ninja(builddir=testsuitebuilddir) + + with w.step("check-testsuite"): + w.run_ninja(["check"], builddir=testsuitebuilddir)