diff --git a/master/custom/__init__.py b/master/custom/__init__.py index e69de29bb..1e17b7054 100644 --- a/master/custom/__init__.py +++ b/master/custom/__init__.py @@ -0,0 +1,6 @@ +MAIN_BRANCH_VERSION = "3.13" +CUSTOM_BRANCH_NAME = "custom" +# The Git branch is called "main", but we give it a different name in buildbot. +# See git_branches in master/master.cfg. +MAIN_BRANCH_NAME = "3.x" +JUNIT_FILENAME = "test-results.xml" diff --git a/master/custom/builders.py b/master/custom/builders.py index 71f42d8d7..774cd1ac8 100644 --- a/master/custom/builders.py +++ b/master/custom/builders.py @@ -20,7 +20,6 @@ ClangUnixInstalledBuild, SharedUnixBuild, SlowNonDebugUnixBuild, - SlowSharedUnixBuild, SlowUnixInstalledBuild, LTONonDebugUnixBuild, LTOPGONonDebugBuild, @@ -240,7 +239,7 @@ def get_builders(settings): # Match builder name (excluding the branch name) of builders that should only # run on the master and "custom" branches. -ONLY_MASTER_BRANCH = ( +ONLY_MAIN_BRANCH = ( "Alpine Linux", # Cygwin is not supported on 2.7, 3.6, 3.7 "Cygwin", diff --git a/master/custom/factories.py b/master/custom/factories.py index 98bc3f608..33ddb3b4b 100644 --- a/master/custom/factories.py +++ b/master/custom/factories.py @@ -9,6 +9,8 @@ from buildbot.plugins import util +from . import (MAIN_BRANCH_VERSION, CUSTOM_BRANCH_NAME, MAIN_BRANCH_NAME, + JUNIT_FILENAME) from .steps import ( Test, Clean, @@ -18,8 +20,13 @@ UploadTestResults, ) -main_branch_version = "3.13" -CUSTOM_BRANCH_NAME = "custom" +# Python branches with regrtest --fail-rerun option +# https://github.com/python/cpython/issues/108834 +BRANCH_WITH_FAIL_RERUN = ( + MAIN_BRANCH_NAME, + #"3.12", + #"3.11", +) # This (default) timeout is for each individual test file. # It is a bit more than the default faulthandler timeout in regrtest.py @@ -40,42 +47,21 @@ def __init__(self, source, *, extra_tags=[], **kwargs): self.tags = self.factory_tags + extra_tags -class FreezeBuild(TaggedBuildFactory): - buildersuffix = ".freeze" # to get unique directory names on master - test_timeout = None - factory_tags = ["freeze"] - - def setup(self, **kwargs): - self.addStep(Configure(command=["./configure", "--prefix", "$(PWD)/target"])) - self.addStep(Compile(command=["make"])) - self.addStep( - ShellCommand( - name="install", description="installing", command=["make", "install"] - ) - ) - self.addStep( - Test( - command=[ - "make", - "-C", - "Tools/freeze/test", - "PYTHON=../../../target/bin/python3", - "OUTDIR=../../../target/freezetest", - ] - ) - ) - - ############################################################################## ############################### UNIX BUILDS ################################ ############################################################################## +def has_option(option, test_options): + # return True for option='-j' and test_options=['-uall', '-j2'] + return option in ' '.join(test_options) + + class UnixBuild(TaggedBuildFactory): configureFlags = ["--with-pydebug"] compile_environ = {} interpreterFlags = "" - testFlags = "-j2" + testFlags = ["-j2"] makeTarget = "all" test_timeout = None test_environ = {} @@ -106,22 +92,24 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs): Configure(command=configure_cmd, **oot_kwargs) ) compile = ["make", self.makeTarget] - testopts = self.testFlags - if "-R" not in self.testFlags: - testopts += " --junit-xml test-results.xml" + testopts = list(self.testFlags) + if not has_option("-R", self.testFlags): + testopts.extend(("--junit-xml", JUNIT_FILENAME)) # Timeout for the buildworker process self.test_timeout = self.test_timeout or TEST_TIMEOUT # Timeout for faulthandler faulthandler_timeout = self.test_timeout - 5 * 60 if parallel: compile = ["make", parallel, self.makeTarget] - testopts = testopts + " " + parallel - if "-j" not in testopts: - testopts = "-j2 " + testopts + testopts.append(parallel) + if not has_option("-j", testopts): + testopts.append("-j2") + if branch in BRANCH_WITH_FAIL_RERUN: + testopts.append("--fail-rerun") test = [ "make", "buildbottest", - "TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}", + "TESTOPTS=" + " ".join(testopts) + " ${BUILDBOT_TESTOPTS}", "TESTPYTHONOPTS=" + self.interpreterFlags, "TESTTIMEOUT=" + str(faulthandler_timeout), ] @@ -148,8 +136,8 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs): **oot_kwargs ) ) - if branch not in ("3",) and "-R" not in self.testFlags: - filename = "test-results.xml" + if branch not in ("3",) and not has_option("-R", self.testFlags): + filename = JUNIT_FILENAME if self.build_out_of_tree: filename = os.path.join(out_of_tree_dir, filename) self.addStep(UploadTestResults(branch, filename=filename)) @@ -174,7 +162,7 @@ class UnixVintageParserBuild(UnixBuild): class UnixRefleakBuild(UnixBuild): buildersuffix = ".refleak" - testFlags = "-R 3:3 -u-cpu" + testFlags = ["-R", "3:3", "-u-cpu"] # -R 3:3 is supposed to only require timeout x 6, but in practice, # it's much more slower. Use timeout x 10 to prevent timeout # caused by --huntrleaks. @@ -199,8 +187,8 @@ class UnixInstalledBuild(TaggedBuildFactory): factory_tags = ["installed"] def setup(self, parallel, branch, test_with_PTY=False, **kwargs): - if branch == "3.x": - branch = main_branch_version + if branch == MAIN_BRANCH_NAME: + branch = MAIN_BRANCH_VERSION elif branch == "custom": branch = "3" installed_python = "./target/bin/python%s" % branch @@ -213,19 +201,23 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs): compile = ["make", self.makeTarget] install = ["make", self.installTarget] - testopts = self.defaultTestOpts[:] + testopts = list(self.defaultTestOpts) # Timeout for the buildworker process self.test_timeout = self.test_timeout or TEST_TIMEOUT # Timeout for faulthandler faulthandler_timeout = self.test_timeout - 5 * 60 - testopts += [f"--timeout={faulthandler_timeout}"] + testopts.append(f"--timeout={faulthandler_timeout}") if parallel: compile = ["make", parallel, self.makeTarget] install = ["make", parallel, self.installTarget] - testopts = testopts + [parallel] + testopts.append(parallel) + if branch in BRANCH_WITH_FAIL_RERUN: + testopts.append("--fail-rerun") - test = [installed_python] + self.interpreterFlags - test += ["-m", "test.regrtest"] + testopts + test = [installed_python, + *self.interpreterFlags, + "-m", "test", + *testopts] self.addStep(Compile(command=compile)) self.addStep(Install(command=install)) @@ -351,10 +343,6 @@ class SlowNonDebugUnixBuild(NonDebugUnixBuild): test_timeout = SLOW_TIMEOUT -class SlowSharedUnixBuild(SharedUnixBuild): - test_timeout = SLOW_TIMEOUT - - class SlowUnixInstalledBuild(UnixInstalledBuild): test_timeout = SLOW_TIMEOUT @@ -505,7 +493,8 @@ class MacOSArmWithBrewBuild(UnixBuild): "LDFLAGS=-L/opt/homebrew/lib", ] # These tests are known to crash on M1 macs (see bpo-45289). - testFlags = UnixBuild.testFlags + " -x test_dbm test_dbm_ndbm test_shelve" + testFlags = [*UnixBuild.testFlags, + "-x", "test_dbm", "test_dbm_ndbm", "test_shelve"] ############################################################################## ############################ WINDOWS BUILDS ################################ @@ -525,12 +514,14 @@ class BaseWindowsBuild(TaggedBuildFactory): def setup(self, parallel, branch, **kwargs): build_command = self.build_command + self.buildFlags - test_command = self.test_command + self.testFlags - if "-R" not in self.testFlags: - test_command += [r"--junit-xml", r"test-results.xml"] + test_command = [*self.test_command, *self.testFlags] + if not has_option("-R", self.testFlags): + test_command.extend((r"--junit-xml", JUNIT_FILENAME)) clean_command = self.clean_command + self.cleanFlags if parallel: test_command.append(parallel) + if branch in BRANCH_WITH_FAIL_RERUN: + test_command.append("--fail-rerun") self.addStep(Compile(command=build_command)) self.addStep( ShellCommand( @@ -541,9 +532,9 @@ def setup(self, parallel, branch, **kwargs): ) ) timeout = self.test_timeout if self.test_timeout else TEST_TIMEOUT - test_command += ["--timeout", timeout - (5 * 60)] + test_command.extend(("--timeout", timeout - (5 * 60))) self.addStep(Test(command=test_command, timeout=timeout)) - if branch not in ("3",) and "-R" not in self.testFlags: + if branch not in ("3",) and not has_option("-R", self.testFlags): self.addStep(UploadTestResults(branch)) self.addStep(Clean(command=clean_command)) @@ -585,7 +576,7 @@ class Windows64BigmemBuild(BaseWindowsBuild): class Windows64RefleakBuild(Windows64Build): buildersuffix = ".refleak" - testFlags = ["-p", "x64"] + WindowsRefleakBuild.testFlags + testFlags = ["-p", "x64", *WindowsRefleakBuild.testFlags] # -R 3:3 is supposed to only require timeout x 6, but in practice, # it's much more slower. Use timeout x 10 to prevent timeout # caused by --huntrleaks. @@ -596,7 +587,7 @@ class Windows64RefleakBuild(Windows64Build): class Windows64ReleaseBuild(Windows64Build): buildersuffix = ".nondebug" buildFlags = Windows64Build.buildFlags + ["-c", "Release"] - testFlags = Windows64Build.testFlags + ["+d"] + testFlags = [*Windows64Build.testFlags, "+d"] # keep default cleanFlags, both configurations get cleaned factory_tags = ["win64", "nondebug"] @@ -611,7 +602,7 @@ class WindowsARM64Build(BaseWindowsBuild): class WindowsARM64ReleaseBuild(WindowsARM64Build): buildersuffix = ".nondebug" buildFlags = WindowsARM64Build.buildFlags + ["-c", "Release"] - testFlags = WindowsARM64Build.testFlags + ["+d"] + testFlags = [*WindowsARM64Build.testFlags, "+d"] # keep default cleanFlags, both configurations get cleaned factory_tags = ["win-arm64", "nondebug"] @@ -703,13 +694,15 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs): ) ) - testopts = self.testFlags - if "-R" not in self.testFlags: - testopts += " --junit-xml test-results.xml" + testopts = list(self.testFlags) + if not has_option("-R", self.testFlags): + testopts.extend((" --junit-xml", JUNIT_FILENAME)) if parallel: - testopts = testopts + " " + parallel - if "-j" not in testopts: - testopts = "-j2 " + testopts + testopts.append(parallel) + if not has_option("-j", self.testFlags): + testopts.append("-j2") + if branch in BRANCH_WITH_FAIL_RERUN: + testopts.append("--fail-rerun") # Timeout for the buildworker process self.test_timeout = self.test_timeout or TEST_TIMEOUT @@ -719,7 +712,7 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs): test = [ "make", "buildbottest", - "TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}", + "TESTOPTS=" + " ".join(testopts) + " ${BUILDBOT_TESTOPTS}", "TESTPYTHONOPTS=" + self.interpreterFlags, "TESTTIMEOUT=" + str(faulthandler_timeout), ] @@ -756,8 +749,8 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs): workdir=oot_host_path, ) ) - if branch not in ("3",) and "-R" not in self.testFlags: - filename = os.path.join(oot_host_path, "test-results.xml") + if branch not in ("3",) and not has_option("-R", self.testFlags): + filename = os.path.join(oot_host_path, JUNIT_FILENAME) self.addStep(UploadTestResults(branch, filename=filename)) self.addStep( Clean( diff --git a/master/custom/steps.py b/master/custom/steps.py index 38a8d2827..5a6cfd3ab 100644 --- a/master/custom/steps.py +++ b/master/custom/steps.py @@ -1,10 +1,13 @@ import re -from buildbot.steps.shell import ShellCommand, Test as BaseTest from buildbot.plugins import steps, util +from buildbot.process.results import SUCCESS, WARNINGS +from buildbot.steps.shell import ShellCommand, Test as BaseTest from buildbot.steps.source.git import Git as _Git from buildbot.steps.source.github import GitHub as _GitHub +from . import JUNIT_FILENAME + class Git(_Git): # GH-68: If "git clone" fails, mark the whole build as WARNING @@ -77,6 +80,25 @@ class Test(BaseTest): # Give SIGTERM 30 seconds to shut things down before SIGKILL. sigtermTime = 30 + # Treat "regrtest --fail-rerun" exit code (5) as WARNINGS + # https://github.com/python/cpython/issues/108834 + decodeRC = { + 0: SUCCESS, + + # Treat --fail-rerun exit code (5) to WARNINGS, when a test failed but + # passed when run again in verbose mode in a fresh process (unstable + # test). + 5: WARNINGS, # EXITCODE_RERUN_FAIL + + # Any exit code not present in the dictionary is treated as FAILURE. + # So there is no need to map each regrtest exit code to FAILURE. + # + # 2: FAILURE, # EXITCODE_BAD_TEST + # 3: FAILURE, # EXITCODE_ENV_CHANGED + # 4: FAILURE, # EXITCODE_NO_TESTS_RAN + # 130: FAILURE, # EXITCODE_INTERRUPTED + } + def evaluateCommand(self, cmd): if cmd.didFail(): self.setProperty("test_failed_to_build", True) @@ -139,7 +161,7 @@ class UploadTestResults(steps.FileUpload): def _has_the_build_failed(self, build): return self.getProperty("test_failed_to_build") - def __init__(self, branch, filename="test-results.xml"): + def __init__(self, branch, filename=JUNIT_FILENAME): super().__init__( doStepIf=self._has_the_build_failed, workersrc=filename, diff --git a/master/custom/workers.py b/master/custom/workers.py index cf1da6cac..70e301ce7 100644 --- a/master/custom/workers.py +++ b/master/custom/workers.py @@ -6,6 +6,8 @@ from buildbot.plugins import worker as _worker +from custom.factories import MAIN_BRANCH_NAME + # By default, the buildmaster sends a simple, non-blocking message to each # worker every hour. These keepalives ensure that traffic is flowing over the @@ -232,7 +234,7 @@ def get_workers(settings): cpw( name="ware-alpine", tags=['linux', 'unix', 'alpine', 'docker', 'amd64', 'x86-64'], - branches=['3.x'], + branches=[MAIN_BRANCH_NAME], ), cpw( name="ware-gentoo-x86", diff --git a/master/master.cfg b/master/master.cfg index f2ea8c6bc..529d0fe1b 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -39,7 +39,7 @@ for k in list(sys.modules): if k.split(".")[0] in ["custom"]: sys.modules.pop(k) -from custom.factories import CUSTOM_BRANCH_NAME +from custom import MAIN_BRANCH_NAME, CUSTOM_BRANCH_NAME from custom.auth import set_up_authorization # noqa: E402 from custom.email_formatter import MESSAGE_FORMATTER # noqa: E402 from custom.pr_reporter import GitHubPullRequestReporter # noqa: E402 @@ -57,7 +57,7 @@ from custom.builders import ( get_builders, STABLE, DAILYBUILDERS, - ONLY_MASTER_BRANCH, + ONLY_MAIN_BRANCH, ) # noqa: E402 def set_up_sentry(): @@ -121,7 +121,7 @@ c["workers"] = [w.bb_worker for w in WORKERS] # repo url, buildbot category name, git branch name git_url = str(settings.git_url) git_branches = [ - (git_url, "3.x", "main"), + (git_url, MAIN_BRANCH_NAME, "main"), (git_url, "3.12", "3.12"), (git_url, "3.11", "3.11"), (git_url, "3.10", "3.10"), @@ -207,8 +207,8 @@ for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches): dailybuildernames = [] for name, worker_name, buildfactory, stability, tier in BUILDERS: if any( - pattern in name for pattern in ONLY_MASTER_BRANCH - ) and branchname not in ("3.x", CUSTOM_BRANCH_NAME): + pattern in name for pattern in ONLY_MAIN_BRANCH + ) and branchname not in (MAIN_BRANCH_NAME, CUSTOM_BRANCH_NAME): # Workers known to be broken on older branches: let's focus on # supporting these platforms in the main branch. Don't run # custom jobs neither.