Skip to content
Permalink
Browse files
[build.webkit.org] Use GitHub checkouts
https://bugs.webkit.org/show_bug.cgi?id=239072
<rdar://problem/91567092>

Reviewed by Ryan Haddad.

* Tools/CISupport/build-webkit-org/config.json: Convert schedulers to GitHub based branch names.
* Tools/CISupport/build-webkit-org/loadConfig.py: Ditto.
* Tools/CISupport/build-webkit-org/master.cfg: Enable incoming hooks from GitHub.
(load_password): Added.
* Tools/CISupport/build-webkit-org/steps.py:
(CheckOutSource): Replace with GitHub based checkout.
(CleanUpGitIndexLock): Replaced SVNCleanup.
(SVNCleanup): Replace with CleanUpGitIndexLock.
(ShowIdentifier.start): `got_revision` will bet hash, not subversion revision.
* Tools/CISupport/build-webkit-org/steps_unittest.py:
(TestShowIdentifier.test_success):
(TestShowIdentifier.test_failure):

Canonical link: https://commits.webkit.org/249946@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@293323 268f45cc-cd09-0410-ab3c-d52691b4dbfc
  • Loading branch information
JonWBedard committed Apr 25, 2022
1 parent 71f3e1b commit 91e76ff30d923951dcf180aca7eb6c93c95825b9
Show file tree
Hide file tree
Showing 6 changed files with 104 additions and 50 deletions.
@@ -614,7 +614,7 @@
}
],

"schedulers": [ { "type": "AnyBranchScheduler", "name": "trunk", "change_filter": "trunk_filter", "treeStableTimer": 45.0,
"schedulers": [ { "type": "AnyBranchScheduler", "name": "main", "change_filter": "main_filter", "treeStableTimer": 45.0,
"builderNames": ["GTK-Linux-64-bit-Release-Build", "GTK-Linux-64-bit-Debug-Build",
"GTK-Linux-64-bit-Release-Clang",
"GTK-Linux-64-bit-Release-Debian-Stable-Build",
@@ -627,7 +627,8 @@
"WPE-Linux-64-bit-Release-Build", "WPE-Linux-64-bit-Debug-Build",
"WPE-Linux-64-bit-Release-Non-Unified-Build"]
},
{ "type": "PlatformSpecificScheduler", "platform": "mac-monterey", "branch": "trunk", "treeStableTimer": 45.0,

{ "type": "PlatformSpecificScheduler", "platform": "mac-monterey", "branch": "main", "treeStableTimer": 45.0,
"builderNames": ["Apple-Monterey-Release-Build", "Apple-Monterey-Debug-Build"]
},
{ "type": "Triggerable", "name": "monterey-release-tests-wk1",
@@ -669,7 +670,7 @@
{ "type": "Triggerable", "name": "monterey-applesilicon-release-tests-jsc",
"builderNames": ["Apple-Monterey-AppleSilicon-Release-JSC-Tests"]
},
{ "type": "PlatformSpecificScheduler", "platform": "mac-bigsur", "branch": "trunk", "treeStableTimer": 45.0,
{ "type": "PlatformSpecificScheduler", "platform": "mac-bigsur", "branch": "main", "treeStableTimer": 45.0,
"builderNames": ["Apple-BigSur-Release-Build", "Apple-BigSur-Debug-Build", "Apple-BigSur-LLINT-CLoop-BuildAndTest", "Apple-BigSur-XCBuild-Release-Build"]
},
{ "type": "Triggerable", "name": "bigsur-release-tests-jsc",
@@ -711,25 +712,25 @@
{ "type": "Triggerable", "name": "bigsur-debug-tests-wk2-gpuprocess",
"builderNames": ["Apple-BigSur-Debug-WK2-GPUProcess-Tests"]
},
{ "type": "PlatformSpecificScheduler", "platform": "ios-15", "branch": "trunk", "treeStableTimer": 45.0,
{ "type": "PlatformSpecificScheduler", "platform": "ios-15", "branch": "main", "treeStableTimer": 45.0,
"builderNames": ["Apple-iOS-15-Release-Build"]
},
{ "type": "PlatformSpecificScheduler", "platform": "ios-simulator-15", "branch": "trunk", "treeStableTimer": 45.0,
{ "type": "PlatformSpecificScheduler", "platform": "ios-simulator-15", "branch": "main", "treeStableTimer": 45.0,
"builderNames": ["Apple-iOS-15-Simulator-Release-Build", "Apple-iOS-15-Simulator-Debug-Build"]
},
{ "type": "PlatformSpecificScheduler", "platform": "tvos-15", "branch": "trunk", "treeStableTimer": 45.0,
{ "type": "PlatformSpecificScheduler", "platform": "tvos-15", "branch": "main", "treeStableTimer": 45.0,
"builderNames": ["Apple-tvOS-15-Release-Build"]
},
{ "type": "PlatformSpecificScheduler", "platform": "tvos-simulator-15", "branch": "trunk", "treeStableTimer": 45.0,
{ "type": "PlatformSpecificScheduler", "platform": "tvos-simulator-15", "branch": "main", "treeStableTimer": 45.0,
"builderNames": ["Apple-tvOS-Simulator-15-Release-Build"]
},
{ "type": "PlatformSpecificScheduler", "platform": "watchos-8", "branch": "trunk", "treeStableTimer": 45.0,
{ "type": "PlatformSpecificScheduler", "platform": "watchos-8", "branch": "main", "treeStableTimer": 45.0,
"builderNames": ["Apple-watchOS-8-Release-Build"]
},
{ "type": "PlatformSpecificScheduler", "platform": "watchos-simulator-8", "branch": "trunk", "treeStableTimer": 45.0,
{ "type": "PlatformSpecificScheduler", "platform": "watchos-simulator-8", "branch": "main", "treeStableTimer": 45.0,
"builderNames": ["Apple-watchOS-Simulator-8-Release-Build"]
},
{ "type": "PlatformSpecificScheduler", "platform": "win", "branch": "trunk", "treeStableTimer": 45.0,
{ "type": "PlatformSpecificScheduler", "platform": "win", "branch": "main", "treeStableTimer": 45.0,
"builderNames": ["Apple-Win-10-Release-Build", "Apple-Win-10-Debug-Build"]
},
{ "type": "Triggerable", "name": "ios-simulator-15-release-gpuprocess-arm64-tests-wk2",
@@ -810,8 +811,8 @@
{ "type": "Triggerable", "name": "wpe-linux-64-debug-tests-js",
"builderNames": ["WPE-Linux-64-bit-Debug-JS-Tests"]
},
{ "type": "Nightly", "name": "NightlyScheduler", "change_filter": "trunk_filter",
"branch": "trunk", "hour": 22, "minute": 0,
{ "type": "Nightly", "name": "NightlyScheduler", "change_filter": "main_filter",
"branch": "main", "hour": 22, "minute": 0,
"builderNames": ["GTK-Linux-64bit-Release-Packaging-Nightly-Ubuntu1804", "GTK-Linux-64bit-Release-Packaging-Nightly-Ubuntu2004",
"WPE-Linux-64bit-Release-Packaging-Nightly-Ubuntu1804", "WPE-Linux-64bit-Release-Packaging-Nightly-Ubuntu2004"]
}
@@ -36,7 +36,7 @@
from factories import *
import wkbuild

trunk_filter = ChangeFilter(branch=["trunk", None])
main_filter = ChangeFilter(branch=["main", None])

BUILDER_NAME_LENGTH_LIMIT = 70
STEP_NAME_LENGTH_LIMIT = 50
@@ -7,6 +7,20 @@ from buildbot.plugins import util
from datetime import timedelta
from loadConfig import *

from buildbot.www.hooks.github import GitHubEventHandler


def load_password(name, default=None):
if os.getenv(name):
return os.getenv(name)
try:
passwords = json.load(open('passwords.json'))
return passwords.get(name, default)
except Exception as e:
print('Error in finding {} in passwords.json'.format(name))
return default


# This is work-around for https://bugs.webkit.org/show_bug.cgi?id=222361
from buildbot.process.buildstep import BuildStep
BuildStep.warn_deprecated_if_oldstyle_subclass = lambda self, name: None
@@ -28,9 +42,8 @@ c['www']['ui_default_config'] = {
}

if not is_test_mode_enabled:
passwords = json.load(open('passwords.json'))
admin_username = passwords.get('ADMIN_USERNAME')
admin_password = passwords.get('ADMIN_PASSWORD')
admin_username = load_password('ADMIN_USERNAME')
admin_password = load_password('ADMIN_PASSWORD')
if not admin_username or not admin_password:
print('\n\nERROR: Admin username/password missing from passwords.json.\n')
sys.exit(1)
@@ -43,6 +56,14 @@ if not is_test_mode_enabled:
c['www']['auth'] = auth
c['www']['authz'] = authz

c['www']['change_hook_dialects'] = dict(
github={
'class': GitHubEventHandler,
'secret': load_password('GITHUB_HOOK_SECRET'),
'token': load_password('GITHUB_COM_ACCESS_TOKEN'),
},
)

c['protocols'] = {'pb': {'port': 17000}}
c['projectName'] = 'WebKit'
c['projectURL'] = 'https://webkit.org'
@@ -51,11 +72,10 @@ if is_test_mode_enabled:
c['buildbotURL'] = 'http://localhost:8710/'
else:
c['buildbotURL'] = 'https://build.webkit.org/'
passwords = json.load(open('passwords.json'))
db_url = passwords.get('DB_URL')
db_name = passwords.get('DB_NAME')
db_username = passwords.get('DB_USERNAME')
db_password = passwords.get('DB_PASSWORD')
db_url = load_password('DB_URL')
db_name = load_password('DB_NAME')
db_username = load_password('DB_USERNAME')
db_password = load_password('DB_PASSWORD')
if None in [db_url, db_name, db_username, db_password]:
print('\n\nERROR: Database information missing from passwords.json.\n')
sys.exit(1)
@@ -24,7 +24,7 @@
from buildbot.process import buildstep, factory, logobserver, properties
from buildbot.process.results import Results, SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION, RETRY
from buildbot.steps import master, shell, transfer, trigger
from buildbot.steps.source.svn import SVN
from buildbot.steps.source import git
from twisted.internet import defer

import json
@@ -128,37 +128,49 @@ def start(self):
return defer.succeed(None)


class CheckOutSource(SVN, object):
class CheckOutSource(git.Git):
name = 'clean-and-update-working-directory'
CHECKOUT_DELAY_AND_MAX_RETRIES_PAIR = (0, 2)
haltOnFailure = False

def __init__(self, **kwargs):
kwargs['repourl'] = 'https://svn.webkit.org/repository/webkit/trunk'
kwargs['mode'] = 'incremental'
kwargs['logEnviron'] = False
super(CheckOutSource, self).__init__(**kwargs)
def __init__(self, repourl='https://github.com/WebKit/WebKit.git', **kwargs):
super(CheckOutSource, self).__init__(repourl=repourl,
retry=self.CHECKOUT_DELAY_AND_MAX_RETRIES_PAIR,
timeout=2 * 60 * 60,
alwaysUseLatest=True,
Copy link
@aj062

aj062 Apr 25, 2022

Member

I think alwaysUseLatest shouldn't be used here, since we don't want to checkout the latest revision for every build.

logEnviron=False,
method='clean',
progress=True,
**kwargs)

def getResultSummary(self):
if self.results == FAILURE:
self.build.addStepsAfterCurrentStep([SVNCleanup()])
self.build.addStepsAfterCurrentStep([CleanUpGitIndexLock()])

if self.results != SUCCESS:
return {'step': 'Failed to updated working directory'}
else:
return {'step': 'Cleaned and updated working directory'}


class SVNCleanup(shell.ShellCommand):
name = 'svn-cleanup'
command = ['svn', 'cleanup']
descriptionDone = ['Run svn cleanup']
class CleanUpGitIndexLock(shell.ShellCommand):
name = 'clean-git-index-lock'
command = ['rm', '-f', '.git/index.lock']
descriptionDone = ['Deleted .git/index.lock']

def __init__(self, **kwargs):
super(SVNCleanup, self).__init__(timeout=10 * 60, logEnviron=False, **kwargs)
super(CleanUpGitIndexLock, self).__init__(timeout=2 * 60, logEnviron=False, **kwargs)

def start(self):
platform = self.getProperty('platform', '*')
if platform == 'wincairo':
self.command = ['del', r'.git\index.lock']

return shell.ShellCommand.start(self)

def evaluateCommand(self, cmd):
self.build.buildFinished(['svn issue, retrying build'], RETRY)
return super(SVNCleanup, self).evaluateCommand(cmd)
self.build.buildFinished(['Git issue, retrying build'], RETRY)
return super(CleanUpGitIndexLock, self).evaluateCommand(cmd)


class InstallWin32Dependencies(shell.Compile):
@@ -1268,7 +1280,7 @@ def start(self):
self.log_observer = logobserver.BufferLogObserver()
self.addLogObserver('stdio', self.log_observer)
revision = self.getProperty('got_revision')
self.setCommand(['python3', 'Tools/Scripts/git-webkit', 'find', 'r{}'.format(revision)])
self.setCommand(['python3', 'Tools/Scripts/git-webkit', 'find', revision])
return shell.ShellCommand.start(self)

def evaluateCommand(self, cmd):
@@ -509,12 +509,12 @@ def tearDown(self):

def test_success(self):
self.setupStep(ShowIdentifier())
self.setProperty('got_revision', '272692')
self.setProperty('got_revision', 'd3f2b739b65eda1eeb651991a3554dfaeebdfe0b')
self.expectRemoteCommands(
ExpectShell(workdir='wkdir',
timeout=600,
logEnviron=False,
command=['python3', 'Tools/Scripts/git-webkit', 'find', 'r272692']) +
command=['python3', 'Tools/Scripts/git-webkit', 'find', 'd3f2b739b65eda1eeb651991a3554dfaeebdfe0b']) +
ExpectShell.log('stdio', stdout='Identifier: 233939@main') +
0,
)
@@ -525,12 +525,12 @@ def test_success(self):

def test_failure(self):
self.setupStep(ShowIdentifier())
self.setProperty('got_revision', '272692')
self.setProperty('got_revision', 'd3f2b739b65eda1eeb651991a3554dfaeebdfe0b')
self.expectRemoteCommands(
ExpectShell(workdir='wkdir',
timeout=600,
logEnviron=False,
command=['python3', 'Tools/Scripts/git-webkit', 'find', 'r272692']) +
command=['python3', 'Tools/Scripts/git-webkit', 'find', 'd3f2b739b65eda1eeb651991a3554dfaeebdfe0b']) +
ExpectShell.log('stdio', stdout='Unexpected failure') +
2,
)
@@ -1059,7 +1059,7 @@ def test_failure(self):
return self.runStep()


class TestSVNCleanup(BuildStepMixinAdditions, unittest.TestCase):
class TestCleanUpGitIndexLock(BuildStepMixinAdditions, unittest.TestCase):
def setUp(self):
self.longMessage = True
return self.setUpBuildStep()
@@ -1068,30 +1068,30 @@ def tearDown(self):
return self.tearDownBuildStep()

def test_success(self):
self.setupStep(SVNCleanup())
self.setupStep(CleanUpGitIndexLock())
self.expectRemoteCommands(
ExpectShell(
workdir='wkdir',
timeout=600,
timeout=120,
logEnviron=False,
command=['svn', 'cleanup'],
command=['rm', '-f', '.git/index.lock'],
) + 0,
)
self.expectOutcome(result=SUCCESS, state_string='Run svn cleanup')
self.expectOutcome(result=SUCCESS, state_string='Deleted .git/index.lock')
return self.runStep()

def test_failure(self):
self.setupStep(SVNCleanup())
self.setupStep(CleanUpGitIndexLock())
self.expectRemoteCommands(
ExpectShell(
workdir='wkdir',
timeout=600,
timeout=120,
logEnviron=False,
command=['svn', 'cleanup'],
command=['rm', '-f', '.git/index.lock'],
) + 2
+ ExpectShell.log('stdio', stdout='Unexpected error.'),
)
self.expectOutcome(result=FAILURE, state_string='Run svn cleanup (failure)')
self.expectOutcome(result=FAILURE, state_string='Deleted .git/index.lock (failure)')
return self.runStep()


@@ -1,3 +1,24 @@
2022-04-13 Jonathan Bedard <jbedard@apple.com>

[build.webkit.org] Use GitHub checkouts
https://bugs.webkit.org/show_bug.cgi?id=239072
<rdar://problem/91567092>

Reviewed by Ryan Haddad.

* CISupport/build-webkit-org/config.json: Convert schedulers to GitHub based branch names.
* CISupport/build-webkit-org/loadConfig.py: Ditto.
* CISupport/build-webkit-org/master.cfg: Enable incoming hooks from GitHub.
(load_password): Added.
* CISupport/build-webkit-org/steps.py:
(CheckOutSource): Replace with GitHub based checkout.
(CleanUpGitIndexLock): Replaced SVNCleanup.
(SVNCleanup): Replace with CleanUpGitIndexLock.
(ShowIdentifier.start): `got_revision` will bet hash, not subversion revision.
* CISupport/build-webkit-org/steps_unittest.py:
(TestShowIdentifier.test_success):
(TestShowIdentifier.test_failure):

2022-04-25 Tim Nguyen <ntim@apple.com>

Fix typo in Tools/CISupport/ews-build/steps.py

0 comments on commit 91e76ff

Please sign in to comment.