Skip to content

Commit

Permalink
[build.webkit.org] Upload WebKit build logs to s3
Browse files Browse the repository at this point in the history 
https://bugs.webkit.org/show_bug.cgi?id=270807
rdar://problem/124396801

Reviewed by Aakash Jain.

Filter logs through filter-build-webkit and upload logs to S3.

* Tools/CISupport/build-webkit-org/steps.py:
(CompileWebKit):
(CompileWebKit.start):
* Tools/CISupport/build-webkit-org/steps_unittest.py:
(TestCompileWebKit.test_success_architecture):

Canonical link: https://commits.webkit.org/276297@main
  • Loading branch information
@briannafan
briannafan committed Mar 18, 2024
1 parent 787725b commit cf61984
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 28 deletions.
70 changes: 54 additions & 16 deletions Tools/CISupport/build-webkit-org/steps.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,7 @@
custom_suffix = '-uat'

BUILD_WEBKIT_HOSTNAMES = ['build.webkit.org', 'build']
TESTING_ENVIRONMENT_HOSTNAMES = ['build.webkit-uat.org', 'build-uat']
DEV_ENVIRONMENT_HOSTNAMES = ['build.webkit-dev.org', 'build-dev']
TESTING_ENVIRONMENT_HOSTNAMES = ['build.webkit-uat.org', 'build-uat', 'build.webkit-dev.org', 'build-dev']
COMMITS_INFO_URL = 'https://commits.webkit.org/'
RESULTS_WEBKIT_URL = 'https://results.webkit.org'
RESULTS_SERVER_API_KEY = 'RESULTS_SERVER_API_KEY'
Expand All @@ -73,6 +72,15 @@ def appendCrossTargetFlags(self, additionalArguments):
self.command.append(additionalArgument)
return

def customBuildFlag(self, platform, fullPlatform):
if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe', 'playstation', 'tvos', 'watchos'):
return []
if 'simulator' in fullPlatform:
platform = platform + '-simulator'
elif platform in ['ios', 'tvos', 'watchos']:
platform = platform + '-device'
return ['--' + platform]

def appendCustomBuildFlags(self, platform, fullPlatform):
if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe', 'playstation', 'tvos', 'watchos',):
return
Expand Down Expand Up @@ -346,7 +354,9 @@ def run(self):


class CompileWebKit(shell.Compile, CustomFlagsMixin):
command = ["perl", "Tools/Scripts/build-webkit", "--no-fatal-warnings", WithProperties("--%(configuration)s")]
build_command = ["perl", "Tools/Scripts/build-webkit", "--no-fatal-warnings"]
filter_command = ['perl', 'Tools/Scripts/filter-build-webkit', '-logfile', 'build-log.txt']
APPLE_PLATFORMS = ('mac', 'ios', 'tvos', 'watchos')
env = {'MFLAGS': ''}
name = "compile-webkit"
description = ["compiling"]
Expand All @@ -360,39 +370,49 @@ def start(self):
buildOnly = self.getProperty('buildOnly')
architecture = self.getProperty('architecture')
additionalArguments = self.getProperty('additionalArguments')
configuration = self.getProperty('configuration')

self.log_observer = ParseByLineLogObserver(self.parseOutputLine)
self.addLogObserver('stdio', self.log_observer)

build_command = self.build_command + [f'--{configuration}']

if additionalArguments:
self.setCommand(self.command + additionalArguments)
if platform in ('mac', 'ios', 'tvos', 'watchos'):
build_command += additionalArguments
if platform in self.APPLE_PLATFORMS:
# FIXME: Once WK_VALIDATE_DEPENDENCIES is set via xcconfigs, it can
# be removed here. We can't have build-webkit pass this by default
# without invalidating local builds made by Xcode, and we set it
# via xcconfigs until all building of Xcode-based webkit is done in
# workspaces (rdar://88135402).
if architecture:
self.setCommand(self.command + ['--architecture', architecture])
self.setCommand(self.command + ['WK_VALIDATE_DEPENDENCIES=YES'])
build_command += ['--architecture', f'"{architecture}"']
build_command += ['WK_VALIDATE_DEPENDENCIES=YES']
if buildOnly:
# For build-only bots, the expectation is that tests will be run on separate machines,
# so we need to package debug info as dSYMs. Only generating line tables makes
# this much faster than full debug info, and crash logs still have line numbers.
# Some projects (namely lldbWebKitTester) require full debug info, and may override this.
self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
self.setCommand(self.command + ['CLANG_DEBUG_INFORMATION_LEVEL=$(WK_OVERRIDE_DEBUG_INFORMATION_LEVEL:default=line-tables-only)'])
build_command += ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym']
build_command += ['CLANG_DEBUG_INFORMATION_LEVEL=\\$\\(WK_OVERRIDE_DEBUG_INFORMATION_LEVEL:default=line-tables-only\\)']
if platform == 'gtk':
prefix = os.path.join("/app", "webkit", "WebKitBuild", self.getProperty("configuration").title(), "install")
self.setCommand(self.command + [f'--prefix={prefix}'])
build_command += [f'--prefix={prefix}']

self.appendCustomBuildFlags(platform, self.getProperty('fullPlatform'))
build_command += self.customBuildFlag(platform, self.getProperty('fullPlatform'))

# filter-build-webkit is specifically designed for Xcode and doesn't work generally
if platform in self.APPLE_PLATFORMS:
full_command = f"{' '.join(build_command)} 2>&1 | {' '.join(self.filter_command)}"
self.setCommand(['/bin/sh', '-c', full_command])
else:
self.setCommand(build_command)

return shell.Compile.start(self)

def buildCommandKwargs(self, warnings):
kwargs = super(CompileWebKit, self).buildCommandKwargs(warnings)
kwargs['timeout'] = 60 * 30
kwargs['timeout'] = 60 * 60
return kwargs

def parseOutputLine(self, line):
Expand All @@ -413,6 +433,21 @@ def handleExcessiveLogging(self):
self.build.stopBuild(reason=MSG_FOR_EXCESSIVE_LOGS, results=FAILURE)
self.build.buildFinished([MSG_FOR_EXCESSIVE_LOGS], FAILURE)

def follow_up_steps(self):
if self.getProperty('platform') in self.APPLE_PLATFORMS and CURRENT_HOSTNAME in BUILD_WEBKIT_HOSTNAMES + TESTING_ENVIRONMENT_HOSTNAMES:
return [
GenerateS3URL(
f"{self.getProperty('fullPlatform')}-{self.getProperty('architecture')}-{self.getProperty('configuration')}-{self.name}",
extension='txt',
content_type='text/plain',
), UploadFileToS3(
'build-log.txt',
links={self.name: 'Full build log'},
content_type='text/plain',
)
]
return []

@defer.inlineCallbacks
def _addToLog(self, logName, message):
try:
Expand All @@ -423,8 +458,11 @@ def _addToLog(self, logName, message):

def evaluateCommand(self, cmd):
rc = super().evaluateCommand(cmd)
steps_to_add = self.follow_up_steps()
if rc in (SUCCESS, WARNINGS) and self.getProperty('user_provided_git_hash'):
self.build.addStepsAfterCurrentStep([ArchiveMinifiedBuiltProduct(), UploadMinifiedBuiltProduct(), TransferToS3(terminate_build=True)])
steps_to_add += [ArchiveMinifiedBuiltProduct(), UploadMinifiedBuiltProduct(), TransferToS3(terminate_build=True)]
# Using a single addStepsAfterCurrentStep because of https://github.com/buildbot/buildbot/issues/4874
self.build.addStepsAfterCurrentStep(steps_to_add)
return rc

def getResultSummary(self):
Expand All @@ -436,17 +474,17 @@ def getResultSummary(self):


class CompileLLINTCLoop(CompileWebKit):
command = ["perl", "Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")]
build_command = ["perl", "Tools/Scripts/build-jsc", "--cloop"]


class Compile32bitJSC(CompileWebKit):
name = 'compile-jsc-32bit'
command = ["perl", "Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")]
build_command = ["perl", "Tools/Scripts/build-jsc", "--32-bit"]


class CompileJSCOnly(CompileWebKit):
name = 'compile-jsc'
command = ["perl", "Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")]
build_command = ["perl", "Tools/Scripts/build-jsc"]


class InstallBuiltProduct(shell.ShellCommandNewStyle):
Expand Down
18 changes: 9 additions & 9 deletions Tools/CISupport/build-webkit-org/steps_unittest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,7 @@ def test_success(self):
self.expectRemoteCommands(
ExpectShell(
workdir='wkdir',
timeout=1800,
timeout=3600,
logEnviron=True,
command=['perl', 'Tools/Scripts/build-webkit', '--no-fatal-warnings', '--release'],
) + 0,
Expand All @@ -420,9 +420,9 @@ def test_success_architecture(self):
self.expectRemoteCommands(
ExpectShell(
workdir='wkdir',
timeout=1800,
timeout=3600,
logEnviron=True,
command=['perl', 'Tools/Scripts/build-webkit', '--no-fatal-warnings', '--release', '--architecture', 'x86_64 arm64', 'WK_VALIDATE_DEPENDENCIES=YES'],
command=['/bin/sh', '-c', 'perl Tools/Scripts/build-webkit --no-fatal-warnings --release --architecture "x86_64 arm64" WK_VALIDATE_DEPENDENCIES=YES 2>&1 | perl Tools/Scripts/filter-build-webkit -logfile build-log.txt'],
) + 0,
)
self.expectOutcome(result=SUCCESS, state_string='compiled')
Expand All @@ -435,7 +435,7 @@ def test_bigsur_timeout(self):
self.expectRemoteCommands(
ExpectShell(
workdir='wkdir',
timeout=1800,
timeout=3600,
logEnviron=True,
command=['perl', 'Tools/Scripts/build-webkit', '--no-fatal-warnings', '--release'],
) + 0,
Expand All @@ -451,7 +451,7 @@ def test_success_gtk(self):
self.expectRemoteCommands(
ExpectShell(
workdir='wkdir',
timeout=1800,
timeout=3600,
logEnviron=True,
command=['perl', 'Tools/Scripts/build-webkit', '--no-fatal-warnings', '--release', '--prefix=/app/webkit/WebKitBuild/Release/install', '--gtk'],
) + 0,
Expand All @@ -467,7 +467,7 @@ def test_success_wpe(self):
self.expectRemoteCommands(
ExpectShell(
workdir='wkdir',
timeout=1800,
timeout=3600,
logEnviron=True,
command=['perl', 'Tools/Scripts/build-webkit', '--no-fatal-warnings', '--release', '--wpe'],
) + 0,
Expand All @@ -482,7 +482,7 @@ def test_failure(self):
self.expectRemoteCommands(
ExpectShell(
workdir='wkdir',
timeout=1800,
timeout=3600,
logEnviron=True,
command=['perl', 'Tools/Scripts/build-webkit', '--no-fatal-warnings', '--debug'],
) + 2
Expand All @@ -507,7 +507,7 @@ def test_success(self):
self.expectRemoteCommands(
ExpectShell(
workdir='wkdir',
timeout=1800,
timeout=3600,
logEnviron=True,
command=['perl', 'Tools/Scripts/build-jsc', '--release'],
) + 0,
Expand All @@ -522,7 +522,7 @@ def test_failure(self):
self.expectRemoteCommands(
ExpectShell(
workdir='wkdir',
timeout=1800,
timeout=3600,
logEnviron=True,
command=['perl', 'Tools/Scripts/build-jsc', '--debug'],
) + 2
Expand Down
7 changes: 4 additions & 3 deletions Tools/CISupport/ews-build/steps.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3133,7 +3133,7 @@ class CompileWebKit(shell.Compile, AddToLogMixin, ShellMixin):
env = {'MFLAGS': ''}
warningPattern = '.*arning: .*'
haltOnFailure = False
build_command = ['perl', 'Tools/Scripts/build-webkit', WithProperties('--%(configuration)s')]
build_command = ['perl', 'Tools/Scripts/build-webkit']
filter_command = ['perl', 'Tools/Scripts/filter-build-webkit', '-logfile', 'build-log.txt']
VALID_ADDITIONAL_ARGUMENTS_LIST = [] # If additionalArguments is added to config.json for CompileWebKit step, it should be added here as well.
APPLE_PLATFORMS = ('mac', 'ios', 'tvos', 'watchos')
Expand All @@ -3150,13 +3150,14 @@ def start(self):
platform = self.getProperty('platform')
buildOnly = self.getProperty('buildOnly')
architecture = self.getProperty('architecture')
configuration = self.getProperty('configuration')

if platform in ['wincairo']:
self.addLogObserver('stdio', BuildLogLineObserver(self.errorReceived, searchString='error ', includeRelatedLines=False, thresholdExceedCallBack=self.handleExcessiveLogging))
else:
self.addLogObserver('stdio', BuildLogLineObserver(self.errorReceived, thresholdExceedCallBack=self.handleExcessiveLogging))

build_command = [part.getRenderingFor(self.build) if isinstance(part, WithProperties) else part for part in self.build_command]
build_command = self.build_command + [f'--{configuration}']

additionalArguments = self.getProperty('additionalArguments')
for additionalArgument in (additionalArguments or []):
Expand Down Expand Up @@ -3503,7 +3504,7 @@ def send_email_for_preexisting_build_failure(self):
class CompileJSC(CompileWebKit):
name = 'compile-jsc'
descriptionDone = ['Compiled JSC']
build_command = ['perl', 'Tools/Scripts/build-jsc', WithProperties('--%(configuration)s')]
build_command = ['perl', 'Tools/Scripts/build-jsc']

def start(self):
self.setProperty('group', 'jsc')
Expand Down

0 comments on commit cf61984

Please sign in to comment.