From 3eeec64b18a5f280bb412571611db77c3cd07037 Mon Sep 17 00:00:00 2001 From: Brianna Fan Date: Fri, 8 Mar 2024 12:14:39 -0800 Subject: [PATCH] [build.webkit.org] Add support for uploading build logs to S3 https://bugs.webkit.org/show_bug.cgi?id=270128 rdar://123655339 Reviewed by Aakash Jain. Adds UploadFileToS3 and GenerateS3URL steps to build.webkit.org. * Tools/CISupport/build-webkit-org/steps.py: (CheckOutSource.getResultSummary): Set 'revision' build property. (UploadFileToS3): (UploadFileToS3.__init__): (UploadFileToS3.getLastBuildStepByName): (UploadFileToS3.run): (UploadFileToS3.doStepIf): (UploadFileToS3.getResultSummary): (GenerateS3URL): (GenerateS3URL.__init__): (GenerateS3URL.run): (GenerateS3URL.hideStepIf): (GenerateS3URL.doStepIf): (GenerateS3URL.getResultSummary): * Tools/CISupport/build-webkit-org/steps_unittest.py: Canonical link: https://commits.webkit.org/275852@main --- Tools/CISupport/build-webkit-org/steps.py | 122 +++++++++++++ .../build-webkit-org/steps_unittest.py | 164 +++++++++++++++++- 2 files changed, 285 insertions(+), 1 deletion(-) diff --git a/Tools/CISupport/build-webkit-org/steps.py b/Tools/CISupport/build-webkit-org/steps.py index 733e19d393b4..fce8705d1a9e 100644 --- a/Tools/CISupport/build-webkit-org/steps.py +++ b/Tools/CISupport/build-webkit-org/steps.py @@ -46,6 +46,8 @@ custom_suffix = '-uat' BUILD_WEBKIT_HOSTNAMES = ['build.webkit.org', 'build'] +TESTING_ENVIRONMENT_HOSTNAMES = ['build.webkit-uat.org', 'build-uat'] +DEV_ENVIRONMENT_HOSTNAMES = ['build.webkit-dev.org', 'build-dev'] COMMITS_INFO_URL = 'https://commits.webkit.org/' RESULTS_WEBKIT_URL = 'https://results.webkit.org' RESULTS_SERVER_API_KEY = 'RESULTS_SERVER_API_KEY' @@ -55,6 +57,7 @@ Interpolate = properties.Interpolate THRESHOLD_FOR_EXCESSIVE_LOGS = 1000000 MSG_FOR_EXCESSIVE_LOGS = f'Stopped due to excessive logging, limit: {THRESHOLD_FOR_EXCESSIVE_LOGS}' +HASH_LENGTH_TO_DISPLAY = 8 DNS_NAME = CURRENT_HOSTNAME if DNS_NAME in BUILD_WEBKIT_HOSTNAMES: @@ -188,6 +191,9 @@ def __init__(self, repourl='https://github.com/WebKit/WebKit.git', **kwargs): **kwargs) def getResultSummary(self): + revision = self.getProperty('got_revision') + self.setProperty('revision', revision[:HASH_LENGTH_TO_DISPLAY], 'CheckOutSource') + if self.results == FAILURE: self.build.addStepsAfterCurrentStep([CleanUpGitIndexLock()]) @@ -1334,6 +1340,122 @@ def __init__(self, **kwargs): transfer.FileUpload.__init__(self, **kwargs) +class UploadFileToS3(shell.ShellCommandNewStyle): + name = 'upload-file-to-s3' + descriptionDone = name + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, file, links=None, content_type=None, **kwargs): + super().__init__(timeout=31 * 60, logEnviron=False, **kwargs) + self.file = file + self.links = links or dict() + self.content_type = content_type + + def getLastBuildStepByName(self, name): + for step in reversed(self.build.executedSteps): + if name in step.name: + return step + return None + + @defer.inlineCallbacks + def run(self): + s3url = self.build.s3url + if not s3url: + rc = FAILURE + yield self._addToLog('stdio', f'Failed to get s3url: {s3url}') + return defer.returnValue(rc) + + self.env = dict(UPLOAD_URL=s3url) + + self.command = [ + 'python3', 'Tools/Scripts/upload-file-to-url', + '--filename', self.file, + ] + if self.content_type: + self.command += ['--content-type', self.content_type] + + rc = yield super().run() + + if rc in [SUCCESS, WARNINGS] and getattr(self.build, 's3_archives', None): + for step_name, message in self.links.items(): + step = self.getLastBuildStepByName(step_name) + if not step: + continue + step.addURL(message, self.build.s3_archives[-1]) + + return defer.returnValue(rc) + + def doStepIf(self, step): + return CURRENT_HOSTNAME in BUILD_WEBKIT_HOSTNAMES + TESTING_ENVIRONMENT_HOSTNAMES + + def getResultSummary(self): + if self.results == FAILURE: + return {'step': 'Failed to upload archive to S3. Please inform an admin.'} + if self.results == SKIPPED: + return {'step': 'Skipped upload to S3'} + if self.results in [SUCCESS, WARNINGS]: + return {'step': 'Uploaded archive to S3'} + return super().getResultSummary() + + +class GenerateS3URL(master.MasterShellCommandNewStyle): + name = 'generate-s3-url' + descriptionDone = ['Generated S3 URL'] + haltOnFailure = False + flunkOnFailure = False + + def __init__(self, identifier, extension='zip', content_type=None, **kwargs): + self.identifier = identifier + self.extension = extension + kwargs['command'] = [ + 'python3', '../Shared/generate-s3-url', + '--revision', WithProperties('%(revision)s'), + '--identifier', self.identifier, + ] + if extension: + kwargs['command'] += ['--extension', extension] + if content_type: + kwargs['command'] += ['--content-type', content_type] + super().__init__(logEnviron=False, **kwargs) + + @defer.inlineCallbacks + def run(self): + self.log_observer = logobserver.BufferLogObserver(wantStderr=True) + self.addLogObserver('stdio', self.log_observer) + + rc = yield super().run() + + self.build.s3url = '' + if not getattr(self.build, 's3_archives', None): + self.build.s3_archives = [] + + log_text = self.log_observer.getStdout() + self.log_observer.getStderr() + match = re.search(r'S3 URL: (?P[^\s]+)', log_text) + # Sample log: S3 URL: https://s3-us-west-2.amazonaws.com/archives.webkit.org/ios-simulator-12-x86_64-release/123456.zip + + build_url = f'{self.master.config.buildbotURL}#/builders/{self.build._builderid}/builds/{self.build.number}' + if match: + self.build.s3url = match.group('url') + print(f'build: {build_url}, url for GenerateS3URL: {self.build.s3url}') + self.build.s3_archives.append(S3URL + f"{S3_BUCKET}/{self.identifier}/{self.getProperty('revision')}.{self.extension}") + defer.returnValue(rc) + else: + print(f'build: {build_url}, logs for GenerateS3URL:\n{log_text}') + defer.returnValue(FAILURE) + + def hideStepIf(self, results, step): + return results == SUCCESS + + def doStepIf(self, step): + return CURRENT_HOSTNAME in BUILD_WEBKIT_HOSTNAMES + TESTING_ENVIRONMENT_HOSTNAMES + + def getResultSummary(self): + if self.results == FAILURE: + return {'step': 'Failed to generate S3 URL'} + return super().getResultSummary() + + class TransferToS3(master.MasterShellCommandNewStyle): name = "transfer-to-s3" description = ["transferring to s3"] diff --git a/Tools/CISupport/build-webkit-org/steps_unittest.py b/Tools/CISupport/build-webkit-org/steps_unittest.py index 347e00caa1f9..e50d1c5b8dbd 100644 --- a/Tools/CISupport/build-webkit-org/steps_unittest.py +++ b/Tools/CISupport/build-webkit-org/steps_unittest.py @@ -40,7 +40,7 @@ from .steps import * CURRENT_HOSTNAME = socket.gethostname().strip() - +FakeBuild._builderid = 1 class ExpectMasterShellCommand(object): def __init__(self, command, workdir=None, env=None, usePTY=0): @@ -1668,3 +1668,165 @@ def test_failure(self): ) self.expectOutcome(result=FAILURE, state_string='webdriver-tests (failure)') return self.runStep() + + +class current_hostname(object): + def __init__(self, hostname): + self.hostname = hostname + self.saved_hostname = None + + def __enter__(self): + from . import steps + self.saved_hostname = steps.CURRENT_HOSTNAME + steps.CURRENT_HOSTNAME = self.hostname + + def __exit__(self, type, value, tb): + from . import steps + steps.CURRENT_HOSTNAME = self.saved_hostname + + +class TestGenerateS3URL(BuildStepMixinAdditions, unittest.TestCase): + def setUp(self): + self.longMessage = True + return self.setUpBuildStep() + + def tearDown(self): + return self.tearDownBuildStep() + + def configureStep(self, identifier='mac-highsierra-x86_64-release', extension='zip', content_type=None): + self.setupStep(GenerateS3URL(identifier, extension=extension, content_type=content_type)) + self.setProperty('revision', '1234') + + def disabled_test_success(self): + # TODO: Figure out how to pass logs to unit-test for MasterShellCommand steps + self.configureStep() + self.expectLocalCommands( + ExpectMasterShellCommand(command=['python3', + '../Shared/generate-s3-url', + '--revision', '1234', + '--identifier', 'mac-highsierra-x86_64-release', + '--extension', 'zip', + ]) + + 0, + ) + self.expectOutcome(result=SUCCESS, state_string='Generated S3 URL') + with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]): + return self.runStep() + + def test_failure(self): + self.configureStep('ios-simulator-16-x86_64-debug') + self.expectLocalCommands( + ExpectMasterShellCommand(command=['python3', + '../Shared/generate-s3-url', + '--revision', '1234', + '--identifier', 'ios-simulator-16-x86_64-debug', + '--extension', 'zip', + ]) + + 2, + ) + self.expectOutcome(result=FAILURE, state_string='Failed to generate S3 URL') + + try: + with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]), open(os.devnull, 'w') as null: + sys.stdout = null + return self.runStep() + finally: + sys.stdout = sys.__stdout__ + + def test_failure_with_extension(self): + self.configureStep('macos-arm64-release-compile-webkit', extension='txt', content_type='text/plain') + self.expectLocalCommands( + ExpectMasterShellCommand(command=['python3', + '../Shared/generate-s3-url', + '--revision', '1234', + '--identifier', 'macos-arm64-release-compile-webkit', + '--extension', 'txt', + '--content-type', 'text/plain', + ]) + + 2, + ) + self.expectOutcome(result=FAILURE, state_string='Failed to generate S3 URL') + + try: + with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]), open(os.devnull, 'w') as null: + sys.stdout = null + return self.runStep() + finally: + sys.stdout = sys.__stdout__ + + def test_skipped(self): + self.configureStep() + self.expectOutcome(result=SKIPPED, state_string='Generated S3 URL (skipped)') + with current_hostname('something-other-than-steps.BUILD_WEBKIT_HOSTNAMES'): + return self.runStep() + + +class TestUploadFileToS3(BuildStepMixinAdditions, unittest.TestCase): + def setUp(self): + self.longMessage = True + return self.setUpBuildStep() + + def tearDown(self): + return self.tearDownBuildStep() + + def configureStep(self, file='WebKitBuild/release.zip', content_type=None): + self.setupStep(UploadFileToS3(file, content_type=content_type)) + self.build.s3url = 'https://test-s3-url' + + def test_success(self): + self.configureStep() + self.assertEqual(UploadFileToS3.haltOnFailure, True) + self.assertEqual(UploadFileToS3.flunkOnFailure, True) + self.expectRemoteCommands( + ExpectShell(workdir='wkdir', + env=dict(UPLOAD_URL='https://test-s3-url'), + logEnviron=False, + command=['python3', 'Tools/Scripts/upload-file-to-url', '--filename', 'WebKitBuild/release.zip'], + timeout=1860, + ) + + 0, + ) + self.expectOutcome(result=SUCCESS, state_string='Uploaded archive to S3') + with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]): + return self.runStep() + + def test_success_content_type(self): + self.configureStep(file='build-log.txt', content_type='text/plain') + self.assertEqual(UploadFileToS3.haltOnFailure, True) + self.assertEqual(UploadFileToS3.flunkOnFailure, True) + self.expectRemoteCommands( + ExpectShell(workdir='wkdir', + env=dict(UPLOAD_URL='https://test-s3-url'), + logEnviron=False, + command=['python3', 'Tools/Scripts/upload-file-to-url', '--filename', 'build-log.txt', '--content-type', 'text/plain'], + timeout=1860, + ) + + 0, + ) + self.expectOutcome(result=SUCCESS, state_string='Uploaded archive to S3') + with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]): + return self.runStep() + + def test_failure(self): + self.configureStep() + self.expectRemoteCommands( + ExpectShell(workdir='wkdir', + env=dict(UPLOAD_URL='https://test-s3-url'), + logEnviron=False, + command=['python3', 'Tools/Scripts/upload-file-to-url', '--filename', 'WebKitBuild/release.zip'], + timeout=1860, + ) + + ExpectShell.log('stdio', stdout='''Uploading WebKitBuild/release.zip +response: , 403, Forbidden +exit 1''') + + 2, + ) + self.expectOutcome(result=FAILURE, state_string='Failed to upload archive to S3. Please inform an admin.') + with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]): + return self.runStep() + + def test_skipped(self): + self.configureStep() + self.expectOutcome(result=SKIPPED, state_string='Skipped upload to S3') + with current_hostname('something-other-than-steps.BUILD_WEBKIT_HOSTNAMES'): + return self.runStep()