[build.webkit.org] Add support for uploading build logs to S3

https://bugs.webkit.org/show_bug.cgi?id=270128
rdar://123655339

Reviewed by Aakash Jain.

Adds UploadFileToS3 and GenerateS3URL steps to build.webkit.org.

* Tools/CISupport/build-webkit-org/steps.py:
(CheckOutSource.getResultSummary): Set 'revision' build property.
(UploadFileToS3):
(UploadFileToS3.__init__):
(UploadFileToS3.getLastBuildStepByName):
(UploadFileToS3.run):
(UploadFileToS3.doStepIf):
(UploadFileToS3.getResultSummary):
(GenerateS3URL):
(GenerateS3URL.__init__):
(GenerateS3URL.run):
(GenerateS3URL.hideStepIf):
(GenerateS3URL.doStepIf):
(GenerateS3URL.getResultSummary):
* Tools/CISupport/build-webkit-org/steps_unittest.py:

Canonical link: https://commits.webkit.org/275852@main

Tools/CISupport/build-webkit-org/steps.py

@@ -46,6 +46,8 @@
     custom_suffix = '-uat'
 
 BUILD_WEBKIT_HOSTNAMES = ['build.webkit.org', 'build']
+TESTING_ENVIRONMENT_HOSTNAMES = ['build.webkit-uat.org', 'build-uat']
+DEV_ENVIRONMENT_HOSTNAMES = ['build.webkit-dev.org', 'build-dev']
 COMMITS_INFO_URL = 'https://commits.webkit.org/'
 RESULTS_WEBKIT_URL = 'https://results.webkit.org'
 RESULTS_SERVER_API_KEY = 'RESULTS_SERVER_API_KEY'
@@ -55,6 +57,7 @@
 Interpolate = properties.Interpolate
 THRESHOLD_FOR_EXCESSIVE_LOGS = 1000000
 MSG_FOR_EXCESSIVE_LOGS = f'Stopped due to excessive logging, limit: {THRESHOLD_FOR_EXCESSIVE_LOGS}'
+HASH_LENGTH_TO_DISPLAY = 8
 
 DNS_NAME = CURRENT_HOSTNAME
 if DNS_NAME in BUILD_WEBKIT_HOSTNAMES:
@@ -188,6 +191,9 @@ def __init__(self, repourl='https://github.com/WebKit/WebKit.git', **kwargs):
                                              **kwargs)
 
     def getResultSummary(self):
+        revision = self.getProperty('got_revision')
+        self.setProperty('revision', revision[:HASH_LENGTH_TO_DISPLAY], 'CheckOutSource')
+
         if self.results == FAILURE:
             self.build.addStepsAfterCurrentStep([CleanUpGitIndexLock()])
 
@@ -1334,6 +1340,122 @@ def __init__(self, **kwargs):
         transfer.FileUpload.__init__(self, **kwargs)
 
 
+class UploadFileToS3(shell.ShellCommandNewStyle):
+    name = 'upload-file-to-s3'
+    descriptionDone = name
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, file, links=None, content_type=None, **kwargs):
+        super().__init__(timeout=31 * 60, logEnviron=False, **kwargs)
+        self.file = file
+        self.links = links or dict()
+        self.content_type = content_type
+
+    def getLastBuildStepByName(self, name):
+        for step in reversed(self.build.executedSteps):
+            if name in step.name:
+                return step
+        return None
+
+    @defer.inlineCallbacks
+    def run(self):
+        s3url = self.build.s3url
+        if not s3url:
+            rc = FAILURE
+            yield self._addToLog('stdio', f'Failed to get s3url: {s3url}')
+            return defer.returnValue(rc)
+
+        self.env = dict(UPLOAD_URL=s3url)
+
+        self.command = [
+            'python3', 'Tools/Scripts/upload-file-to-url',
+            '--filename', self.file,
+        ]
+        if self.content_type:
+            self.command += ['--content-type', self.content_type]
+
+        rc = yield super().run()
+
+        if rc in [SUCCESS, WARNINGS] and getattr(self.build, 's3_archives', None):
+            for step_name, message in self.links.items():
+                step = self.getLastBuildStepByName(step_name)
+                if not step:
+                    continue
+                step.addURL(message, self.build.s3_archives[-1])
+
+        return defer.returnValue(rc)
+
+    def doStepIf(self, step):
+        return CURRENT_HOSTNAME in BUILD_WEBKIT_HOSTNAMES + TESTING_ENVIRONMENT_HOSTNAMES
+
+    def getResultSummary(self):
+        if self.results == FAILURE:
+            return {'step': 'Failed to upload archive to S3. Please inform an admin.'}
+        if self.results == SKIPPED:
+            return {'step': 'Skipped upload to S3'}
+        if self.results in [SUCCESS, WARNINGS]:
+            return {'step': 'Uploaded archive to S3'}
+        return super().getResultSummary()
+
+
+class GenerateS3URL(master.MasterShellCommandNewStyle):
+    name = 'generate-s3-url'
+    descriptionDone = ['Generated S3 URL']
+    haltOnFailure = False
+    flunkOnFailure = False
+
+    def __init__(self, identifier, extension='zip', content_type=None, **kwargs):
+        self.identifier = identifier
+        self.extension = extension
+        kwargs['command'] = [
+            'python3', '../Shared/generate-s3-url',
+            '--revision', WithProperties('%(revision)s'),
+            '--identifier', self.identifier,
+        ]
+        if extension:
+            kwargs['command'] += ['--extension', extension]
+        if content_type:
+            kwargs['command'] += ['--content-type', content_type]
+        super().__init__(logEnviron=False, **kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.log_observer = logobserver.BufferLogObserver(wantStderr=True)
+        self.addLogObserver('stdio', self.log_observer)
+
+        rc = yield super().run()
+
+        self.build.s3url = ''
+        if not getattr(self.build, 's3_archives', None):
+            self.build.s3_archives = []
+
+        log_text = self.log_observer.getStdout() + self.log_observer.getStderr()
+        match = re.search(r'S3 URL: (?P<url>[^\s]+)', log_text)
+        # Sample log: S3 URL: https://s3-us-west-2.amazonaws.com/archives.webkit.org/ios-simulator-12-x86_64-release/123456.zip
+
+        build_url = f'{self.master.config.buildbotURL}#/builders/{self.build._builderid}/builds/{self.build.number}'
+        if match:
+            self.build.s3url = match.group('url')
+            print(f'build: {build_url}, url for GenerateS3URL: {self.build.s3url}')
+            self.build.s3_archives.append(S3URL + f"{S3_BUCKET}/{self.identifier}/{self.getProperty('revision')}.{self.extension}")
+            defer.returnValue(rc)
+        else:
+            print(f'build: {build_url}, logs for GenerateS3URL:\n{log_text}')
+            defer.returnValue(FAILURE)
+
+    def hideStepIf(self, results, step):
+        return results == SUCCESS
+
+    def doStepIf(self, step):
+        return CURRENT_HOSTNAME in BUILD_WEBKIT_HOSTNAMES + TESTING_ENVIRONMENT_HOSTNAMES
+
+    def getResultSummary(self):
+        if self.results == FAILURE:
+            return {'step': 'Failed to generate S3 URL'}
+        return super().getResultSummary()
+
+
 class TransferToS3(master.MasterShellCommandNewStyle):
     name = "transfer-to-s3"
     description = ["transferring to s3"]

Tools/CISupport/build-webkit-org/steps_unittest.py

@@ -40,7 +40,7 @@
 from .steps import *
 
 CURRENT_HOSTNAME = socket.gethostname().strip()
-
+FakeBuild._builderid = 1
 
 class ExpectMasterShellCommand(object):
     def __init__(self, command, workdir=None, env=None, usePTY=0):
@@ -1668,3 +1668,165 @@ def test_failure(self):
         )
         self.expectOutcome(result=FAILURE, state_string='webdriver-tests (failure)')
         return self.runStep()
+
+
+class current_hostname(object):
+    def __init__(self, hostname):
+        self.hostname = hostname
+        self.saved_hostname = None
+
+    def __enter__(self):
+        from . import steps
+        self.saved_hostname = steps.CURRENT_HOSTNAME
+        steps.CURRENT_HOSTNAME = self.hostname
+
+    def __exit__(self, type, value, tb):
+        from . import steps
+        steps.CURRENT_HOSTNAME = self.saved_hostname
+
+
+class TestGenerateS3URL(BuildStepMixinAdditions, unittest.TestCase):
+    def setUp(self):
+        self.longMessage = True
+        return self.setUpBuildStep()
+
+    def tearDown(self):
+        return self.tearDownBuildStep()
+
+    def configureStep(self, identifier='mac-highsierra-x86_64-release', extension='zip', content_type=None):
+        self.setupStep(GenerateS3URL(identifier, extension=extension, content_type=content_type))
+        self.setProperty('revision', '1234')
+
+    def disabled_test_success(self):
+        # TODO: Figure out how to pass logs to unit-test for MasterShellCommand steps
+        self.configureStep()
+        self.expectLocalCommands(
+            ExpectMasterShellCommand(command=['python3',
+                                              '../Shared/generate-s3-url',
+                                              '--revision', '1234',
+                                              '--identifier', 'mac-highsierra-x86_64-release',
+                                              '--extension', 'zip',
+                                              ])
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, state_string='Generated S3 URL')
+        with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]):
+            return self.runStep()
+
+    def test_failure(self):
+        self.configureStep('ios-simulator-16-x86_64-debug')
+        self.expectLocalCommands(
+            ExpectMasterShellCommand(command=['python3',
+                                              '../Shared/generate-s3-url',
+                                              '--revision', '1234',
+                                              '--identifier', 'ios-simulator-16-x86_64-debug',
+                                              '--extension', 'zip',
+                                              ])
+            + 2,
+        )
+        self.expectOutcome(result=FAILURE, state_string='Failed to generate S3 URL')
+
+        try:
+            with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]), open(os.devnull, 'w') as null:
+                sys.stdout = null
+                return self.runStep()
+        finally:
+            sys.stdout = sys.__stdout__
+
+    def test_failure_with_extension(self):
+        self.configureStep('macos-arm64-release-compile-webkit', extension='txt', content_type='text/plain')
+        self.expectLocalCommands(
+            ExpectMasterShellCommand(command=['python3',
+                                              '../Shared/generate-s3-url',
+                                              '--revision', '1234',
+                                              '--identifier', 'macos-arm64-release-compile-webkit',
+                                              '--extension', 'txt',
+                                              '--content-type', 'text/plain',
+                                              ])
+            + 2,
+        )
+        self.expectOutcome(result=FAILURE, state_string='Failed to generate S3 URL')
+
+        try:
+            with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]), open(os.devnull, 'w') as null:
+                sys.stdout = null
+                return self.runStep()
+        finally:
+            sys.stdout = sys.__stdout__
+
+    def test_skipped(self):
+        self.configureStep()
+        self.expectOutcome(result=SKIPPED, state_string='Generated S3 URL (skipped)')
+        with current_hostname('something-other-than-steps.BUILD_WEBKIT_HOSTNAMES'):
+            return self.runStep()
+
+
+class TestUploadFileToS3(BuildStepMixinAdditions, unittest.TestCase):
+    def setUp(self):
+        self.longMessage = True
+        return self.setUpBuildStep()
+
+    def tearDown(self):
+        return self.tearDownBuildStep()
+
+    def configureStep(self, file='WebKitBuild/release.zip', content_type=None):
+        self.setupStep(UploadFileToS3(file, content_type=content_type))
+        self.build.s3url = 'https://test-s3-url'
+
+    def test_success(self):
+        self.configureStep()
+        self.assertEqual(UploadFileToS3.haltOnFailure, True)
+        self.assertEqual(UploadFileToS3.flunkOnFailure, True)
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        env=dict(UPLOAD_URL='https://test-s3-url'),
+                        logEnviron=False,
+                        command=['python3', 'Tools/Scripts/upload-file-to-url', '--filename', 'WebKitBuild/release.zip'],
+                        timeout=1860,
+                        )
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, state_string='Uploaded archive to S3')
+        with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]):
+            return self.runStep()
+
+    def test_success_content_type(self):
+        self.configureStep(file='build-log.txt', content_type='text/plain')
+        self.assertEqual(UploadFileToS3.haltOnFailure, True)
+        self.assertEqual(UploadFileToS3.flunkOnFailure, True)
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        env=dict(UPLOAD_URL='https://test-s3-url'),
+                        logEnviron=False,
+                        command=['python3', 'Tools/Scripts/upload-file-to-url', '--filename', 'build-log.txt', '--content-type', 'text/plain'],
+                        timeout=1860,
+                        )
+            + 0,
+        )
+        self.expectOutcome(result=SUCCESS, state_string='Uploaded archive to S3')
+        with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]):
+            return self.runStep()
+
+    def test_failure(self):
+        self.configureStep()
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        env=dict(UPLOAD_URL='https://test-s3-url'),
+                        logEnviron=False,
+                        command=['python3', 'Tools/Scripts/upload-file-to-url', '--filename', 'WebKitBuild/release.zip'],
+                        timeout=1860,
+                        )
+            + ExpectShell.log('stdio', stdout='''Uploading WebKitBuild/release.zip
+response: <Response [403]>, 403, Forbidden
+exit 1''')
+            + 2,
+        )
+        self.expectOutcome(result=FAILURE, state_string='Failed to upload archive to S3. Please inform an admin.')
+        with current_hostname(BUILD_WEBKIT_HOSTNAMES[0]):
+            return self.runStep()
+
+    def test_skipped(self):
+        self.configureStep()
+        self.expectOutcome(result=SKIPPED, state_string='Skipped upload to S3')
+        with current_hostname('something-other-than-steps.BUILD_WEBKIT_HOSTNAMES'):
+            return self.runStep()