Make Commit Queue robust by adding few git commands to clean up repos…

…itory

https://bugs.webkit.org/show_bug.cgi?id=224444

Reviewed by Jonathan Bedard.

* CISupport/ews-build/steps.py:
(CleanGitRepo): Build step to run few commands to cleanup git repository.
(CleanGitRepo.run):
(CleanGitRepo.getResultSummary): Custom failure message.
* CISupport/ews-build/steps_unittest.py: Added unit-tests.
* CISupport/ews-build/factories.py:
(CommitQueueFactory.__init__):
* CISupport/ews-build/factories_unittest.py:
(TestCommitQueueFactory.test_commit_queue_factory): Updated unit-test.


Canonical link: https://commits.webkit.org/236418@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@275853 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/CISupport/ews-build/factories.py

@@ -25,7 +25,7 @@
 from buildbot.steps import trigger
 
 from steps import (ApplyPatch, ApplyWatchList, CheckOutSource, CheckOutSpecificRevision, CheckPatchRelevance,
-                   CheckPatchStatusOnEWSQueues, CheckStyle, CompileJSC, CompileWebKit, ConfigureBuild, CreateLocalGITCommit,
+                   CheckPatchStatusOnEWSQueues, CheckStyle, CleanGitRepo, CompileJSC, CompileWebKit, ConfigureBuild, CreateLocalGITCommit,
                    DownloadBuiltProduct, ExtractBuiltProduct, FetchBranches, FindModifiedChangeLogs, FindModifiedLayoutTests,
                    InstallGtkDependencies, InstallWpeDependencies, KillOldProcesses, PrintConfiguration, PushCommitToWebKitRepo,
                    RunAPITests, RunBindingsTests, RunBuildWebKitOrgUnitTests, RunBuildbotCheckConfigForBuildWebKit, RunBuildbotCheckConfigForEWS,
@@ -281,6 +281,7 @@ def __init__(self, platform, configuration=None, architectures=None, additionalA
         self.addStep(ValidatePatch(verifycqplus=True))
         self.addStep(ValidateCommiterAndReviewer())
         self.addStep(PrintConfiguration())
+        self.addStep(CleanGitRepo())
         self.addStep(CheckOutSource())
         self.addStep(FetchBranches())
         self.addStep(ShowIdentifier())

Tools/CISupport/ews-build/factories_unittest.py

@@ -411,6 +411,7 @@ def test_commit_queue_factory(self):
             _BuildStepFactory(steps.ValidatePatch, verifycqplus=True),
             _BuildStepFactory(steps.ValidateCommiterAndReviewer),
             _BuildStepFactory(steps.PrintConfiguration),
+            _BuildStepFactory(steps.CleanGitRepo),
             _BuildStepFactory(steps.CheckOutSource),
             _BuildStepFactory(steps.FetchBranches),
             _BuildStepFactory(steps.ShowIdentifier),

Tools/CISupport/ews-build/steps.py

@@ -3128,6 +3128,31 @@ def getResultSummary(self):
         return {'step': configuration}
 
 
+class CleanGitRepo(steps.ShellSequence):
+    name = 'clean-up-git-repo'
+    haltOnFailure = False
+    flunkOnFailure = False
+    logEnviron = False
+    # This somewhat quirky sequence of steps seems to clear up all the broken
+    # git situations we've gotten ourself into in the past.
+    command_list = [['git', 'clean', '-f'],  # Remove any left-over layout test results, added files, etc.
+                    ['git', 'fetch', 'origin'],  # Avoid updating the working copy to a stale revision.
+                    ['git', 'checkout', 'origin/main', '-f'],
+                    ['git', 'branch', '-D', 'main'],
+                    ['git', 'checkout', 'origin/main', '-b', 'main']]
+
+    def run(self):
+        self.commands = []
+        for command in self.command_list:
+            self.commands.append(util.ShellArg(command=command, logname='stdio'))
+        return super(CleanGitRepo, self).run()
+
+    def getResultSummary(self):
+        if self.results != SUCCESS:
+            return {'step': 'Encountered some issues during cleanup'}
+        return {'step': 'Cleaned up git repository'}
+
+
 class ApplyWatchList(shell.ShellCommand):
     name = 'apply-watch-list'
     description = ['applying watchilist']

Tools/CISupport/ews-build/steps_unittest.py

@@ -43,8 +43,8 @@
 from steps import (AnalyzeAPITestsResults, AnalyzeCompileWebKitResults, AnalyzeJSCTestsResults,
                    AnalyzeLayoutTestsResults, ApplyPatch, ApplyWatchList, ArchiveBuiltProduct, ArchiveTestResults,
                    CheckOutSource, CheckOutSpecificRevision, CheckPatchRelevance, CheckPatchStatusOnEWSQueues, CheckStyle,
-                   CleanBuild, CleanUpGitIndexLock, CleanWorkingDirectory, CompileJSC, CompileJSCWithoutPatch, CompileWebKit,
-                   CompileWebKitWithoutPatch, ConfigureBuild, CreateLocalGITCommit,
+                   CleanBuild, CleanUpGitIndexLock, CleanGitRepo, CleanWorkingDirectory, CompileJSC, CompileJSCWithoutPatch,
+                   CompileWebKit, CompileWebKitWithoutPatch, ConfigureBuild, CreateLocalGITCommit,
                    DownloadBuiltProduct, DownloadBuiltProductFromMaster, EWS_BUILD_HOSTNAME, ExtractBuiltProduct, ExtractTestResults,
                    FetchBranches, FindModifiedChangeLogs, FindModifiedLayoutTests, GitResetHard, InstallGtkDependencies, InstallWpeDependencies,
                    KillOldProcesses, PrintConfiguration, PushCommitToWebKitRepo, ReRunAPITests, ReRunJavaScriptCoreTests, ReRunWebKitPerlTests,
@@ -3889,6 +3889,53 @@ def test_failure(self):
         return self.runStep()
 
 
+class TestCleanGitRepo(BuildStepMixinAdditions, unittest.TestCase):
+    def setUp(self):
+        self.longMessage = True
+        return self.setUpBuildStep()
+
+    def tearDown(self):
+        return self.tearDownBuildStep()
+
+    def test_success(self):
+        self.setupStep(CleanGitRepo())
+        self.setProperty('buildername', 'Commit-Queue')
+
+        self.expectRemoteCommands(
+            ExpectShell(command=['git', 'clean', '-f'], workdir='wkdir', timeout=1200, logEnviron=False) + 0
+            + ExpectShell.log('stdio', stdout=''),
+            ExpectShell(command=['git', 'fetch', 'origin'], workdir='wkdir', timeout=1200, logEnviron=False) + 0
+            + ExpectShell.log('stdio', stdout=''),
+            ExpectShell(command=['git', 'checkout', 'origin/main', '-f'], workdir='wkdir', timeout=1200, logEnviron=False) + 0
+            + ExpectShell.log('stdio', stdout='You are in detached HEAD state.'),
+            ExpectShell(command=['git', 'branch', '-D', 'main'], workdir='wkdir', timeout=1200, logEnviron=False) + 0
+            + ExpectShell.log('stdio', stdout='Deleted branch main (was 57015967fef9).'),
+            ExpectShell(command=['git', 'checkout', 'origin/main', '-b', 'main'], workdir='wkdir', timeout=1200, logEnviron=False) + 0
+            + ExpectShell.log('stdio', stdout="Switched to a new branch 'main'"),
+        )
+        self.expectOutcome(result=SUCCESS, state_string='Cleaned up git repository')
+        return self.runStep()
+
+    def test_failure(self):
+        self.setupStep(CleanGitRepo())
+        self.setProperty('buildername', 'Commit-Queue')
+
+        self.expectRemoteCommands(
+            ExpectShell(command=['git', 'clean', '-f'], workdir='wkdir', timeout=1200, logEnviron=False) + 0
+            + ExpectShell.log('stdio', stdout=''),
+            ExpectShell(command=['git', 'fetch', 'origin'], workdir='wkdir', timeout=1200, logEnviron=False) + 128
+            + ExpectShell.log('stdio', stdout='fatal: unable to access https://github.com/WebKit/WebKit.git/: Could not resolve host: github.com'),
+            ExpectShell(command=['git', 'checkout', 'origin/main', '-f'], workdir='wkdir', timeout=1200, logEnviron=False) + 0
+            + ExpectShell.log('stdio', stdout='You are in detached HEAD state.'),
+            ExpectShell(command=['git', 'branch', '-D', 'main'], workdir='wkdir', timeout=1200, logEnviron=False) + 0
+            + ExpectShell.log('stdio', stdout='Deleted branch main (was 57015967fef9).'),
+            ExpectShell(command=['git', 'checkout', 'origin/main', '-b', 'main'], workdir='wkdir', timeout=1200, logEnviron=False) + 0
+            + ExpectShell.log('stdio', stdout="Switched to a new branch 'main'"),
+        )
+        self.expectOutcome(result=FAILURE, state_string='Encountered some issues during cleanup')
+        return self.runStep()
+
+
 class TestFindModifiedChangeLogs(BuildStepMixinAdditions, unittest.TestCase):
     def setUp(self):
         self.longMessage = True

Tools/ChangeLog

@@ -1,3 +1,20 @@
+2021-04-12  Aakash Jain  <aakash_jain@apple.com>
+
+        Make Commit Queue robust by adding few git commands to clean up repository
+        https://bugs.webkit.org/show_bug.cgi?id=224444
+
+        Reviewed by Jonathan Bedard.
+
+        * CISupport/ews-build/steps.py:
+        (CleanGitRepo): Build step to run few commands to cleanup git repository.
+        (CleanGitRepo.run):
+        (CleanGitRepo.getResultSummary): Custom failure message.
+        * CISupport/ews-build/steps_unittest.py: Added unit-tests.
+        * CISupport/ews-build/factories.py:
+        (CommitQueueFactory.__init__):
+        * CISupport/ews-build/factories_unittest.py:
+        (TestCommitQueueFactory.test_commit_queue_factory): Updated unit-test.
+
 2021-04-11  Commit Queue  <commit-queue@webkit.org>
 
         Unreviewed, reverting r275801.