[TEAMMATES#12059] Separate stable and unstable E2E tests (TEAMMATES#1…

…2515) * fix: separate stable and unstable E2E tests * fix: separate unstable tests into a separate job * fix: revert name change * fix: reduce duplication for unstable e2e tests * fix: lint errors --------- Co-authored-by: Wilson Kurniawan <wkurniawan.92@gmail.com>
jasonqiu212 · Jul 27, 2023 · 5e58bfa · 5e58bfa
1 parent db5805e
commit 5e58bfa
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 9 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -18,6 +18,7 @@ jobs:
       fail-fast: false #ensure both tests run even if one fails
       matrix:
         browser: [firefox]
+        tests: [stable, unstable]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
@@ -52,4 +53,5 @@ jobs:
           ./gradlew serverRun &
           ./wait-for-server.sh
       - name: Start Tests
-        run: xvfb-run --server-args="-screen 0 1024x768x24" ./gradlew e2eTests
+        run: xvfb-run --server-args="-screen 0 1024x768x24" ./gradlew -P${{ matrix.tests }} e2eTests
+
diff --git a/build.gradle b/build.gradle
@@ -534,18 +534,22 @@ task componentTests(type: Test) {
 }
 
 task e2eTests {
-    description "Runs the full E2E test suite and retries failed test up to ${numOfTestRetries} times."
+    description "Runs the E2E test suite and retries failed test up to ${numOfTestRetries} times."
     group "Test"
 }
 
 (1..numOfTestRetries + 1).each { id ->
     def isFirstTry = id == 1
     def isLastRetry = id == numOfTestRetries + 1
+    def runUnstableTests = project.hasProperty('unstable')
+    def outputFileName = runUnstableTests ? "e2e-unstable-test-try-" : "e2e-test-try-"
 
     task "e2eTestTry${id}"(type: Test) {
         useTestNG()
-        options.suites isFirstTry ? "src/e2e/resources/testng-e2e.xml" : file("${buildDir}/reports/e2e-test-try-${id - 1}/testng-failed.xml")
-        options.outputDirectory = file("${buildDir}/reports/e2e-test-try-${id}")
+        options.suites isFirstTry
+                ? (runUnstableTests ? "src/e2e/resources/testng-unstable-e2e.xml" : "src/e2e/resources/testng-e2e.xml")
+                : file("${buildDir}/reports/${outputFileName}${id - 1}/testng-failed.xml")
+        options.outputDirectory = file("${buildDir}/reports/${outputFileName}${id}")
         options.useDefaultListeners = true
         ignoreFailures = !isLastRetry
         maxHeapSize = "1g"
@@ -560,7 +564,7 @@ task e2eTests {
             afterSuite checkTestNgFailureClosure
         }
         onlyIf {
-            isFirstTry || file("${buildDir}/reports/e2e-test-try-${id - 1}/testng-failed.xml").exists()
+            isFirstTry || file("${buildDir}/reports/${outputFileName}${id - 1}/testng-failed.xml").exists()
         }
     }
     e2eTests.dependsOn "e2eTestTry${id}"

diff --git a/src/e2e/resources/testng-e2e.xml b/src/e2e/resources/testng-e2e.xml
@@ -24,10 +24,10 @@
             <class name="teammates.e2e.cases.FeedbackRankOptionQuestionE2ETest" />
             <class name="teammates.e2e.cases.FeedbackResultsPageE2ETest" />
             <class name="teammates.e2e.cases.FeedbackRubricQuestionE2ETest" />
-            <class name="teammates.e2e.cases.FeedbackSubmitPageE2ETest" />
+<!--            <class name="teammates.e2e.cases.FeedbackSubmitPageE2ETest" />-->
             <class name="teammates.e2e.cases.FeedbackTextQuestionE2ETest" />
             <class name="teammates.e2e.cases.InstructorCourseDetailsPageE2ETest" />
-            <class name="teammates.e2e.cases.InstructorStudentActivityLogsPageE2ETest" />
+<!--            <class name="teammates.e2e.cases.InstructorStudentActivityLogsPageE2ETest" />-->
             <class name="teammates.e2e.cases.InstructorCourseEditPageE2ETest" />
             <class name="teammates.e2e.cases.InstructorCourseEnrollPageE2ETest" />
             <class name="teammates.e2e.cases.InstructorCourseJoinConfirmationPageE2ETest" />
@@ -37,8 +37,8 @@
             <class name="teammates.e2e.cases.InstructorFeedbackEditPageE2ETest" />
             <class name="teammates.e2e.cases.InstructorFeedbackReportPageE2ETest" />
             <class name="teammates.e2e.cases.InstructorFeedbackSessionsPageE2ETest" />
-            <class name="teammates.e2e.cases.InstructorHomePageE2ETest" />
-            <class name="teammates.e2e.cases.InstructorSearchPageE2ETest" />
+<!--            <class name="teammates.e2e.cases.InstructorHomePageE2ETest" />-->
+<!--            <class name="teammates.e2e.cases.InstructorSearchPageE2ETest" />-->
             <class name="teammates.e2e.cases.InstructorSessionIndividualExtensionPageE2ETest" />
             <class name="teammates.e2e.cases.InstructorStudentListPageE2ETest" />
             <class name="teammates.e2e.cases.InstructorStudentRecordsPageE2ETest" />

diff --git a/src/e2e/resources/testng-unstable-e2e.xml b/src/e2e/resources/testng-unstable-e2e.xml
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
+<suite name="e2e-unstable-tests" parallel="tests" thread-count="2">
+    <test name="e2e-unstable-tests" parallel="classes">
+        <packages>
+            <package name="teammates.e2e.util" />
+        </packages>
+        <classes>
+            <class name="teammates.e2e.cases.FeedbackSubmitPageE2ETest" />
+            <class name="teammates.e2e.cases.InstructorStudentActivityLogsPageE2ETest" />
+            <class name="teammates.e2e.cases.InstructorHomePageE2ETest" />
+            <class name="teammates.e2e.cases.InstructorSearchPageE2ETest" />
+        </classes>
+    </test>
+</suite>