Add accessibility tests workflow (#12082)

* Add skeleton for axe tests * Add testcases * Add remaining axe tests * Add remaining axe tests to workflow * FIx TestNgXmlTest * Add user-friendliness branch to workflow
TEAMMATES · Feb 19, 2023 · f737a0e · f737a0e
1 parent 376ef12
commit f737a0e
Show file tree

Hide file tree

Showing 41 changed files with 1,313 additions and 18 deletions.
diff --git a/.github/workflows/axe.yml b/.github/workflows/axe.yml
@@ -0,0 +1,54 @@
+name: Accessibility Tests
+
+on:
+  push:
+    branches:
+      - master 
+      - release
+      - user-friendliness
+  pull_request:
+    branches:
+      - master 
+      - release
+      - user-friendliness
+jobs:
+  axe-testing:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        browser: [firefox]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+      - name: Cache Gradle packages
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - name: Update Property File 
+        run: mv src/e2e/resources/test.ci-${{ matrix.browser }}.properties src/e2e/resources/test.properties
+      - name: Run Solr search service + local Datastore emulator
+        run: docker-compose up -d
+      - name: Create Config Files
+        run: ./gradlew createConfigs testClasses generateTypes 
+      - name: Install Frontend Dependencies
+        run: npm ci 
+      - name: Build Frontend Bundle 
+        run: npm run build
+      - name: Start Server
+        run: |
+          ./gradlew serverRun &
+          ./wait-for-server.sh
+      - name: Start Tests 
+        run: xvfb-run --server-args="-screen 0 1024x768x24" ./gradlew axeTests 
diff --git a/build.gradle b/build.gradle
@@ -76,6 +76,7 @@ dependencies {
     testImplementation("com.tngtech.archunit:archunit:0.11.0")
     testImplementation("junit:junit:4.13.2")
     testImplementation("org.seleniumhq.selenium:selenium-java:4.3.0")
+    testImplementation("com.deque.html.axe-core:selenium:4.6.0")
     testImplementation(testng)
     // For supporting authorization code flow locally
     testImplementation("com.google.oauth-client:google-oauth-client-jetty:1.34.1")
@@ -565,6 +566,24 @@ task e2eTests {
     e2eTests.dependsOn "e2eTestTry${id}"
 }
 
+task axeTests(type: Test) {
+    description "Runs the full accessibility test suite."
+    useTestNG()
+    options.suites "src/e2e/resources/testng-axe.xml"
+    options.outputDirectory = file("${buildDir}/reports/axe-test")
+    options.useDefaultListeners = true
+    ignoreFailures = false
+    maxHeapSize = "1g"
+    reports.html.required = false
+    reports.junitXml.required = false
+    jvmArgs "-Xss2m", "-Dfile.encoding=UTF-8"
+    testLogging {
+        events "passed"
+    }
+    afterTest afterTestClosure
+    afterSuite checkTestNgFailureClosure
+}
+
 // COVERAGE TASKS
 
 jacoco {

diff --git a/src/e2e/java/teammates/e2e/cases/AdminNotificationsPageE2ETest.java b/src/e2e/java/teammates/e2e/cases/AdminNotificationsPageE2ETest.java
@@ -4,6 +4,7 @@
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.Test;
 
 import teammates.common.datatransfer.NotificationStyle;
@@ -88,12 +89,13 @@ public void testAll() {
         notificationsPage.verifyStatusMessage("Notification has been deleted.");
         verifyAbsentInDatabase(newNotification);
 
-        ______TS("delete test notifications from database");
-        for (NotificationAttributes notification : notifications) {
-            notificationsPage.deleteNotification(notification);
-            verifyAbsentInDatabase(notification);
-        }
+    }
 
+    @AfterClass
+    public void classTeardown() {
+        for (NotificationAttributes notification : testData.notifications.values()) {
+            BACKDOOR.deleteNotification(notification.getNotificationId());
+        }
     }
 
 }
diff --git a/src/e2e/java/teammates/e2e/cases/BaseE2ETestCase.java b/src/e2e/java/teammates/e2e/cases/BaseE2ETestCase.java
@@ -43,9 +43,16 @@
  */
 public abstract class BaseE2ETestCase extends BaseTestCaseWithDatabaseAccess {
 
-    static final BackDoor BACKDOOR = BackDoor.getInstance();
+    /**
+     * Backdoor used to call APIs.
+     */
+    protected static final BackDoor BACKDOOR = BackDoor.getInstance();
+
+    /**
+     * Data to be used in the test.
+     */
+    protected DataBundle testData;
 
-    DataBundle testData;
     private Browser browser;
 
     @BeforeClass
@@ -295,7 +302,10 @@ protected InstructorAttributes getInstructor(InstructorAttributes instructor) {
         return getInstructor(instructor.getCourseId(), instructor.getEmail());
     }
 
-    String getKeyForInstructor(String courseId, String instructorEmail) {
+    /**
+     * Gets registration key for a given instructor.
+     */
+    protected String getKeyForInstructor(String courseId, String instructorEmail) {
         return getInstructor(courseId, instructorEmail).getKey();
     }
 
@@ -304,7 +314,10 @@ protected StudentAttributes getStudent(StudentAttributes student) {
         return BACKDOOR.getStudent(student.getCourse(), student.getEmail());
     }
 
-    String getKeyForStudent(StudentAttributes student) {
+    /**
+     * Gets registration key for a given student.
+     */
+    protected String getKeyForStudent(StudentAttributes student) {
         return getStudent(student).getKey();
     }
 

diff --git a/src/e2e/java/teammates/e2e/cases/InstructorNotificationsPageE2ETest.java b/src/e2e/java/teammates/e2e/cases/InstructorNotificationsPageE2ETest.java
@@ -4,6 +4,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.Test;
 
 import teammates.common.datatransfer.attributes.AccountAttributes;
@@ -61,11 +62,13 @@ public void testAll() {
 
         ______TS("notification banner is not visible");
         assertFalse(notificationsPage.isBannerVisible());
+    }
 
-        ______TS("delete test notifications from database");
+    @AfterClass
+    public void classTeardown() {
         for (NotificationAttributes notification : testData.notifications.values()) {
             BACKDOOR.deleteNotification(notification.getNotificationId());
-            verifyAbsentInDatabase(notification);
         }
     }
+
 }
diff --git a/src/e2e/java/teammates/e2e/cases/NotificationBannerE2ETest.java b/src/e2e/java/teammates/e2e/cases/NotificationBannerE2ETest.java
@@ -4,6 +4,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.Test;
 
 import teammates.common.datatransfer.attributes.AccountAttributes;
@@ -58,10 +59,12 @@ protected void testAll() {
         account.setReadNotifications(readNotifications);
         verifyPresentInDatabase(account);
 
-        ______TS("delete test notifications from database");
-        for (NotificationAttributes n : testData.notifications.values()) {
-            BACKDOOR.deleteNotification(n.getNotificationId());
-            verifyAbsentInDatabase(n);
+    }
+
+    @AfterClass
+    public void classTeardown() {
+        for (NotificationAttributes notification : testData.notifications.values()) {
+            BACKDOOR.deleteNotification(notification.getNotificationId());
         }
     }
 }
diff --git a/src/e2e/java/teammates/e2e/cases/StudentNotificationsPageE2ETest.java b/src/e2e/java/teammates/e2e/cases/StudentNotificationsPageE2ETest.java
@@ -4,6 +4,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import org.testng.annotations.AfterClass;
 import org.testng.annotations.Test;
 
 import teammates.common.datatransfer.attributes.AccountAttributes;
@@ -61,11 +62,13 @@ public void testAll() {
 
         ______TS("notification banner is not visible");
         assertFalse(notificationsPage.isBannerVisible());
+    }
 
-        ______TS("delete test notifications from database");
+    @AfterClass
+    public void classTeardown() {
         for (NotificationAttributes notification : testData.notifications.values()) {
             BACKDOOR.deleteNotification(notification.getNotificationId());
-            verifyAbsentInDatabase(notification);
         }
     }
+
 }
diff --git a/src/e2e/java/teammates/e2e/cases/axe/AdminAccountsPageAxeTest.java b/src/e2e/java/teammates/e2e/cases/axe/AdminAccountsPageAxeTest.java
@@ -0,0 +1,35 @@
+package teammates.e2e.cases.axe;
+
+import org.testng.annotations.Test;
+
+import com.deque.html.axecore.results.Results;
+
+import teammates.common.util.AppUrl;
+import teammates.common.util.Const;
+import teammates.e2e.cases.BaseE2ETestCase;
+import teammates.e2e.pageobjects.AdminAccountsPage;
+import teammates.e2e.util.AxeUtil;
+
+/**
+ * SUT: {@link Const.WebPageURIs#ADMIN_ACCOUNTS_PAGE}.
+ */
+public class AdminAccountsPageAxeTest extends BaseE2ETestCase {
+
+    @Override
+    protected void prepareTestData() {
+        testData = loadDataBundle("/AdminAccountsPageE2ETest.json");
+        removeAndRestoreDataBundle(testData);
+    }
+
+    @Test
+    @Override
+    public void testAll() {
+        AppUrl accountsPageUrl = createFrontendUrl(Const.WebPageURIs.ADMIN_ACCOUNTS_PAGE)
+                .withParam(Const.ParamsNames.INSTRUCTOR_ID, "tm.e2e.AAccounts.instr2");
+        AdminAccountsPage accountsPage = loginAdminToPage(accountsPageUrl, AdminAccountsPage.class);
+
+        Results results = AxeUtil.AXE_BUILDER.analyze(accountsPage.getBrowser().getDriver());
+        assertTrue(AxeUtil.formatViolations(results), results.violationFree());
+    }
+
+}
diff --git a/src/e2e/java/teammates/e2e/cases/axe/AdminHomePageAxeTest.java b/src/e2e/java/teammates/e2e/cases/axe/AdminHomePageAxeTest.java
@@ -0,0 +1,34 @@
+package teammates.e2e.cases.axe;
+
+import org.testng.annotations.Test;
+
+import com.deque.html.axecore.results.Results;
+
+import teammates.common.util.AppUrl;
+import teammates.common.util.Const;
+import teammates.e2e.cases.BaseE2ETestCase;
+import teammates.e2e.pageobjects.AdminHomePage;
+import teammates.e2e.util.AxeUtil;
+
+/**
+ * SUT: {@link Const.WebPageURIs#ADMIN_HOME_PAGE}.
+ */
+public class AdminHomePageAxeTest extends BaseE2ETestCase {
+
+    @Override
+    protected void prepareTestData() {
+        testData = loadDataBundle("/AdminHomePageE2ETest.json");
+        removeAndRestoreDataBundle(testData);
+    }
+
+    @Test
+    @Override
+    public void testAll() {
+        AppUrl url = createFrontendUrl(Const.WebPageURIs.ADMIN_HOME_PAGE);
+        AdminHomePage homePage = loginAdminToPage(url, AdminHomePage.class);
+
+        Results results = AxeUtil.AXE_BUILDER.analyze(homePage.getBrowser().getDriver());
+        assertTrue(AxeUtil.formatViolations(results), results.violationFree());
+    }
+
+}
diff --git a/src/e2e/java/teammates/e2e/cases/axe/AdminNotificationsPageAxeTest.java b/src/e2e/java/teammates/e2e/cases/axe/AdminNotificationsPageAxeTest.java
@@ -0,0 +1,34 @@
+package teammates.e2e.cases.axe;
+
+import org.testng.annotations.Test;
+
+import com.deque.html.axecore.results.Results;
+
+import teammates.common.util.AppUrl;
+import teammates.common.util.Const;
+import teammates.e2e.cases.BaseE2ETestCase;
+import teammates.e2e.pageobjects.AdminNotificationsPage;
+import teammates.e2e.util.AxeUtil;
+
+/**
+ * SUT: {@link Const.WebPageURIs#ADMIN_NOTIFICATIONS_PAGE}.
+ */
+public class AdminNotificationsPageAxeTest extends BaseE2ETestCase {
+
+    @Override
+    protected void prepareTestData() {
+        testData = loadDataBundle("/AdminNotificationsPageE2ETest.json");
+        removeAndRestoreDataBundle(testData);
+    }
+
+    @Test
+    @Override
+    public void testAll() {
+        AppUrl url = createFrontendUrl(Const.WebPageURIs.ADMIN_NOTIFICATIONS_PAGE);
+        AdminNotificationsPage notificationsPage = loginAdminToPage(url, AdminNotificationsPage.class);
+
+        Results results = AxeUtil.AXE_BUILDER.analyze(notificationsPage.getBrowser().getDriver());
+        assertTrue(AxeUtil.formatViolations(results), results.violationFree());
+    }
+
+}
diff --git a/src/e2e/java/teammates/e2e/cases/axe/AdminSearchPageAxeTest.java b/src/e2e/java/teammates/e2e/cases/axe/AdminSearchPageAxeTest.java
@@ -0,0 +1,47 @@
+package teammates.e2e.cases.axe;
+
+import org.testng.annotations.Test;
+
+import com.deque.html.axecore.results.Results;
+
+import teammates.common.util.AppUrl;
+import teammates.common.util.Const;
+import teammates.e2e.cases.BaseE2ETestCase;
+import teammates.e2e.pageobjects.AdminSearchPage;
+import teammates.e2e.util.AxeUtil;
+import teammates.e2e.util.TestProperties;
+
+/**
+ * SUT: {@link Const.WebPageURIs#ADMIN_SEARCH_PAGE}.
+ */
+public class AdminSearchPageAxeTest extends BaseE2ETestCase {
+
+    @Override
+    protected void prepareTestData() {
+        if (!TestProperties.INCLUDE_SEARCH_TESTS) {
+            return;
+        }
+
+        testData = loadDataBundle("/AdminSearchPageE2ETest.json");
+        removeAndRestoreDataBundle(testData);
+        putDocuments(testData);
+    }
+
+    @Test
+    @Override
+    public void testAll() {
+        if (!TestProperties.INCLUDE_SEARCH_TESTS) {
+            return;
+        }
+
+        AppUrl url = createFrontendUrl(Const.WebPageURIs.ADMIN_SEARCH_PAGE);
+        AdminSearchPage searchPage = loginAdminToPage(url, AdminSearchPage.class);
+
+        searchPage.inputSearchContent(testData.students.get("student1InCourse1").getEmail());
+        searchPage.clickSearchButton();
+
+        Results results = AxeUtil.AXE_BUILDER.analyze(searchPage.getBrowser().getDriver());
+        assertTrue(AxeUtil.formatViolations(results), results.violationFree());
+    }
+
+}