fix reporting variables in the community report

.github/workflows/compatibility-matrix-testing.yml

@@ -107,7 +107,6 @@ jobs:
     secrets: inherit
     with:
       run-ios-tests: true
-      run-android-tests: false
       run-type: "RELEASE"
       MM_TEST_SERVER_URL: ${{ matrix.server.url }}
       MOBILE_VERSION: ${{ inputs.MOBILE_VERSION }}

.github/workflows/e2e-detox-pr.yml

@@ -55,7 +55,6 @@ jobs:
       - build-ios-simulator
     with:
       run-ios-tests: true
-      run-android-tests: false
       run-type: "PR"
       MOBILE_VERSION: ${{ github.event.pull_request.head.sha }}
     secrets: inherit
@@ -99,11 +98,3 @@ jobs:
               });
             }
 
-            if (labels.includes(androidLabel)) {
-              github.rest.issues.removeLabel({
-                issue_number: context.issue.number,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                name: androidLabel,
-              });
-            }

.github/workflows/e2e-detox-release.yml

@@ -52,8 +52,8 @@ jobs:
       - build-ios-simulator
     with:
       run-ios-tests: true
-      run-android-tests: false
       run-type: "RELEASE"
+      record_tests_in_zephyr: 'true'
       MOBILE_VERSION: ${{ github.ref }}
     secrets: inherit
 

.github/workflows/e2e-detox-scheduled.yml

@@ -2,7 +2,7 @@ name: Detox E2E Tests (Scheduled)
 
 on:
   schedule:
-    - cron: "0 0 * * *" # every day at midnight
+    - cron: "0 0 * * 4,5" # Wednesday and Thursday midnight
 
 jobs:
   update-initial-status:
@@ -51,8 +51,8 @@ jobs:
       - build-ios-simulator
     with:
       run-ios-tests: true
-      run-android-tests: false
       run-type: "MAIN"
+      record_tests_in_zephyr: 'true'
       MOBILE_VERSION: ${{ github.ref }}
     secrets: inherit
 

.github/workflows/e2e-detox-template.yml

@@ -24,10 +24,6 @@ on:
         description: "Run iOS tests"
         required: true
         type: boolean
-      run-android-tests:
-        description: "Run Android tests"
-        required: true
-        type: boolean
       run-type:
         type: string
         required: false
@@ -37,6 +33,21 @@ on:
         required: false
         type: boolean
         default: false
+      record_tests_in_zephyr:
+        description: "Record test results in Zephyr, typically for nightly and release runs"
+        required: false
+        type: string
+        default: 'false'
+      ios_device_name:
+        description: "iPhone simulator name"
+        required: false
+        type: string
+        default: "iPhone 14"
+      ios_device_os_name:
+        description: "iPhone simulator OS version"
+        required: false
+        type: string
+        default: "iOS 17.0"
     outputs:
       STATUS:
         value: ${{ jobs.generate-report.outputs.STATUS }}
@@ -48,16 +59,20 @@ on:
 env:
   AWS_REGION: "us-east-1"
   ADMIN_EMAIL: ${{ secrets.MM_MOBILE_E2E_ADMIN_EMAIL }}
-  ADMIN_USERNAME: ${{ inputs.MM_TEST_USER_NAME || secrets.MM_MOBILE_E2E_ADMIN_USERNAME }}
-  ADMIN_PASSWORD: ${{ inputs.MM_TEST_PASSWORD || secrets.MM_MOBILE_E2E_ADMIN_PASSWORD }}
+  ADMIN_USERNAME: ${{ secrets.MM_MOBILE_E2E_ADMIN_USERNAME }}
+  ADMIN_PASSWORD: ${{ secrets.MM_MOBILE_E2E_ADMIN_PASSWORD }}
+  # BRANCH: ${{ github.head_ref || github.ref_name }} 
+  # COMMIT_HASH: ${{ github.sha }}
+  DEVICE_NAME: ${{ inputs.ios_device_name }}
+  DEVICE_OS_VERSION: ${{ inputs.ios_device_os_name }}
   DETOX_AWS_S3_BUCKET: "mattermost-detox-report"
   HEADLESS: "true"
   TYPE: ${{ inputs.run-type }}
   PULL_REQUEST: "https://github.com/mattermost/mattermost-mobile/pull/${{ github.event.number }}"
-  SITE_1_URL: ${{ inputs.MM_TEST_SERVER_URL || secrets.MM_MOBILE_E2E_SITE_1_URL }}
-  SITE_2_URL: ${{ secrets.MM_MOBILE_E2E_SITE_2_URL }}
-  SITE_3_URL: ${{ secrets.MM_MOBILE_E2E_SITE_3_URL }}
-  ZEPHYR_ENABLE: false
+  SITE_1_URL: ${{ inputs.MM_TEST_SERVER_URL || 'https://mobile-e2e-site-1.test.mattermost.cloud' }}
+  SITE_2_URL: "https://mobile-e2e-site-2.test.mattermost.cloud"
+  SITE_3_URL: "https://mobile-e2e-site-3.test.mattermost.cloud"
+  ZEPHYR_ENABLE: ${{ inputs.record_tests_in_zephyr }}
   JIRA_PROJECT_KEY: "MM"
   ZEPHYR_API_KEY: ${{ secrets.MM_MOBILE_E2E_ZEPHYR_API_KEY }}
   ZEPHYR_FOLDER_ID: "3233873"
@@ -68,12 +83,10 @@ env:
 jobs:
   generate-specs:
     runs-on: ubuntu-22.04
-    if: ${{ inputs.run-ios-tests || inputs.run-android-tests }}
+    if: ${{ inputs.run-ios-tests }}
     outputs:
       specs: ${{ steps.generate-specs.outputs.specs }}
       build_id: ${{ steps.resolve-device.outputs.BUILD_ID }}
-      device_name: ${{ steps.resolve-device.outputs.DEVICE_NAME }}
-      device_os_version: ${{ steps.resolve-device.outputs.DEVICE_OS_VERSION }}
       mobile_sha: ${{ steps.resolve-device.outputs.MOBILE_SHA }}
       mobile_ref: ${{ steps.resolve-device.outputs.MOBILE_REF }}
       workflow_hash: ${{ steps.resolve-device.outputs.WORKFLOW_HASH }}
@@ -86,18 +99,13 @@ jobs:
       - name: Set Build ID
         id: resolve-device
         run: |
-          cd detox
-          DEVICE_NAME=$(node -p "require('./.detoxrc').devices['ios.simulator'].device.type")
-          DEVICE_OS_VERSION=$(node -p "require('./.detoxrc').devices['ios.simulator'].device.os")
+          BUILD_ID="${{ github.run_id }}-${{ env.DEVICE_NAME }}-${{ env.DEVICE_OS_VERSION}}"
           WORKFLOW_HASH=$(tr -dc a-z0-9 </dev/urandom | head -c 10)
-          BUILD_ID="${{ github.run_id }}-${DEVICE_NAME}-${DEVICE_OS_VERSION}-${WORKFLOW_HASH}"
 
           ## We need that hash to separate the artifacts
           echo "WORKFLOW_HASH=${WORKFLOW_HASH}" >> ${GITHUB_OUTPUT}
 
           echo "BUILD_ID=$(echo ${BUILD_ID} | sed 's/ /_/g')" >> ${GITHUB_OUTPUT}
-          echo "DEVICE_NAME=${DEVICE_NAME}" >> ${GITHUB_OUTPUT}
-          echo "DEVICE_OS_VERSION=${DEVICE_OS_VERSION}" >> ${GITHUB_OUTPUT}
           echo "MOBILE_SHA=$(git rev-parse HEAD)" >> ${GITHUB_OUTPUT}
           echo "MOBILE_REF=$(git rev-parse --abbrev-ref HEAD)" >> ${GITHUB_OUTPUT}
 
@@ -107,8 +115,8 @@ jobs:
         with:
           parallelism: 10
           search_path: detox/e2e/test
-          device_name: ${{ steps.resolve-device.outputs.DEVICE_NAME }}
-          device_os_version: ${{ steps.resolve-device.outputs.DEVICE_OS_VERSION }}
+          device_name: ${{ env.DEVICE_NAME }}
+          device_os_version: ${{ env.DEVICE_OS_VERSION }}
 
   e2e-ios:
     name: ios-detox-e2e-${{ matrix.runId }}-${{ matrix.deviceName }}-${{ matrix.deviceOsVersion }}
@@ -154,7 +162,13 @@ jobs:
 
       - name: Run Detox E2E Tests
         continue-on-error: true # Label might have been removed manually
-        run: cd detox && npm run e2e:ios-test -- ${{ matrix.specs }}
+        run: |
+          cd detox
+          npm run detox:config-gen
+          npm run e2e:ios-test -- ${{ matrix.specs }}
+        env:
+          DETOX_DEVICE_TYPE: ${{ env.DEVICE_NAME }}
+          DETOX_OS_VERSION: ${{ env.DEVICE_OS_VERSION }}
 
       - name: Upload iOS Test Report
         if: always()
@@ -165,7 +179,7 @@ jobs:
 
   generate-report:
     runs-on: ubuntu-22.04
-    if: ${{ inputs.run-ios-tests || inputs.run-android-tests }}
+    if: ${{ inputs.run-ios-tests}}
     needs:
       - generate-specs
       - e2e-ios
@@ -205,8 +219,6 @@ jobs:
           DETOX_AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_MOBILE_DETOX_AWS_SECRET_ACCESS_KEY }}
           IOS: ${{ inputs.run-ios-tests }}
           BUILD_ID: ${{ needs.generate-specs.outputs.build_id }}
-          DEVICE_NAME: ${{ needs.generate-specs.outputs.device_name }}
-          DEVICE_OS_VERSION: ${{ needs.generate-specs.outputs.device_os_version }}
           REPORT_PATH: ${{ steps.s3.outputs.path }}
           ## These are needed for the MM Webhook report
           COMMIT_HASH: ${{ needs.generate-specs.outputs.mobile_sha }}
@@ -225,11 +237,7 @@ jobs:
       - name: Set Target URL
         id: set-url
         run: |
-          if ${{ inputs.run-ios-tests }}; then
             echo "TARGET_URL=https://${{ env.DETOX_AWS_S3_BUCKET }}.s3.amazonaws.com/${{ steps.s3.outputs.path }}/jest-stare/ios-report.html" >> ${GITHUB_OUTPUT}
-          else
-            echo "TARGET_URL=https://${{ env.DETOX_AWS_S3_BUCKET }}.s3.amazonaws.com/${{ steps.s3.outputs.path }}/jest-stare/android-report.html" >> ${GITHUB_OUTPUT}
-          fi
 
       - name: Determine Status
         id: determine-status

detox/.detoxrc.json

@@ -30,8 +30,8 @@
     "ios.simulator": {
       "type": "ios.simulator",
       "device": {
-        "type": "iPhone 15 Pro",
-        "os": "iOS 17.4"
+        "type": "__DEVICE_TYPE__",
+        "os": "__DEVICE_OS__"
       }
     },
     "android.emulator": {

detox/e2e/support/server_api/common.ts

@@ -32,7 +32,7 @@ export const getCookiesFromConfig = (config: AxiosRequestConfig<any>) => {
 export const getResponseFromError = (err: any) => {
     const {response} = err;
     if (!response) {
-        const message = `No response from server at "${err.config.baseURL}".
+        const message = `No response from server at "${err}".
 If testing against a server other than the default local instance, you may set the server URL via "SITE_URL" environment variable.
 `;
 

detox/package.json

@@ -51,6 +51,7 @@
     "e2e:ios-test": "IOS=true detox test -c ios.sim.debug",
     "e2e:ios-build-release": "detox build -c ios.sim.release",
     "e2e:ios-test-release": "IOS=true detox test -c ios.sim.release --record-logs failing --take-screenshots failing",
+    "detox:config-gen": "cd utils && node generate_detox_config_ci.js",
     "check": "npm run lint && npm run tsc",
     "clean-detox": "detox clean",
     "lint": "eslint --ignore-pattern node_modules --quiet .",

detox/save_report.js

@@ -69,7 +69,6 @@ const saveReport = async () => {
         WEBHOOK_URL,
         ZEPHYR_ENABLE,
         ZEPHYR_CYCLE_KEY,
-        REPORT_PATH,
     } = process.env;
 
     // Remove old generated reports
@@ -100,7 +99,7 @@ const saveReport = async () => {
     // Merge all XML reports into one single XML report
     const platform = process.env.IOS === 'true' ? 'ios' : 'android';
     const combinedFilePath = `${ARTIFACTS_DIR}/${platform}-combined.xml`;
-    await mergeFiles(path.join(__dirname, combinedFilePath), [`${ARTIFACTS_DIR}/ios-results*/${platform}-junit*.xml`]);
+    await mergeFiles(path.join(__dirname, combinedFilePath), [`${ARTIFACTS_DIR}/${platform}-results*/${platform}-junit*.xml`]);
     console.log(`Merged, check ${combinedFilePath}`);
 
     // Read XML from a file
@@ -120,27 +119,27 @@ const saveReport = async () => {
         fs.mkdirSync(jestStareOutputDir, {recursive: true});
     }
 
-    // 'ios-results-*' is the path in CI where the parallel detox jobs save artifacts
-    await mergeJestStareJsonFiles(jestStareCombinedFilePath, [`${ARTIFACTS_DIR}/ios-results*/jest-stare/${platform}-data*.json`]);
+    // "ios-results-*" or "android-results-*" is the path in CI where the parallel detox jobs save the artifacts
+    await mergeJestStareJsonFiles(jestStareCombinedFilePath, [`${ARTIFACTS_DIR}/${platform}-results*/jest-stare/${platform}-data*.json`]);
     generateJestStareHtmlReport(jestStareOutputDir, `${platform}-report.html`, jestStareCombinedFilePath);
 
     if (process.env.CI) {
-        // Delete folders starting with "iOS-results-" only in CI environment
+        // Delete folders starting with "ios-results-" or "android-results-" only in CI environment
         const entries = fs.readdirSync(ARTIFACTS_DIR, {withFileTypes: true});
         for (const entry of entries) {
-            if (entry.isDirectory() && entry.name.startsWith('ios-results-')) {
+            if (entry.isDirectory() && entry.name.startsWith(`${platform}-results-`)) {
                 fs.rmSync(path.join(ARTIFACTS_DIR, entry.name), {recursive: true});
             }
         }
     }
-    const result = await saveArtifacts(REPORT_PATH);
+    const result = await saveArtifacts();
     if (result && result.success) {
         console.log('Successfully uploaded artifacts to S3:', result.reportLink);
     }
 
     // Create or use an existing test cycle
     let testCycle = {};
-    if (ZEPHYR_ENABLE === 'true') {
+    if (ZEPHYR_ENABLE === true) {
         const {start, end} = summary.stats;
         testCycle = ZEPHYR_CYCLE_KEY ? {key: ZEPHYR_CYCLE_KEY} : await createTestCycle(start, end);
     }

detox/utils/artifacts.js

@@ -21,6 +21,7 @@ const {
     DETOX_AWS_ACCESS_KEY_ID,
     DETOX_AWS_SECRET_ACCESS_KEY,
     IOS,
+    REPORT_PATH,
 } = process.env;
 const platform = IOS === 'true' ? 'ios' : 'android';
 
@@ -35,7 +36,7 @@ function getFiles(dirPath) {
     return fs.existsSync(dirPath) ? readdir(dirPath) : [];
 }
 
-async function saveArtifacts(s3Folder) {
+async function saveArtifacts() {
     if (!DETOX_AWS_S3_BUCKET || !DETOX_AWS_ACCESS_KEY_ID || !DETOX_AWS_SECRET_ACCESS_KEY) {
         console.log('No AWS credentials found. Test artifacts not uploaded to S3.');
 
@@ -50,7 +51,7 @@ async function saveArtifacts(s3Folder) {
             filesToUpload,
             10,
             async.asyncify(async (file) => {
-                const Key = file.replace(uploadPath, s3Folder);
+                const Key = file.replace(uploadPath, REPORT_PATH);
                 const contentType = mime.lookup(file);
                 const charset = mime.charset(contentType);
 
@@ -76,7 +77,7 @@ async function saveArtifacts(s3Folder) {
                     return reject(new Error(err));
                 }
 
-                const reportLink = `https://${DETOX_AWS_S3_BUCKET}.s3.amazonaws.com/${s3Folder}/jest-stare/${platform}-report.html`;
+                const reportLink = `https://${DETOX_AWS_S3_BUCKET}.s3.amazonaws.com/${REPORT_PATH}/jest-stare/${platform}-report.html`;
                 resolve({success: true, reportLink});
             },
         );

detox/utils/generate_detox_config_ci.js

@@ -0,0 +1,14 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+/* eslint-disable no-process-env */
+/* eslint-disable no-console */
+const fs = require('fs');
+const deviceType = process.env.DETOX_DEVICE_TYPE || 'iPhone 15 Pro';
+const deviceOS = process.env.DETOX_DEVICE_OS || 'iOS 17.4';
+const detoxConfigTemplate = fs.readFileSync('../.detoxrc.json', 'utf8');
+const detoxConfig = detoxConfigTemplate.replace('__DEVICE_TYPE__', deviceType).replace('__DEVICE_OS__', deviceOS);
+
+fs.writeFileSync('../.detoxrc.json', detoxConfig);
+
+console.log('Detox configuration generated successfully');

detox/utils/report.js

@@ -308,8 +308,8 @@ function generateTestReport(summary, isUploadedToS3, reportLink, environment, te
 function generateTitle() {
     const {
         BRANCH,
-        DETOX_AWS_S3_BUCKET,
         COMMIT_HASH,
+        DETOX_AWS_S3_BUCKET,
         IOS,
         PULL_REQUEST,
         RELEASE_BUILD_NUMBER,
@@ -324,7 +324,7 @@ function generateTitle() {
     const appExtension = IOS === 'true' ? 'ipa' : 'apk';
     const appFileName = `Mattermost_Beta.${appExtension}`;
     const appBuildType = 'mattermost-mobile-beta';
-    const s3Folder = `${platform.toLocaleLowerCase()}/${REPORT_PATH}`.replace(/\./g, '-');
+    const s3Folder = `${platform.toLocaleLowerCase()}/${REPORT_PATH}`;
     const appFilePath = IOS === 'true' ? 'Mattermost-simulator-x86_64.app.zip' : 'android/app/build/outputs/apk/release/app-release.apk';
     let buildLink = '';
     let releaseDate = '';