Test coverage (#5775)

* Adding a coverage workflow * Ignoring test coverage artifacts * Installing istanbul and wireit commands * Fixed --workspace parameter * Must use CONTEXT from "@realm/integration-tests" * Updated the package-lock.json * Adjusting timeouts and reporter * Update pr-coverage.yml * Using docker image to increase speed * Adding a wait-on the server * Authenticate docker to pull the server * Disabling app reuse now that we're running against docker * Moved ignore of coverage artifacts to the root * Instrumenting SDK for coverage * Documented reuseApp context variable * Removed unneeded "source-map-support"
realm · May 1, 2023 · 2ba48d3 · 2ba48d3
1 parent 0f5c45c
commit 2ba48d3
Show file tree

Hide file tree

Showing 10 changed files with 1,118 additions and 10 deletions.
diff --git a/.github/workflows/pr-coverage.yml b/.github/workflows/pr-coverage.yml
@@ -0,0 +1,73 @@
+name: Test Coverage
+on: ["push", "pull_request"]
+
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  REALM_DISABLE_ANALYTICS: 1
+  MOCHA_TIMEOUT: 60000 # 1 minute
+  LONG_TIMEOUT: 300000 # 5 minutes
+  SERVER_TAG: latest
+  # REALM_BASE_URL: ${{ secrets.REALM_QA_BASE_URL }}
+  # REALM_PUBLIC_KEY: ${{ secrets.ATLAS_QA_PUBLIC_API_KEY }}
+  # REALM_PRIVATE_KEY: ${{ secrets.ATLAS_QA_PRIVATE_API_KEY }}
+
+jobs:
+  run:
+    name: Build and tests with coverage
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Setup node version
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          cache: npm
+
+      - name: Setup Wireit cache
+        uses: google/wireit@setup-github-actions-caching/v1
+
+      - name: Environment setup
+        run: sudo apt-get install ccache ninja-build
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1
+        with:
+          key: coverage
+          max-size: '2.0G'
+
+      - name: Docker Login
+        uses: azure/docker-login@v1
+        with:
+          login-server: ghcr.io
+          username: realm-ci
+          password: ${{ secrets.REALM_CI_GITHUB_API_KEY }}
+
+      - name: Start server
+        run: docker run --detach --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --publish 9090:9090 ghcr.io/realm/ci/mongodb-realm-test-server:${{ env.SERVER_TAG }}
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.BAAS_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.BAAS_AWS_SECRET_ACCESS_KEY }}
+
+      - name: Install dependencies
+        # Ignoring scripts to prevent a prebuilt from getting fetched / built
+        run: npm ci --ignore-scripts
+
+      - name: Wait for the server to start
+        run: npx wait-on http://localhost:9090
+
+      - name: Run tests with coverage
+        timeout-minutes: 60
+        env:
+          CONTEXT: syncLogLevel=warn,longTimeout=${{ env.LONG_TIMEOUT }},reuseApp=false
+          # CONTEXT: syncLogLevel=warn,longTimeout=${{ env.LONG_TIMEOUT }},realmBaseUrl=${{ secrets.REALM_QA_BASE_URL }},mongodbClusterName=${{ secrets.ATLAS_QA_DAILY_CLUSTER_NAME }},privateKey=${{ secrets.ATLAS_QA_PRIVATE_API_KEY }},publicKey=${{ secrets.ATLAS_QA_PUBLIC_API_KEY }}
+        run: npm run ci:coverage --workspace @realm/integration-tests -- --reporter mocha-github-actions-reporter --timeout ${{ env.MOCHA_TIMEOUT }}
+
+      - name: Coveralls
+        uses: coverallsapp/github-action@v2
diff --git a/.gitignore b/.gitignore
@@ -88,6 +88,10 @@ integration-tests/tests/src/bench/rolledup.js
 # TODO: Solve .lock files are produced in "throws when called with an empty string"
 integration-tests/**/.lock
 
+# Test coverage artifacts
+coverage/
+.nyc_output/
+
 # Realm App Importer artifacts
 **/imported-apps/
 **/realm-config

diff --git a/integration-tests/README.md b/integration-tests/README.md
@@ -79,6 +79,7 @@ Examples of context variables used:
 - `integration=false`: Skip the integration test (which performance tests are not considered a part of).
 - `preserveAppAfterRun`: Skip deleting the Realm app after the test run
 - `syncLogLevel=all`: Set the sync client log level to help debugging sync client issues.
+- `reuseApp=true`: Instructs the app importer to reuse and reconfigure a single app. Defaults to `true`.
 - `realmBaseUrl=https://localhost:9090`: Set the base URL used when connecting the the server.
 - `mongodbClusterName=Cluster0`: Set the name of the cluster, used when setting up the "mongodb-atlas" service on imported apps.
 - `mongodbServiceType`: Set the type of mongodb service, used when importing. Defaults to `mongodb` or `mongodb-atlas` if `mongodbClusterName` is set.

diff --git a/integration-tests/tests/.gitignore b/integration-tests/tests/.gitignore
@@ -1 +1 @@
-realm-apps
+realm-apps/
diff --git a/integration-tests/tests/.nycrc.json b/integration-tests/tests/.nycrc.json
@@ -0,0 +1,8 @@
+{
+  "extends": "@istanbuljs/nyc-config-typescript",
+  "all": true,
+  "cwd": "../..",
+  "include": [
+    "packages/realm/src/**"
+  ]
+}
diff --git a/integration-tests/tests/package.json b/integration-tests/tests/package.json
@@ -10,20 +10,50 @@
 	"private": true,
 	"scripts": {
 		"build": "wireit",
-		"start": "mocha --watch",
-		"test": "mocha --exit",
-		"lint": "eslint --ext .js,.ts ."
+		"start": "wireit",
+		"test": "wireit",
+		"lint": "eslint --ext .js,.ts .",
+		"coverage": "wireit",
+		"ci:coverage": "wireit"
 	},
 	"wireit": {
 		"build": {
 			"command": "tsc",
 			"dependencies": [
+				"../../packages/realm:bundle",
 				"build-dependencies"
 			]
 		},
-		"build-dependencies": {
+		"start": {
+			"command": "mocha --watch",
 			"dependencies": [
 				"../../packages/realm:bundle",
+				"build-dependencies"
+			]
+		},
+		"test": {
+			"command": "mocha --exit",
+			"dependencies": [
+				"../../packages/realm:bundle",
+				"build-dependencies"
+			]
+		},
+		"coverage": {
+			"command": "nyc mocha --exit",
+			"dependencies": [
+				"../../packages/realm:bundle:coverage",
+				"build-dependencies"
+			]
+		},
+		"ci:coverage": {
+			"command": "nyc --reporter=lcov -- mocha --exit",
+			"dependencies": [
+				"../../packages/realm:bundle:coverage",
+				"build-dependencies"
+			]
+		},
+		"build-dependencies": {
+			"dependencies": [
 				"../../packages/realm:build:node",
 				"../../packages/mocha-reporter:bundle",
 				"../../packages/realm-network-transport:bundle"
@@ -34,6 +64,7 @@
 		"realm": "*"
 	},
 	"devDependencies": {
+		"@istanbuljs/nyc-config-typescript": "^1.0.2",
 		"@realm/app-importer": "*",
 		"@realm/mocha-reporter": "*",
 		"@thi.ng/bench": "^3.1.16",
@@ -45,12 +76,13 @@
 		"concurrently": "^6.5.1",
 		"mocha": "^10.1.0",
 		"node-fetch": "^2.6.9",
+		"nyc": "^15.1.0",
 		"platform": "^1.3.6",
 		"realm": "*"
 	},
 	"dependencies": {
-		"@thi.ng/bench": "^3.1.16",
 		"@realm/network-transport": "^0.7.2",
+		"@thi.ng/bench": "^3.1.16",
 		"chai": "4.3.6",
 		"chai-as-promised": "^7.1.1",
 		"concurrently": "^6.0.2",

diff --git a/integration-tests/tests/src/utils/import-app.ts b/integration-tests/tests/src/utils/import-app.ts
@@ -23,7 +23,7 @@ export type ErrorResponse = { message: string; appId: never };
 export type ImportResponse = { appId: string; message: never };
 export type Response = ImportResponse | ErrorResponse;
 
-const { realmBaseUrl = "http://localhost:9090" } = environment;
+const { realmBaseUrl = "http://localhost:9090", reuseApp = true } = environment;
 
 export const baseUrl = realmBaseUrl;
 
@@ -49,7 +49,7 @@ const credentials = getCredentials();
 const importer = new AppImporter({
   baseUrl: realmBaseUrl,
   credentials,
-  reuseApp: true,
+  reuseApp,
 });
 
 export async function importApp(config: AppConfig): Promise<{ appId: string; baseUrl: string }> {