lynxkite · darabos · Nov 30, 2022 · Nov 14, 2022 · Nov 14, 2022 · Nov 21, 2022
diff --git a/.github/workflows/earthly-test.yml b/.github/workflows/earthly-test.yml
@@ -1,8 +1,9 @@
-name: Run tests with Earthly
+name: Tests
 on: pull_request
 
 jobs:
   backend-test-spark:
+    name: Backend test with only Spark
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -40,6 +41,7 @@ jobs:
       run: earthly --ci --remote-cache=us-central1-docker.pkg.dev/external-lynxkite/github-actions-us/earthly:cache +backend-test-spark
 
   backend-test-sphynx:
+    name: Backend test with Sphynx
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -77,6 +79,7 @@ jobs:
       run: earthly --ci --remote-cache=us-central1-docker.pkg.dev/external-lynxkite/github-actions-us/earthly:cache +backend-test-sphynx
 
   python-test:
+    name: Python API test
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -112,3 +115,41 @@ jobs:
       run: earthly --version
     - name: Run Python test
       run: earthly --ci --remote-cache=us-central1-docker.pkg.dev/external-lynxkite/github-actions-us/earthly:cache +python-test
+
+  frontend-test:
+    name: Frontend test with Playwright
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    runs-on: ubuntu-latest
+    env:
+      FORCE_COLOR: 1
+    steps:
+    - uses: actions/checkout@v3
+    - id: 'auth'
+      name: 'Authenticate to Google Cloud'
+      uses: 'google-github-actions/auth@v0'
+      with:
+        workload_identity_provider: 'projects/422846954881/locations/global/workloadIdentityPools/github-actions/providers/github'
+        service_account: 'github-actions@external-lynxkite.iam.gserviceaccount.com'
+        token_format: 'access_token'
+    - uses: 'docker/login-action@v1'
+      with:
+        registry: 'us-central1-docker.pkg.dev'
+        username: 'oauth2accesstoken'
+        password: '${{ steps.auth.outputs.access_token }}'
+    - name: Put back the git branch into git (Earthly uses it for tagging)
+      run: |
+        branch=""
+        if [ -n "$GITHUB_HEAD_REF" ]; then
+          branch="$GITHUB_HEAD_REF"
+        else
+          branch="${GITHUB_REF##*/}"
+        fi
+        git checkout -b "$branch" || true
+    - name: Download latest earthly
+      run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.6.26/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'"
+    - name: Earthly version
+      run: earthly --version
+    - name: Run frontend test
+      run: earthly --ci --remote-cache=us-central1-docker.pkg.dev/external-lynxkite/github-actions-us/earthly:cache +frontend-test
diff --git a/Earthfile b/Earthfile
@@ -25,24 +25,24 @@ sbt-deps:
 
 npm-deps:
   COPY web/package.json web/yarn.lock web/
-  RUN cd web; yarn --frozen-lockfile
+  RUN cd web && yarn --frozen-lockfile
   RUN mkdir dependency-licenses
-  RUN cd web; yarn licenses list | egrep '^└─|^├─|^│  └─|^│  ├─|^   └─|^   ├─' > ../dependency-licenses/javascript.md
-  RUN cd web; yarn licenses generate-disclaimer > ../dependency-licenses/javascript.txt
+  RUN cd web && yarn licenses list | egrep '^└─|^├─|^│  └─|^│  ├─|^   └─|^   ├─' > ../dependency-licenses/javascript.md
+  RUN cd web && yarn licenses generate-disclaimer > ../dependency-licenses/javascript.txt
   SAVE ARTIFACT dependency-licenses
   SAVE ARTIFACT web/node_modules
   SAVE IMAGE --cache-hint
 
 grpc:
   COPY sphynx/go.mod sphynx/go.sum sphynx/proto_compile.sh sphynx/sphynx_common.sh sphynx/
   COPY sphynx/proto/*.proto sphynx/proto/
-  RUN mkdir app; sphynx/proto_compile.sh
+  RUN mkdir app && sphynx/proto_compile.sh
   SAVE ARTIFACT app/com/lynxanalytics/biggraph/graph_api/proto
 
 sphynx-build:
   FROM +grpc
   # Download dependencies.
-  RUN cd sphynx; go mod download
+  RUN cd sphynx && go mod download
   COPY sphynx sphynx
   RUN sphynx/build.sh
   SAVE ARTIFACT sphynx/.build/lynxkite-sphynx
@@ -55,7 +55,7 @@ web-build:
   COPY .eslintrc.yaml .
   COPY tools/gen_templates.py tools/
   COPY conf/kiterc_template conf/
-  RUN cd web; npx gulp
+  RUN cd web && npx gulp
   SAVE ARTIFACT web/dist
   SAVE IMAGE --cache-hint
 
@@ -130,23 +130,30 @@ frontend-test:
   RUN apt-get update && apt-get install -y xvfb
   RUN apt-get update && apt-get install -y chromium-browser
   USER mambauser
-  COPY tools/wait_for_port.sh tools/
-  COPY tools/with_lk.sh tools/
-  COPY tools/e2e_test.sh tools/
+  COPY web/package.json web/yarn.lock web/
   COPY +assembly/lynxkite.jar target/scala-2.12/lynxkite-0.1-SNAPSHOT.jar
   COPY +npm-deps/node_modules web/node_modules
+  USER root
+  # Playwright's "install-deps" wants to run "su" which asks for a password.
+  # Instead we're already root here, so we want to just run the command.
+  RUN cd web && npx playwright install-deps chromium --dry-run 2>/dev/null | sed -n 's/su root/bash/p' | bash
+  USER mambauser
+  RUN cd web && npx playwright install
+  COPY tools/wait_for_port.sh tools/
+  COPY tools/with_lk.sh tools/
   COPY web web
   COPY conf/kiterc_template conf/
   COPY test/localhost.self-signed.cert* test/
-  RUN xvfb-run -a tools/with_lk.sh tools/e2e_test.sh
+  ENV CI true
+  RUN cd web && ../tools/with_lk.sh yarn playwright test --trace on
 
 docker:
   FROM mambaorg/micromamba:jammy
   COPY conda-env.* .
   RUN ./conda-env.sh > env.yml && micromamba install -y -n base -f env.yml
   COPY +assembly/lynxkite.jar .
   COPY conf/kiterc_template .
-  CMD ["bash", "-c", ". kiterc_template; spark-submit lynxkite.jar"]
+  CMD ["bash", "-c", ". kiterc_template && spark-submit lynxkite.jar"]
   ENV KITE_ALLOW_PYTHON yes
   ENV KITE_ALLOW_R yes
   ENV KITE_META_DIR /meta

diff --git a/Makefile b/Makefile
@@ -29,7 +29,7 @@ clean:
 .build/frontend-test-passed: \
 		$(shell $(find) web/test) build.sbt .build/backend-done \
 		.build/documentation-verified .build/gulp-done
-	./test_frontend.sh && touch $@
+	cd web && ../tools/with_lk.sh yarn playwright test --trace on && touch $@
 .build/remote_api-python-test-passed: $(shell $(find) python/remote_api) .build/backend-done
 	tools/with_lk.sh python/remote_api/test.sh && python/remote_api/managed_tests/run.sh && touch $@
 dependency-licenses/scala.md: build.sbt

diff --git a/app/com/lynxanalytics/biggraph/LynxKite.scala b/app/com/lynxanalytics/biggraph/LynxKite.scala
@@ -83,10 +83,14 @@ object LynxKite {
 
   // Starts Sphynx and returns.
   private def startSphynx(): Process = synchronized {
+    val ldLibraryPath = Environment.envOrNone("LD_LIBRARY_PATH")
+    if (!ldLibraryPath.getOrElse("").startsWith(".:")) {
+      Environment.set("LD_LIBRARY_PATH" -> ("." +: ldLibraryPath.toSeq).mkString(":"))
+    }
     sphynxProcess = Process(
       Seq("./lynxkite-sphynx"),
       new java.io.File("lynxkite-sphynx"),
-      (Seq("LD_LIBRARY_PATH" -> ".") ++ Environment.get): _*).run()
+      Environment.get.toSeq: _*).run()
     sphynxProcess
   }
 }
diff --git a/dependency-licenses/javascript.md b/dependency-licenses/javascript.md
@@ -52,6 +52,7 @@
 │  ├─ @firebase/webchannel-wrapper@0.5.0
 │  ├─ @grpc/grpc-js@1.3.2
 │  ├─ @grpc/proto-loader@0.5.6
+│  ├─ @playwright/test@1.27.1
 │  ├─ aws-sign2@0.7.0
 │  ├─ browser-sync-ui@2.26.14
 │  ├─ browser-sync@2.26.14
@@ -67,6 +68,7 @@
 │  ├─ long@4.0.0
 │  ├─ oauth-sign@0.9.0
 │  ├─ pause-stream@0.0.11
+│  ├─ playwright-core@1.27.1
 │  ├─ request@2.88.2
 │  ├─ rx@4.1.0
 │  ├─ rxjs@5.5.12
@@ -268,6 +270,7 @@
 │  ├─ @types/fast-json-stable-stringify@2.1.0
 │  ├─ @types/long@4.0.2
 │  ├─ @types/node@17.0.38
+│  ├─ @types/node@18.11.9
 │  ├─ @types/q@0.0.32
 │  ├─ @types/selenium-webdriver@3.0.17
 │  ├─ accepts@1.3.7