Screenshot test tracing view (#2858)

* add simple test that screenshots the tracing view using puppeteer, not running under docker yet (#2822)

* use puppeteer docker image for screenshot tests, add test file that tests dataset on dev server and does pixel diffs with existing screenshots (#2822)

* revert some earlier changes (#2822)

* fix linting (#2822)

* apply PR feedback (#2822)

* add retry of 3 for screenshot tests (#2822)

* parallelize some awaits (#2822)

* insert x-auth-token with predefined value for scmboy

* remove puppeteer docker file from this repo, use pre-built docker image instead

* replace dev auth token for screenshot tests

* change BRANCH env variable to URL

.gitignore

@@ -64,5 +64,7 @@ app/assets/javascripts/test/snapshots/debug-htmls/*
 .eslintcache
 *.sublime-project
 *.sublime-workspace
+**/screenshots/*.diff.png
+**/screenshots/*.new.png
 .nyc_output
 coverage

app/assets/javascripts/test/puppeteer/dataset_rendering.screenshot.js

@@ -0,0 +1,78 @@
+/* eslint import/no-extraneous-dependencies: ["error", {"peerDependencies": true}] */
+const test = require("ava");
+const puppeteer = require("puppeteer");
+const path = require("path");
+const { screenshotDataset, DEV_AUTH_TOKEN } = require("./dataset_rendering_helpers");
+const { compareScreenshot } = require("./screenshot_helpers");
+
+process.on("unhandledRejection", (err, promise) => {
+  console.error("Unhandled rejection (promise: ", promise, ", reason: ", err, ").");
+});
+
+const BASE_PATH = path.join(__dirname, "../screenshots");
+
+let URL = "https://master.webknossos.xyz";
+if (!process.env.URL) {
+  console.warn(
+    "[Warning] No url specified, assuming dev master. If you want to specify a URL, prepend URL=<url> to the command.",
+  );
+} else {
+  URL = process.env.URL;
+  // Prepend https:// if not specified
+  if (!/^https?:\/\//i.test(URL)) {
+    URL = `https://${URL}`;
+  }
+}
+console.log(`[Info] Executing tests on URL ${URL}.`);
+
+async function getNewPage(browser) {
+  const page = await browser.newPage();
+  page.setViewport({ width: 1920, height: 1080 });
+  page.setExtraHTTPHeaders({
+    "X-Auth-Token": DEV_AUTH_TOKEN,
+  });
+  return page;
+}
+
+test.beforeEach(async t => {
+  t.context.browser = await puppeteer.launch({
+    args: ["--headless", "--hide-scrollbars", "--no-sandbox", "--disable-setuid-sandbox"],
+  });
+});
+
+// These are the datasets that are available on our dev instance
+const datasetNames = [
+  "ROI2017_wkw",
+  "Cortex_knossos",
+  "2017-05-31_mSEM_aniso-test",
+  "e2006_knossos",
+  "confocal-multi_knossos",
+  "fluro-rgb_knossos",
+];
+
+datasetNames.map(async datasetName => {
+  test(`it should render dataset ${datasetName} correctly`, async t => {
+    const { screenshot, width, height } = await screenshotDataset(
+      await getNewPage(t.context.browser),
+      URL,
+      datasetName,
+    );
+    const changedPixels = await compareScreenshot(
+      screenshot,
+      width,
+      height,
+      BASE_PATH,
+      datasetName,
+    );
+
+    t.is(
+      changedPixels,
+      0,
+      `Dataset with name: "${datasetName}" does not look the same, see ${datasetName}.diff.png for the difference and ${datasetName}.new.png for the new screenshot.`,
+    );
+  });
+});
+
+test.afterEach(async t => {
+  await t.context.browser.close();
+});

app/assets/javascripts/test/puppeteer/dataset_rendering_helpers.js

@@ -0,0 +1,58 @@
+/* eslint import/no-extraneous-dependencies: ["error", {"peerDependencies": true}], no-await-in-loop: 0 */
+const urljoin = require("url-join");
+const fetch = require("node-fetch");
+const pixelmatch = require("pixelmatch");
+
+const DEV_AUTH_TOKEN = "secretScmBoyToken";
+
+async function createExplorational(datasetName, typ, withFallback, baseUrl) {
+  const fullUrl = urljoin(baseUrl, `/api/datasets/${datasetName}/createExplorational`);
+  return (await fetch(fullUrl, {
+    body: JSON.stringify({ typ, withFallback }),
+    method: "POST",
+    headers: {
+      "X-Auth-Token": DEV_AUTH_TOKEN,
+      "Content-Type": "application/json",
+    },
+  })).json();
+}
+
+async function screenshotDataset(page, baseUrl, datasetName) {
+  const createdExplorational = await createExplorational(datasetName, "skeleton", false, baseUrl);
+  return openTracingViewAndScreenshot(page, baseUrl, createdExplorational.id);
+}
+
+async function openTracingViewAndScreenshot(page, baseUrl, annotationId) {
+  await page.goto(urljoin(baseUrl, `/annotations/Explorational/${annotationId}`), {
+    timeout: 0,
+  });
+
+  let canvas;
+  while (canvas == null) {
+    canvas = await page.$("#render-canvas");
+    await page.waitFor(500);
+  }
+  const { width, height } = await canvas.boundingBox();
+
+  let currentShot;
+  let lastShot = await canvas.screenshot();
+  let changedPixels = Infinity;
+  // If the screenshot didn't change in the last x seconds, we're probably done
+  while (currentShot == null || changedPixels > 0) {
+    await page.waitFor(10000);
+    currentShot = await canvas.screenshot();
+    if (lastShot != null) {
+      changedPixels = pixelmatch(lastShot, currentShot, {}, width, height, {
+        threshold: 0.0,
+      });
+    }
+    lastShot = currentShot;
+  }
+
+  return { screenshot: currentShot, width, height };
+}
+
+module.exports = {
+  screenshotDataset,
+  DEV_AUTH_TOKEN,
+};

app/assets/javascripts/test/puppeteer/screenshot_helpers.js

@@ -0,0 +1,73 @@
+/* eslint import/no-extraneous-dependencies: ["error", {"peerDependencies": true}] */
+const pixelmatch = require("pixelmatch");
+const { PNG } = require("pngjs");
+const fs = require("fs");
+
+function openScreenshot(path, name) {
+  return new Promise(resolve => {
+    fs
+      .createReadStream(`${path}/${name}.png`)
+      .on("error", error => {
+        if (error.code === "ENOENT") {
+          resolve(null);
+        } else {
+          throw error;
+        }
+      })
+      .pipe(new PNG())
+      .on("parsed", function() {
+        resolve(this);
+      });
+  });
+}
+
+function saveScreenshot(png, path, name) {
+  return new Promise(resolve => {
+    png
+      .pack()
+      .pipe(fs.createWriteStream(`${path}/${name}.png`))
+      .on("finish", () => resolve());
+  });
+}
+
+function bufferToPng(buffer, width, height) {
+  return new Promise(resolve => {
+    const png = new PNG({ width, height });
+    png.parse(buffer, () => resolve(png));
+  });
+}
+
+async function compareScreenshot(screenshotBuffer, width, height, path, name) {
+  const [newScreenshot, existingScreenshot] = await Promise.all([
+    bufferToPng(screenshotBuffer, width, height),
+    openScreenshot(path, name),
+  ]);
+  if (existingScreenshot == null) {
+    // If there is no existing screenshot, save the current one
+    await saveScreenshot(newScreenshot, path, name);
+    return 0;
+  }
+
+  const diff = new PNG({ width, height });
+  const pixelErrors = pixelmatch(
+    existingScreenshot.data,
+    newScreenshot.data,
+    diff.data,
+    existingScreenshot.width,
+    existingScreenshot.height,
+    { threshold: 0.0 },
+  );
+
+  if (pixelErrors > 0) {
+    // If the screenshots are not equal, save the diff and the new screenshot
+    await Promise.all([
+      saveScreenshot(diff, path, `${name}.diff`),
+      saveScreenshot(newScreenshot, path, `${name}.new`),
+    ]);
+  }
+  return pixelErrors;
+}
+
+module.exports = {
+  compareScreenshot,
+};

app/controllers/InitialDataController.scala

@@ -1,5 +1,6 @@
 package controllers
 
+import com.mohiva.play.silhouette.api.LoginInfo
 import com.scalableminds.util.accesscontext.GlobalAccessContext
 import com.scalableminds.util.security.SCrypt
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
@@ -11,6 +12,8 @@ import models.task.{TaskType, TaskTypeDAO}
 import models.team._
 import models.user.{User, UserDAO, UserService}
 import net.liftweb.common.Full
+import org.joda.time.DateTime
+import oxalis.security.{TokenSQL, TokenSQLDAO, TokenType}
 import play.api.i18n.MessagesApi
 import play.api.Play.current
 import oxalis.security.WebknossosSilhouette.UserAwareAction
@@ -54,6 +57,7 @@ Samplecountry
       _ <- insertOrganization
       _ <- insertTeams
       _ <- insertDefaultUser
+      _ <- insertToken
       _ <- insertTaskType
       _ <- insertProject
       _ <- insertLocalDataStoreIfEnabled
@@ -94,6 +98,24 @@ Samplecountry
     }.toFox
   }
 
+  def insertToken = {
+    val expiryTime = Play.configuration.underlying.getDuration("silhouette.tokenAuthenticator.authenticatorExpiry").toMillis
+    TokenSQLDAO.findOneByLoginInfo("credentials", defaultUserEmail, TokenType.Authentication).futureBox.flatMap {
+      case Full(_) => Fox.successful(())
+      case _ =>
+        val newToken = TokenSQL(
+          ObjectId.generate,
+          "secretScmBoyToken",
+          LoginInfo("credentials", defaultUserEmail),
+          new DateTime(System.currentTimeMillis()),
+          new DateTime(System.currentTimeMillis() + expiryTime),
+          None,
+          TokenType.Authentication
+        )
+      TokenSQLDAO.insertOne(newToken)
+    }
+  }
+
   def insertOrganization = {
     OrganizationDAO.findOneByName(defaultOrganization.name).futureBox.flatMap {
       case Full(_) => Fox.successful(())

clean

@@ -2,6 +2,7 @@
 
 echo "[info] Cleaning target directories..."
 rm -rf target
+rm -rf util/target
 rm -rf project/target
 rm -rf project/project/target
 rm -rf webknossos-datastore/target

docker-compose.yml

@@ -130,6 +130,16 @@ services:
           -Ddatastore.fossildb.address=fossildb
           -Dpostgres.url=$${POSTGRES_URL}"
 
+  screenshot-tests:
+    image: scalableminds/puppeteer:master
+    environment:
+      - URL
+    working_dir: /home/pptruser/webknossos
+    command: bash -c 'for i in {1..3}; do yarn test-screenshot && break; done'
+    volumes:
+      - ".:/home/pptruser/webknossos"
+    user: ${USER_UID:-1000}:${USER_GID:-1000}
+
   postgres:
     image: postgres:10-alpine
     environment:

package.json

@@ -48,15 +48,18 @@
     "less-loader": "^4.1.0",
     "lint-staged": "^7.1.2",
     "mock-require": "^1.2.1",
-    "node-fetch": "^1.7.2",
-    "pngjs": "latest",
+    "node-fetch": "^2.1.2",
+    "pixelmatch": "^4.0.2",
+    "pngjs": "^3.3.3",
     "prettier": "1.11.1",
     "proto-loader6": "^0.4.0",
+    "puppeteer": "^1.5.0",
     "react-test-renderer": "^16.2.0",
     "redux-mock-store": "^1.2.2",
     "sinon": "^1.17.3",
     "style-loader": "^0.20.2",
     "uglifyjs-webpack-plugin": "^1.2.2",
+    "url-join": "^4.0.0",
     "url-loader": "^1.0.1",
     "webpack": "^4.7.0",
     "webpack-cli": "^2.1.3"
@@ -76,6 +79,7 @@
     "test-prepare": "tools/test.sh prepare",
     "test-prepare-watch": "tools/test.sh prepare -w",
     "test-e2e": "tools/postgres/prepareTestDB.sh && tools/test.sh test-e2e --timeout=60s --verbose",
+    "test-screenshot": "ava app/assets/javascripts/test/puppeteer/*.screenshot.js --timeout=5m",
     "test-help": "echo For development it is recommended to run yarn test-prepare-watch in one terminal and yarn test-watch in another. This is the fastest way to perform incremental testing. If you are only interested in running test once, use yarn test.",
     "remove-snapshots": "rm -r app/assets/javascripts/test/snapshots/public",
     "create-snapshots": "rm -r app/assets/javascripts/test/snapshots/public && docker-compose up e2e-tests",