-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Screenshot test tracing view (#2858)
* add simple test that screenshots the tracing view using puppeteer, not running under docker yet (#2822) * use puppeteer docker image for screenshot tests, add test file that tests dataset on dev server and does pixel diffs with existing screenshots (#2822) * revert some earlier changes (#2822) * fix linting (#2822) * apply PR feedback (#2822) * add retry of 3 for screenshot tests (#2822) * parallelize some awaits (#2822) * insert x-auth-token with predefined value for scmboy * remove puppeteer docker file from this repo, use pre-built docker image instead * replace dev auth token for screenshot tests * change BRANCH env variable to URL
- Loading branch information
1 parent
305eac5
commit d8fae1a
Showing
15 changed files
with
374 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
78 changes: 78 additions & 0 deletions
78
app/assets/javascripts/test/puppeteer/dataset_rendering.screenshot.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
/* eslint import/no-extraneous-dependencies: ["error", {"peerDependencies": true}] */ | ||
const test = require("ava"); | ||
const puppeteer = require("puppeteer"); | ||
const path = require("path"); | ||
const { screenshotDataset, DEV_AUTH_TOKEN } = require("./dataset_rendering_helpers"); | ||
const { compareScreenshot } = require("./screenshot_helpers"); | ||
|
||
process.on("unhandledRejection", (err, promise) => { | ||
console.error("Unhandled rejection (promise: ", promise, ", reason: ", err, ")."); | ||
}); | ||
|
||
const BASE_PATH = path.join(__dirname, "../screenshots"); | ||
|
||
let URL = "https://master.webknossos.xyz"; | ||
if (!process.env.URL) { | ||
console.warn( | ||
"[Warning] No url specified, assuming dev master. If you want to specify a URL, prepend URL=<url> to the command.", | ||
); | ||
} else { | ||
URL = process.env.URL; | ||
// Prepend https:// if not specified | ||
if (!/^https?:\/\//i.test(URL)) { | ||
URL = `https://${URL}`; | ||
} | ||
} | ||
console.log(`[Info] Executing tests on URL ${URL}.`); | ||
|
||
async function getNewPage(browser) { | ||
const page = await browser.newPage(); | ||
page.setViewport({ width: 1920, height: 1080 }); | ||
page.setExtraHTTPHeaders({ | ||
"X-Auth-Token": DEV_AUTH_TOKEN, | ||
}); | ||
return page; | ||
} | ||
|
||
test.beforeEach(async t => { | ||
t.context.browser = await puppeteer.launch({ | ||
args: ["--headless", "--hide-scrollbars", "--no-sandbox", "--disable-setuid-sandbox"], | ||
}); | ||
}); | ||
|
||
// These are the datasets that are available on our dev instance | ||
const datasetNames = [ | ||
"ROI2017_wkw", | ||
"Cortex_knossos", | ||
"2017-05-31_mSEM_aniso-test", | ||
"e2006_knossos", | ||
"confocal-multi_knossos", | ||
"fluro-rgb_knossos", | ||
]; | ||
|
||
datasetNames.map(async datasetName => { | ||
test(`it should render dataset ${datasetName} correctly`, async t => { | ||
const { screenshot, width, height } = await screenshotDataset( | ||
await getNewPage(t.context.browser), | ||
URL, | ||
datasetName, | ||
); | ||
const changedPixels = await compareScreenshot( | ||
screenshot, | ||
width, | ||
height, | ||
BASE_PATH, | ||
datasetName, | ||
); | ||
|
||
t.is( | ||
changedPixels, | ||
0, | ||
`Dataset with name: "${datasetName}" does not look the same, see ${datasetName}.diff.png for the difference and ${datasetName}.new.png for the new screenshot.`, | ||
); | ||
}); | ||
}); | ||
|
||
test.afterEach(async t => { | ||
await t.context.browser.close(); | ||
}); |
58 changes: 58 additions & 0 deletions
58
app/assets/javascripts/test/puppeteer/dataset_rendering_helpers.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
/* eslint import/no-extraneous-dependencies: ["error", {"peerDependencies": true}], no-await-in-loop: 0 */ | ||
const urljoin = require("url-join"); | ||
const fetch = require("node-fetch"); | ||
const pixelmatch = require("pixelmatch"); | ||
|
||
const DEV_AUTH_TOKEN = "secretScmBoyToken"; | ||
|
||
async function createExplorational(datasetName, typ, withFallback, baseUrl) { | ||
const fullUrl = urljoin(baseUrl, `/api/datasets/${datasetName}/createExplorational`); | ||
return (await fetch(fullUrl, { | ||
body: JSON.stringify({ typ, withFallback }), | ||
method: "POST", | ||
headers: { | ||
"X-Auth-Token": DEV_AUTH_TOKEN, | ||
"Content-Type": "application/json", | ||
}, | ||
})).json(); | ||
} | ||
|
||
async function screenshotDataset(page, baseUrl, datasetName) { | ||
const createdExplorational = await createExplorational(datasetName, "skeleton", false, baseUrl); | ||
return openTracingViewAndScreenshot(page, baseUrl, createdExplorational.id); | ||
} | ||
|
||
async function openTracingViewAndScreenshot(page, baseUrl, annotationId) { | ||
await page.goto(urljoin(baseUrl, `/annotations/Explorational/${annotationId}`), { | ||
timeout: 0, | ||
}); | ||
|
||
let canvas; | ||
while (canvas == null) { | ||
canvas = await page.$("#render-canvas"); | ||
await page.waitFor(500); | ||
} | ||
const { width, height } = await canvas.boundingBox(); | ||
|
||
let currentShot; | ||
let lastShot = await canvas.screenshot(); | ||
let changedPixels = Infinity; | ||
// If the screenshot didn't change in the last x seconds, we're probably done | ||
while (currentShot == null || changedPixels > 0) { | ||
await page.waitFor(10000); | ||
currentShot = await canvas.screenshot(); | ||
if (lastShot != null) { | ||
changedPixels = pixelmatch(lastShot, currentShot, {}, width, height, { | ||
threshold: 0.0, | ||
}); | ||
} | ||
lastShot = currentShot; | ||
} | ||
|
||
return { screenshot: currentShot, width, height }; | ||
} | ||
|
||
module.exports = { | ||
screenshotDataset, | ||
DEV_AUTH_TOKEN, | ||
}; |
73 changes: 73 additions & 0 deletions
73
app/assets/javascripts/test/puppeteer/screenshot_helpers.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,73 @@ | ||
/* eslint import/no-extraneous-dependencies: ["error", {"peerDependencies": true}] */ | ||
const pixelmatch = require("pixelmatch"); | ||
const { PNG } = require("pngjs"); | ||
const fs = require("fs"); | ||
|
||
function openScreenshot(path, name) { | ||
return new Promise(resolve => { | ||
fs | ||
.createReadStream(`${path}/${name}.png`) | ||
.on("error", error => { | ||
if (error.code === "ENOENT") { | ||
resolve(null); | ||
} else { | ||
throw error; | ||
} | ||
}) | ||
.pipe(new PNG()) | ||
.on("parsed", function() { | ||
resolve(this); | ||
}); | ||
}); | ||
} | ||
|
||
function saveScreenshot(png, path, name) { | ||
return new Promise(resolve => { | ||
png | ||
.pack() | ||
.pipe(fs.createWriteStream(`${path}/${name}.png`)) | ||
.on("finish", () => resolve()); | ||
}); | ||
} | ||
|
||
function bufferToPng(buffer, width, height) { | ||
return new Promise(resolve => { | ||
const png = new PNG({ width, height }); | ||
png.parse(buffer, () => resolve(png)); | ||
}); | ||
} | ||
|
||
async function compareScreenshot(screenshotBuffer, width, height, path, name) { | ||
const [newScreenshot, existingScreenshot] = await Promise.all([ | ||
bufferToPng(screenshotBuffer, width, height), | ||
openScreenshot(path, name), | ||
]); | ||
if (existingScreenshot == null) { | ||
// If there is no existing screenshot, save the current one | ||
await saveScreenshot(newScreenshot, path, name); | ||
return 0; | ||
} | ||
|
||
const diff = new PNG({ width, height }); | ||
const pixelErrors = pixelmatch( | ||
existingScreenshot.data, | ||
newScreenshot.data, | ||
diff.data, | ||
existingScreenshot.width, | ||
existingScreenshot.height, | ||
{ threshold: 0.0 }, | ||
); | ||
|
||
if (pixelErrors > 0) { | ||
// If the screenshots are not equal, save the diff and the new screenshot | ||
await Promise.all([ | ||
saveScreenshot(diff, path, `${name}.diff`), | ||
saveScreenshot(newScreenshot, path, `${name}.new`), | ||
]); | ||
} | ||
return pixelErrors; | ||
} | ||
|
||
module.exports = { | ||
compareScreenshot, | ||
}; |
Binary file added
BIN
+1.13 MB
app/assets/javascripts/test/screenshots/2017-05-31_mSEM_aniso-test.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.