diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 230c9035f..e98c93bcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,9 +74,15 @@ jobs: $f || exit 1 done + - name: Run GUI tests + run: ./dockerfiles/run-gui-tests.sh + - name: Clean up the database run: docker-compose down --volumes + steps: + - uses: actions/checkout@master + build_tests: runs-on: ubuntu-latest needs: build diff --git a/docker-compose.yml b/docker-compose.yml index b5124bc0c..e562c3de4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,6 +89,14 @@ services: timeout: 5s retries: 10 + gui_tests: + build: + context: . + dockerfile: ./dockerfiles/Dockerfile-gui-tests + network_mode: "host" + volumes: + - "${PWD}:/build/out" + volumes: postgres-data: {} minio-data: {} diff --git a/dockerfiles/Dockerfile-gui-tests b/dockerfiles/Dockerfile-gui-tests new file mode 100644 index 000000000..cd157c2ce --- /dev/null +++ b/dockerfiles/Dockerfile-gui-tests @@ -0,0 +1,77 @@ +FROM ubuntu:16.04 as build +SHELL ["/bin/bash", "-c"] + +RUN apt-get update && apt-get install -y \ + ca-certificates \ + curl \ + docker.io \ + gcc \ + git \ + libssl-dev \ + pkg-config \ + xz-utils + +# Install dependencies for chromium browser +RUN apt-get install -y \ + gconf-service \ + libasound2 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libc6 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libexpat1 \ + libfontconfig1 \ + libgbm-dev \ + libgcc1 \ + libgconf-2-4 \ + libgdk-pixbuf2.0-0 \ + libglib2.0-0 \ + libgtk-3-0 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libstdc++6 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + fonts-liberation \ + libappindicator1 \ + libnss3 \ + lsb-release \ + xdg-utils \ + wget + +# Install rust +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y --default-toolchain nightly --no-modify-path --profile minimal +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN curl -sL https://nodejs.org/dist/v14.4.0/node-v14.4.0-linux-x64.tar.xz | tar -xJ +ENV PATH="/node-v14.4.0-linux-x64/bin:${PATH}" +ENV NODE_PATH="/node-v14.4.0-linux-x64/lib/node_modules/" + +WORKDIR /build + +RUN mkdir out + +# For now, we need to use `--unsafe-perm=true` to go around an issue when npm tries +# to create a new folder. For reference: +# https://github.com/puppeteer/puppeteer/issues/375 +# +# We also specify the version in case we need to update it to go around cache limitations. +RUN npm install -g browser-ui-test@0.8.5 --unsafe-perm=true + +EXPOSE 3000 + +CMD ["node", "/build/out/gui-tests/tester.js"] diff --git a/dockerfiles/run-gui-tests.sh b/dockerfiles/run-gui-tests.sh new file mode 100755 index 000000000..81aae4165 --- /dev/null +++ b/dockerfiles/run-gui-tests.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Just in case it's running, we stop the web server. +docker-compose stop web + +docker-compose up -d db s3 + +# We add the information we need. +cargo run -- database migrate +cargo run -- build crate sysinfo 0.23.4 +cargo run -- build crate sysinfo 0.23.5 +cargo run -- build add-essential-files + +# In case we don't have a `.env`, we create one. +if [ ! -f .env ] +then +cp .env.sample .env +source .env +fi + +cargo run -- start-web-server & +SERVER_PID=$! + +docker build . -f dockerfiles/Dockerfile-gui-tests -t gui_tests + +echo "Sleeping a bit to be sure the web server will be started..." +sleep 5 + +# status="docker run . -v `pwd`:/build/out:ro gui_tests" +docker-compose run gui_tests +status=$? +kill -9 $SERVER_PID +exit $status diff --git a/gui-tests/404.goml b/gui-tests/404.goml new file mode 100644 index 000000000..d71da15a5 --- /dev/null +++ b/gui-tests/404.goml @@ -0,0 +1,3 @@ +// Checks the content of the 404 page. +goto: |DOC_PATH|/non-existing-crate +assert-text: ("#crate-title", "The requested crate does not exist") diff --git a/gui-tests/basic.goml b/gui-tests/basic.goml new file mode 100644 index 000000000..787df11f8 --- /dev/null +++ b/gui-tests/basic.goml @@ -0,0 +1,12 @@ +// Checks that the "latest" URL leads us to the last version of the `sysinfo` crate. +goto: |DOC_PATH|/sysinfo +// We first check if the redirection worked as expected: +assert-document-property: ({"URL": "/sysinfo/latest/sysinfo/"}, ENDS_WITH) +assert: "//*[@class='title' and text()='sysinfo-0.23.5']" +// And we also confirm we're on a rustdoc page. +assert: "#rustdoc_body_wrapper" + +// Let's go to the docs.rs page of the crate. +goto: |DOC_PATH|/crate/sysinfo/latest +assert-false: "#rustdoc_body_wrapper" +assert-text: ("#crate-title", "sysinfo 0.23.5", CONTAINS) diff --git a/gui-tests/tester.js b/gui-tests/tester.js new file mode 100644 index 000000000..34bd77d7c --- /dev/null +++ b/gui-tests/tester.js @@ -0,0 +1,252 @@ +// This package needs to be install: +// +// ``` +// npm install browser-ui-test +// ``` + +const fs = require("fs"); +const path = require("path"); +const os = require('os'); +const {Options, runTest} = require('browser-ui-test'); + +function showHelp() { + console.log("docs-rs-gui-js options:"); + console.log(" --file [PATH] : file to run (can be repeated)"); + console.log(" --debug : show extra information about script run"); + console.log(" --show-text : render font in pages"); + console.log(" --no-headless : disable headless mode"); + console.log(" --help : show this message then quit"); + console.log(" --jobs [NUMBER] : number of threads to run tests on"); +} + +function isNumeric(s) { + return /^\d+$/.test(s); +} + +function parseOptions(args) { + var opts = { + "files": [], + "debug": false, + "show_text": false, + "no_headless": false, + "jobs": -1, + }; + var correspondances = { + "--debug": "debug", + "--show-text": "show_text", + "--no-headless": "no_headless", + }; + + for (var i = 0; i < args.length; ++i) { + if (args[i] === "--file" + || args[i] === "--jobs") { + i += 1; + if (i >= args.length) { + console.log("Missing argument after `" + args[i - 1] + "` option."); + return null; + } + if (args[i - 1] === "--jobs") { + if (!isNumeric(args[i])) { + console.log( + "`--jobs` option expects a positive number, found `" + args[i] + "`"); + return null; + } + opts["jobs"] = parseInt(args[i]); + } else if (args[i - 1] !== "--file") { + opts[correspondances[args[i - 1]]] = args[i]; + } else { + opts["files"].push(args[i]); + } + } else if (args[i] === "--help") { + showHelp(); + process.exit(0); + } else if (correspondances[args[i]]) { + opts[correspondances[args[i]]] = true; + } else { + console.log("Unknown option `" + args[i] + "`."); + console.log("Use `--help` to see the list of options"); + return null; + } + } + return opts; +} + +/// Print single char status information without \n +function char_printer(n_tests) { + const max_per_line = 10; + let current = 0; + return { + successful: function() { + current += 1; + if (current % max_per_line === 0) { + process.stdout.write(`. (${current}/${n_tests})${os.EOL}`); + } else { + process.stdout.write("."); + } + }, + erroneous: function() { + current += 1; + if (current % max_per_line === 0) { + process.stderr.write(`F (${current}/${n_tests})${os.EOL}`); + } else { + process.stderr.write("F"); + } + }, + finish: function() { + if (current % max_per_line === 0) { + // Don't output if we are already at a matching line end + console.log(""); + } else { + const spaces = " ".repeat(max_per_line - (current % max_per_line)); + process.stdout.write(`${spaces} (${current}/${n_tests})${os.EOL}${os.EOL}`); + } + }, + }; +} + +/// Sort array by .file_name property +function by_filename(a, b) { + return a.file_name - b.file_name; +} + +async function main(argv) { + let opts = parseOptions(argv.slice(2)); + if (opts === null) { + process.exit(1); + } + + // Print successful tests too + let debug = false; + // Run tests in sequentially + let headless = true; + const options = new Options(); + try { + // This is more convenient that setting fields one by one. + let args = [ + "--no-screenshot-comparison", + "--no-sandbox", + ]; + if (typeof process.env.SERVER_URL !== 'undefined') { + args.push("--variable", "DOC_PATH", process.env.SERVER_URL); + } else { + args.push("--variable", "DOC_PATH", "http://127.0.0.1:3000"); + } + if (opts["debug"]) { + debug = true; + args.push("--debug"); + } + if (opts["show_text"]) { + args.push("--show-text"); + } + if (opts["no_headless"]) { + args.push("--no-headless"); + headless = false; + } + options.parseArguments(args); + } catch (error) { + console.error(`invalid argument: ${error}`); + process.exit(1); + } + + let failed = false; + let files; + if (opts["files"].length === 0) { + files = fs.readdirSync(__dirname); + } else { + files = opts["files"]; + } + files = files.filter(file => path.extname(file) == ".goml"); + if (files.length === 0) { + console.error("No test selected"); + process.exit(2); + } + files.sort(); + + if (!headless) { + opts["jobs"] = 1; + console.log("`--no-headless` option is active, disabling concurrency for running tests."); + } + + console.log(`Running ${files.length} docs.rs GUI (${opts["jobs"]} concurrently) ...`); + + if (opts["jobs"] < 1) { + process.setMaxListeners(files.length + 1); + } else if (headless) { + process.setMaxListeners(opts["jobs"] + 1); + } + + const tests_queue = []; + let results = { + successful: [], + failed: [], + errored: [], + }; + const status_bar = char_printer(files.length); + for (let i = 0; i < files.length; ++i) { + const file_name = files[i]; + const testPath = path.join(__dirname, file_name); + const callback = runTest(testPath, options) + .then(out => { + const [output, nb_failures] = out; + results[nb_failures === 0 ? "successful" : "failed"].push({ + file_name: testPath, + output: output, + }); + if (nb_failures > 0) { + status_bar.erroneous(); + failed = true; + } else { + status_bar.successful(); + } + }) + .catch(err => { + results.errored.push({ + file_name: testPath + file_name, + output: err, + }); + status_bar.erroneous(); + failed = true; + }) + .finally(() => { + // We now remove the promise from the tests_queue. + tests_queue.splice(tests_queue.indexOf(callback), 1); + }); + tests_queue.push(callback); + if (opts["jobs"] > 0 && tests_queue.length >= opts["jobs"]) { + await Promise.race(tests_queue); + } + } + if (tests_queue.length > 0) { + await Promise.all(tests_queue); + } + status_bar.finish(); + + if (debug) { + results.successful.sort(by_filename); + results.successful.forEach(r => { + console.log(r.output); + }); + } + + if (results.failed.length > 0) { + console.log(""); + results.failed.sort(by_filename); + results.failed.forEach(r => { + console.log(r.file_name, r.output); + }); + } + if (results.errored.length > 0) { + console.log(os.EOL); + // print run errors on the bottom so developers see them better + results.errored.sort(by_filename); + results.errored.forEach(r => { + console.error(r.file_name, r.output); + }); + } + + if (failed) { + process.exit(1); + } +} + +main(process.argv);