feat: Docker integration into Travis to decrease false negative scree…

…nshots (#91)
material-components · Jun 5, 2018 · d8b670e · d8b670e
1 parent c16eed6
commit d8b670e
Show file tree

Hide file tree

Showing 5 changed files with 169 additions and 22 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -27,9 +27,13 @@ matrix:
       script: npm run lint
 
     - node_js: 8
-      env: TEST_SUITE=screenshots
-      install: npm i
+      services:
+        - docker
+      env:
+        - TEST_SUITE=screenshots
+        - CURRENT_BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo $TRAVIS_PULL_REQUEST_BRANCH; fi)
+      before_install:
+        - docker pull mdcreact/screenshots
+        - docker image ls
       script:
-        - ./test/screenshot/start.sh
-        - sleep 200s
-        - npm run test:image-diff
+        - docker run -it --rm --cap-add=SYS_ADMIN -e MDC_GCLOUD_SERVICE_ACCOUNT_KEY="${MDC_GCLOUD_SERVICE_ACCOUNT_KEY}" mdcreact/screenshots /bin/sh -c "git fetch --all; git checkout \"${CURRENT_BRANCH}\"; npm i; /home/pptruser/material-components-web-react/test/screenshot/start.sh; sleep 200s; npm run test:image-diff"
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,48 @@
+# based off https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-puppeteer-in-docker
+FROM node:8-slim
+
+# See https://crbug.com/795759
+RUN apt-get update && apt-get install -yq libgconf-2-4 git
+
+# Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others)
+# Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer
+# installs, work.
+RUN apt-get update && apt-get install -y wget --no-install-recommends \
+    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+    && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+    && apt-get update \
+    && apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \
+      --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get purge --auto-remove -y curl \
+    && rm -rf /src/*.deb
+
+# It's a good idea to use dumb-init to help prevent zombie chrome processes.
+# ADD https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 /usr/local/bin/dumb-init
+# RUN chmod +x /usr/local/bin/dumb-init
+
+# Uncomment to skip the chromium download when installing puppeteer. If you do,
+# you'll need to launch puppeteer with:
+#     browser.launch({executablePath: 'google-chrome-unstable'})
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+
+# Install puppeteer so it's available in the container.
+RUN npm i -g npm
+RUN npm i -g lerna
+RUN npm i puppeteer
+
+# Add user so we don't need --no-sandbox.
+RUN groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \
+    && mkdir -p /home/pptruser/Downloads \
+    && chown -R pptruser:pptruser /home/pptruser \
+    && chown -R pptruser:pptruser /node_modules
+
+# Run everything after as non-privileged user.
+USER pptruser
+
+RUN cd /home/pptruser/ \
+  && git clone https://github.com/material-components/material-components-web-react.git \
+  && cd material-components-web-react \
+  && npm i
+
+WORKDIR /home/pptruser/material-components-web-react
diff --git a/docs/screenshot-tests.md b/docs/screenshot-tests.md
@@ -0,0 +1,95 @@
+# Screenshot Testing
+
+Screenshot testing is an important ingredient in building and maintaining the MDC React Library. This is a how-to guide on our home grown solution to screenshot testing. We use [resemblejs](https://www.npmjs.com/package/resemblejs) to perform image diffs, and [puppeteer](https://github.com/GoogleChrome/puppeteer) to collect the screenshots.
+
+## Tech Stack
+
+- [resemblejs](https://www.npmjs.com/package/resemblejs)
+- [mocha](https://github.com/mochajs/mocha)
+- [puppeteer](https://github.com/GoogleChrome/puppeteer)
+
+## Running Tests Locally
+
+> If this is your first time using Docker, you may find it helpful to sift through the [Getting Started Guide](https://docs.docker.com/get-started/#images-and-containers). This [3rd Party Guide](http://odewahn.github.io/docker-jumpstart/building-images-with-dockerfiles.html) is also a helpful quick reference for some terms.
+
+You should have docker already installed on your machine. Please follow [instructions](https://docs.docker.com/install/) to install.
+
+#### STEP 1: Build pull the Docker Image
+
+If you already pulled this image before you can skip this step. If there has been a change to the `Dockerfile`, please follow [these steps](#building-new-docker-image).
+
+```
+docker pull mdcreact/screenshots
+```
+
+
+#### STEP 2: Run a Docker Container
+
+Ensure that you have a `MDC_GCLOUD_SERVICE_ACCOUNT_KEY` environment variable (ie. ~/.bashrc, ~/.bash_profile). Otherwise you will need to paste it inline in the command below. You can get this off of the Google Cloud Platform Admin Tool (3rd party contributors will not be able to download/upload screenshots until this is part of the process is changed).
+
+```
+docker run -it --rm --cap-add=SYS_ADMIN -e MDC_GCLOUD_SERVICE_ACCOUNT_KEY="${MDC_GCLOUD_SERVICE_ACCOUNT_KEY}" mdcreact/screenshots /bin/sh
+```
+
+This command opens a container based from the `mdcreact/screenshots` image you created in Step 1.
+
+##### Breaking it down
+
+Option | Description
+--- | ---
+`-it` | Allow CLI interaction with the container
+`--rm` | When the container is closed, remove it from memory
+`--cap-add=SYS_ADMIN` | Run container with system admin privileges
+`-e MDC_...="${MDC_...}"` | Set MDC_GCLOUD_SERVICE_ACCOUNT_KEY env variable
+
+#### STEP 3: Run Tests
+
+At this point you are ready to run. This Docker container is mainly designed for running the screenshot tests. The container already has 1) cloned the repo and 2) `npm install` the project dependencies.
+
+The next thing you need to do is start the server and run the tests.
+
+```
+git checkout <INSERT_YOUR_BRANCH_NAME>
+npm install
+./test/screenshot/start.sh
+```
+
+Wait for about 2 - 3 minutes for the server to start up (we know the long wait is an issue). The `start.sh` script is a proxy to `npm start`, but doesn't output to the terminal so you can run other commands.
+
+```
+npm run test:image-diff
+```
+
+This will kick off the screenshot tests. You will see the regular terminal output from mocha.
+
+## Troubleshooting
+
+### Building New Docker Image
+
+You may need to update the Docker image. Follow these steps:
+
+From the project's root directory run the following command:
+
+```
+docker build -t screenshots .
+```
+
+This command builds a Docker image named `screenshots` based off the `Dockerfile` found in the root directory. Next push this to the Docker Hub for Travis tests. If you are testing you may want to use a different [tag](https://hub.docker.com/r/mdcreact/screenshots/tags/) in the interim.
+
+```
+docker images
+```
+
+This will output a list of your local images. You will want to grab the most recent `IMAGE_ID` with the `REPOSITORY` name `screenshots` (the name you used in the `docker build` step). Next create the tag:
+
+```
+docker tag <IMAGE_ID> mdcreact/screenshots:<YOUR_TAG_NAME> # defaults to :latest
+```
+
+This will create the tag, which you can then push to Docker Hub:
+
+```
+docker push mdcreact/screenshots:<YOUR_TAG_NAME>
+```
+
+Now you're ready to share the image with the world!
diff --git a/test/screenshot/golden.json b/test/screenshot/golden.json
@@ -1,20 +1,20 @@
 {
-  "top-app-bar/short.html": "728c9a355129f68dc19b653593dabbc8538c65a356fc12ad78cd463e5a0f38f7",
-  "top-app-bar/prominent.html": "e637647594906569c0048195870bb6f7b2b370120a97d26cb99ed6b1254b2423",
-  "top-app-bar/standard.html": "7b2c178a191d2d405a8552644254974fc66739b30ad20193b0d1bc5546958d40",
-  "top-app-bar/standardNoActionItems.html": "cd62247a9bdd613adb405f99e58030faf2c17e26ba4365f84fd50b3d735cc702",
-  "top-app-bar/shortCollapsed.html": "d95df9e94851ffc70ccc38daed159e274209c54f3d1d18961f4512504defda6f",
-  "top-app-bar/standardWithNavigationIconElement.html": "5e789c63e175c3fa87551ff3ecfe65e66854ba1aee4d1d7ccb44fdffa0e622c7",
+  "top-app-bar/short.html": "fe50f5577ef8c9def95450090eae4033192034dd0e0890407d4ab80e04e3e6a1",
+  "top-app-bar/prominent.html": "34d03c5809067cc06a67dce1e03d3e1edaf306360479ac1955299d3eeffc6113",
+  "top-app-bar/standard.html": "90534d59d40f8c050aae022e94b0afa022a00d1e00c19e3f92dcc1908efcd831",
+  "top-app-bar/standardNoActionItems.html": "9102ece0efc0a040e0dc2b097c66d2ee5ecc9c91759aeb16ea5aa2baabe7307a",
+  "top-app-bar/shortCollapsed.html": "3565597194402f65bfc48edcea298a49c890c70100befe3ae19dd3cf24f17716",
+  "top-app-bar/standardWithNavigationIconElement.html": "91b10df9c4faf85ba63c84b1cf82a1c1143a3d8c4ba7ed8af477fabdc65ec25e",
   "material-icon/menu.html": "790b2076760612c6c91ee0f6bb42aa3603b30c2a60f302de22429c2572645640",
-  "fab/standard.html": "c0a68de7fa657ac62622e4f541c8e1eaef7544d12afb26b3421c3b04d58effa6",
-  "card/index.html": "523f29906da05a09cea03356bc0f5b5dfbecee4ea20db2236603953284cc10b3",
-  "button/index.html": "66a897d49f43d12844333b54370e7a5c441a9f5c563a8703f5e61251fdd6bde7",
-  "material-icon/index.html": "20c6c6bfe5ff444b52952343661151226a8786001fdd3e66e51e80903c3a9292",
+  "fab/standard.html": "71ec6ca75f5531667db8910bdf7bd9600184cbda70e0ecfda0a79c3a0c457270",
+  "card/index.html": "282326d3039d8ffddfa6a247fceb09a3114de7ea440fb8a9b1dd136ef4fbc598",
+  "button/index.html": "da830cff009a5f2b850a7dd29c63ae6fefdbe2bb726f02dbf87a8ce52619be9a",
+  "material-icon/index.html": "34abef2f26fd75d6d9021f455dc6acafc45943ab7515bb82d958302b92de6a98",
   "line-ripple/index.html": "4b8eca29eb61834715ca0eeaec2c620358fa943623b5235ca02717c351f1485b",
-  "floating-label/index.html": "67d9b6caad3a494563e2584ac6af071bbb3b73afc3e945f8299ca720b4015256",
+  "floating-label/index.html": "deb881d16e120610e700d467a2d89d1cb5796513ad7765635a390c9c8f4c2fbd",
   "notched-outline/index.html": "73394b6d7dcd94bc930dc740db8199000b94d365d7d703381ef01b3318b97d8c",
-  "text-field/icon/index.html": "ce761ca24e2d5ca425ad1783c9e2d476b84f65232d86c4fdc0e8c3e756311c8f",
-  "text-field/helper-text/index.html": "722a8eb27cbde5e32fc92c70e6c107f8ef75bff1abb3914fd2ef58fd49b678bb",
-  "text-field/index.html": "8cac264e77fe54efcc659edeb8ef1e530963f9efbe8a26edd2e54780beafcb71",
-  "top-app-bar/fixed.html": "7b2c178a191d2d405a8552644254974fc66739b30ad20193b0d1bc5546958d40"
+  "text-field/icon/index.html": "8a1d1969d6e7df5ea9def6ad1eb1734d5d3486c6bdda6c7721c494d8c46d37c3",
+  "text-field/helper-text/index.html": "c246d6f593fe01901d26973d7c7384232892cc44aada19e82c8709b82f1a20e1",
+  "text-field/index.html": "b8cc0a1fb0044d68269d8caa055ead35ae2de2a7d568fe65ea2e164ce67c72a0",
+  "top-app-bar/fixed.html": "90534d59d40f8c050aae022e94b0afa022a00d1e00c19e3f92dcc1908efcd831"
 }
diff --git a/test/screenshot/screenshot.js b/test/screenshot/screenshot.js
@@ -85,7 +85,7 @@ export default class Screenshot {
         this.saveImage_(diffPath, diff, metadata),
       ]);
 
-      return assert.isBelow(Number(data.misMatchPercentage), 0.021);
+      return assert.equal(Number(data.misMatchPercentage), 0);
     });
   }
 
@@ -199,7 +199,7 @@ export default class Screenshot {
    * @private
    */
   async takeScreenshot_() {
-    const browser = await puppeteer.launch();
+    const browser = await puppeteer.launch({executablePath: 'google-chrome-unstable'});
     const page = await browser.newPage();
     await page.goto(`http://localhost:8080/${this.urlPath_}`, {'waitUntil': 'networkidle2'});
     const imageBuffer = await page.screenshot({fullPage: true});