diff --git a/.travis.yml b/.travis.yml
index 1ae6e415c9c..36dae5b7a35 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,8 +26,14 @@ script:
   - coverage erase
   - python setup.py clean
   - python setup.py build_ext --inplace
-  - coverage run --append --include=PIL/* selftest.py
-  - python Tests/run.py --coverage
+
+    # Don't cover PyPy: it fails intermittently and is x5.8 slower (#640)
+  - if [ "$TRAVIS_PYTHON_VERSION" == "pypy" ]; then python selftest.py; fi
+  - if [ "$TRAVIS_PYTHON_VERSION" == "pypy" ]; then python Tests/run.py; fi
+
+    # Cover the others
+  - if [ "$TRAVIS_PYTHON_VERSION" != "pypy" ]; then coverage run --append --include=PIL/* selftest.py; fi
+  - if [ "$TRAVIS_PYTHON_VERSION" != "pypy" ]; then python Tests/run.py --coverage; fi
 
 after_success:
   - coverage report
@@ -37,7 +43,3 @@ after_success:
   - pyflakes PIL/*.py
   - pep8 Tests/*.py
   - pyflakes Tests/*.py
-
-matrix:
-  allow_failures:
-    - python: "pypy"