Merge pull request #173 from /issues/157

fixes #157 - remove PhantomJS from docker image, as it's broken

CHANGES.rst

@@ -4,8 +4,9 @@ Changelog
 Unreleased Changes
 ------------------
 
-* `Issue #170 <https://github.com/jantman/biweeklybudget/issues/170>`_ Upgrade **all** python dependencies to their latest versions.
-* `Issue #171 <https://github.com/jantman/biweeklybudget/issues/171>`_ Upgrade Docker base image from ``python:3.6.3-alpine3.4`` to ``python:3.6.4-alpine3.7``.
+* `Issue #170 <https://github.com/jantman/biweeklybudget/issues/170>`_ - Upgrade **all** python dependencies to their latest versions.
+* `Issue #171 <https://github.com/jantman/biweeklybudget/issues/171>`_ - Upgrade Docker base image from ``python:3.6.3-alpine3.4`` to ``python:3.6.4-alpine3.7``.
+* `Issue #157 <https://github.com/jantman/biweeklybudget/issues/157>`_ - Remove PhantomJS from Docker image, as it's broken and shouldn't be needed.
 
 0.7.0 (2018-01-07)
 ------------------

README.rst

@@ -90,7 +90,7 @@ Vault (if you choose to take advantage of the OFX downloading), which you can al
 * To use the automated OFX transaction downloading functionality:
 
   * A running, reachable instance of `Hashicorp Vault <https://www.vaultproject.io/>`_ with your financial institution web credentials stored in it.
-  * `PhantomJS <http://phantomjs.org/>`_ for downloading transaction data from institutions that do not support OFX remote access ("Direct Connect").
+  * If your bank does not support OFX remote access ("Direct Connect"), you will need to write a custom screen-scraper class using Selenium and a browser.
 
 Installation
 ------------

biweeklybudget/tests/docker_build.py

@@ -69,7 +69,6 @@
 from io import BytesIO
 import tarfile
 from biweeklybudget.version import VERSION
-from textwrap import dedent
 import subprocess
 
 FORMAT = "[%(asctime)s %(levelname)s] %(message)s"
@@ -114,19 +113,8 @@
         openssl-dev \
     && /app/bin/pip install {install} \
     && /app/bin/pip install gunicorn==19.7.1 \
-    && apk del .build-deps
-
-# install phantomjs per https://github.com/fgrehm/docker-phantomjs2
-RUN set -ex \
-    && apk add --no-cache --virtual .fetch-deps \
-        curl \
-    && mkdir -p /usr/share \
-    && cd /usr/share \
-    && curl -Ls https://github.com/fgrehm/docker-phantomjs2/releases/download/\
-v2.0.0-20150722/dockerized-phantomjs.tar.gz | tar xz -C / \
-    && ln -s /usr/local/bin/phantomjs /usr/bin/phantomjs \
-    && apk del .fetch-deps \
-    && phantomjs --version
+    && apk del .build-deps \
+    && rm -Rf /root/.cache
 
 RUN install -g root -o root -m 755 /tmp/entrypoint.sh /app/bin/entrypoint.sh
 
@@ -147,7 +135,6 @@
 class DockerImageBuilder(object):
 
     image_name = 'jantman/biweeklybudget'
-    _phantomjs_version = '2.0.0'
 
     def __init__(self, toxinidir, distdir):
         """
@@ -387,12 +374,6 @@ def _run_tests(self, db_container, container):
         else:
             logger.error('GET /payperiods: %s', r.status_code)
             failures = True
-        try:
-            self._test_phantomjs(container)
-            logger.info('phantomjs test PASSED')
-        except Exception as exc:
-            logger.error('PhantomJS test failed: %s', exc, exc_info=True)
-            failures = True
         try:
             self._test_scripts(container)
             logger.info('script tests PASSED')
@@ -483,59 +464,6 @@ def _container_ip(self, container):
         logger.debug('Container IP: %s / NetworkSettings: %s', c_ip, cnet)
         return c_ip
 
-    def _test_phantomjs(self, container):
-        """
-        Test phantomjs on the container.
-
-        :param container: biweeklybudget Docker container
-        :type container: ``docker.models.containers.Container``
-        """
-        cmd = [
-            '/bin/sh',
-            '-c',
-            '/usr/bin/phantomjs --version'
-        ]
-        logger.debug('Running: %s', cmd)
-        res = container.exec_run(cmd).decode().strip()
-        logger.debug('Command output:\n%s', res)
-        if res != self._phantomjs_version:
-            raise RuntimeError('ERROR: expected phantomjs version to be %s'
-                               ' but got %s', self._phantomjs_version, res)
-        logger.debug('phamtomjs version correct: %s', res)
-        phantomtest = dedent("""
-        "use strict";
-        var page = require("webpage").create();
-        page.open("http://127.0.0.1:80/", function(status) {
-          console.log("Status: " + status);
-          if(status === "success") {
-            console.log(page.content);
-          }
-          phantom.exit();
-        });
-        """)
-        phantom_script = self._string_to_tarfile('phantomtest.js', phantomtest)
-        container.put_archive('/', phantom_script)
-        cmd = [
-            '/bin/sh',
-            '-c',
-            '/usr/bin/phantomjs /phantomtest.js; '
-            'echo "exitcode=$?"'
-        ]
-        logger.debug('Running: %s', cmd)
-        res = container.exec_run(cmd).decode().strip()
-        logger.debug('Command output:\n%s', res)
-        if 'FATAL' in res or 'PhantomJS has crashed' in res:
-            raise RuntimeError('PhantomJS crashed during test')
-        exitcode = res.split("\n")[-1]
-        if exitcode != 'exitcode=0':
-            raise RuntimeError('Expected PhantomJS to exit with code 0, but'
-                               ' got %s' % exitcode)
-        if '<title>BiweeklyBudget</title>' not in res:
-            raise RuntimeError('PhantomJS page source does not look correct')
-        if '<a class="navbar-brand" href="index.html">BiweeklyBud' not in res:
-            raise RuntimeError('PhantomJS page source does not look correct')
-        logger.info('PhantomJS test SUCCEEDED')
-
     def _test_scripts(self, container):
         """
         Test scripts/entrypoints in the container.

docs/source/development.rst

@@ -79,7 +79,7 @@ Acceptance Tests
 
 There are acceptance tests, which use a real MySQL DB (see the connection string in ``tox.ini`` and ``conftest.py``) and a real Flask HTTP server, and selenium. Run them via the ``acceptance`` tox environment. Note that they're currently configured to use Headless Chrome; running them locally will require a modern Chrome version that supports the ``--headless`` flag (Chrome 59+) and a matching version of `chromedriver <https://sites.google.com/a/chromium.org/chromedriver/>`_.
 
-The acceptance tests connect to a local MySQL database using a connection string specified by the ``DB_CONNSTRING`` environment variable, or defaulting to a DB name and user/password that can be seen in ``conftest.py``. Once connected, the tests will drop all tables in the test DB, re-create all models/tables, and then load sample data. After the DB is initialized, tests will run the local Flask app on a random port, and run Selenium backed by PhantomJS.
+The acceptance tests connect to a local MySQL database using a connection string specified by the ``DB_CONNSTRING`` environment variable, or defaulting to a DB name and user/password that can be seen in ``conftest.py``. Once connected, the tests will drop all tables in the test DB, re-create all models/tables, and then load sample data. After the DB is initialized, tests will run the local Flask app on a random port, and run Selenium backed by headless Chrome.
 
 If you want to run the acceptance tests without dumping and refreshing the test database, export the ``NO_REFRESH_DB`` environment variable. Setting the ``NO_CLASS_REFRESH_DB``
 environment variable will prevent refreshing the DB after classes that manipulate data;

docs/source/getting_started.rst

@@ -18,7 +18,7 @@ Vault (the latter only if you choose to take advantage of the OFX downloading),
 * To use the automated OFX transaction downloading functionality:
 
   * A running, reachable instance of `Hashicorp Vault <https://www.vaultproject.io/>`_ with your financial institution web credentials stored in it.
-  * `PhantomJS <http://phantomjs.org/>`_ for downloading transaction data from institutions that do not support OFX remote access ("Direct Connect").
+  * If your bank does not support OFX remote access ("Direct Connect"), you will need to write a custom screen-scraper class using Selenium and a browser.
 
 Installation
 ------------

docs/source/ofx.rst

@@ -126,8 +126,6 @@ OFX statements should be saved. After instantiating the class, ``ofxgetter`` wil
 call the class's ``run()`` method with no arguments, and expect to receive an OFX
 statement string back.
 
-If cookies are a concern, be aware that saving and loading cookies is
-`broken in PhantomJS 2.x <https://github.com/ariya/phantomjs/issues/13115>`_.
 If you need to persist cookies across sessions, look into the
 :py:class:`~biweeklybudget.screenscraper.ScreenScraper` class'
 :py:meth:`~biweeklybudget.screenscraper.ScreenScraper.load_cookies` and
@@ -187,7 +185,7 @@ Here's a simple, contrived example of such a class:
             super(MyScraper, self).__init__(
                 savedir=savedir, screenshot=screenshot
             )
-            self.browser = self.get_browser('phantomjs')
+            self.browser = self.get_browser('chrome-headless')
             self.username = username
             self.password = password
             self.acct_num = acct_num