From d7041042e38ef0c2a4c052571b19a2e3c680dcc9 Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Fri, 23 Mar 2018 13:59:48 -0700 Subject: [PATCH 01/15] Add passenv; change py3 to py36 --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 511db66..4501b9b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,10 @@ [tox] -envlist = py27, py3, flake8 +envlist = py27, py36, flake8 skipsdist = true [testenv] +passenv = HOME DISPLAY MOZ_HEADLESS PYTEST_ADDOPTS PYTEST_BASE_URL \ + SAUCELABS_USR SAUCELABS_PSW JENKINS_URL JOB_NAME BUILD_NUMBER deps = -raxe_selenium_python/tests/requirements/tests.txt commands = pytest \ --verbose \ From 20d331a86b940fbfd4d82c7dea61fe2409ffe5ee Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Fri, 23 Mar 2018 15:26:36 -0700 Subject: [PATCH 02/15] Fix Jenkinsfile? --- Jenkinsfile | 53 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0460512..56cb524 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,6 +9,20 @@ def capabilities = [ pipeline { agent any + libraries { + lib('fxtest@1.10') + } + environment { + PYTEST_PROCESSES = "${PYTEST_PROCESSES ?: "auto"}" + PYTEST_ADDOPTS = + "-n=${PYTEST_PROCESSES} " + + "--tb=short " + + "--color=yes " + + "--driver=SauceLabs " + + "--variables=capabilities.json" + PULSE = credentials('PULSE') + SAUCELABS = credentials('SAUCELABS') + } stages { stage('Checkout') { steps { @@ -21,27 +35,36 @@ pipeline { steps { deleteDir() unstash 'workspace' - ansiColor('xterm') { - sh "tox -e flake8" - } + ansiColor('xterm') + writeCapabilities(capabilities, 'capabilities.json') + sh "tox -e flake8" } } stage('Test') { - environment { - SAUCELABS = credentials('SAUCELABS') - } - steps { - unstash 'workspace' - ansiColor('xterm') { - sh "tox" + parallel { + stage('py27') { + steps { + unstash 'workspace' + ansiColor('xterm') + writeCapabilities(capabilities, 'capabilities.json') + sh "tox -e py27" + } } - } - post { - always { - stash includes: 'results/*', name: 'results' - archiveArtifacts 'results/*' + stage('py36') { + steps { + unstash 'workspace' + ansiColor('xterm') + writeCapabilities(capabilities, 'capabilities.json') + sh "tox -e py36" + } } } } } + post { + always { + stash includes: 'results/*', name: 'results' + archiveArtifacts 'results/*' + } + } } From 2ec256af00f1fe70a3d1358610fbc9d8ac35c0cc Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 16:33:39 -0700 Subject: [PATCH 03/15] Add Dockerfile --- Dockerfile | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c18eabd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:xenial + +ENV DEBIAN_FRONTEND=noninteractive \ + MOZ_HEADLESS=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +RUN apt-get update \ + && apt-get install -y software-properties-common \ + && add-apt-repository ppa:deadsnakes/ppa \ + && apt-get update \ + && apt-get install -y bzip2 curl firefox git python2.7 python3.6 python3-pip \ + && rm -rf /var/lib/apt/lists/* + +ENV FIREFOX_VERSION=59.0 + +RUN curl -fsSLo /tmp/firefox.tar.bz2 https://download-installer.cdn.mozilla.net/pub/firefox/releases/$FIREFOX_VERSION/linux-x86_64/en-US/firefox-$FIREFOX_VERSION.tar.bz2 \ + && apt-get -y purge firefox \ + && rm -rf /opt/firefox \ + && tar -C /opt -xjf /tmp/firefox.tar.bz2 \ + && rm /tmp/firefox.tar.bz2 \ + && mv /opt/firefox /opt/firefox-$FIREFOX_VERSION \ + && ln -fs /opt/firefox-$FIREFOX_VERSION/firefox /usr/bin/firefox + +ENV GECKODRIVER_VERSION=0.20.0 +RUN curl -fsSLo /tmp/geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v$GECKODRIVER_VERSION/geckodriver-v$GECKODRIVER_VERSION-linux64.tar.gz \ + && rm -rf /opt/geckodriver \ + && tar -C /opt -zxf /tmp/geckodriver.tar.gz \ + && rm /tmp/geckodriver.tar.gz \ + && mv /opt/geckodriver /opt/geckodriver-$GECKODRIVER_VERSION \ + && chmod 755 /opt/geckodriver-$GECKODRIVER_VERSION \ + && ln -fs /opt/geckodriver-$GECKODRIVER_VERSION /usr/bin/geckodriver + +ENV TOX_VERSION=2.9.1 +RUN pip3 install tox==$TOX_VERSION + +ADD . /src +WORKDIR /src From fc07a20a6d72e93aa1c69c50e9bdffafb2d510b4 Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 16:38:08 -0700 Subject: [PATCH 04/15] Fix a few flake8 issues --- Jenkinsfile | 69 ++++++++++++++++++--------- axe_selenium_python/axe.py | 2 +- axe_selenium_python/tests/conftest.py | 2 +- 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 56cb524..e037988 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,12 @@ def capabilities = [ pipeline { agent any libraries { - lib('fxtest@1.10') + lib('fxtest@1.9') + } + options { + ansiColor('xterm') + timestamps() + timeout(time: 30, unit: 'MINUTES') } environment { PYTEST_PROCESSES = "${PYTEST_PROCESSES ?: "auto"}" @@ -24,38 +29,39 @@ pipeline { SAUCELABS = credentials('SAUCELABS') } stages { - stage('Checkout') { - steps { - deleteDir() - checkout scm - stash 'workspace' - } - } stage('Lint') { steps { - deleteDir() - unstash 'workspace' - ansiColor('xterm') - writeCapabilities(capabilities, 'capabilities.json') sh "tox -e flake8" } } stage('Test') { parallel { - stage('py27') { + stage('py36') { steps { - unstash 'workspace' - ansiColor('xterm') writeCapabilities(capabilities, 'capabilities.json') - sh "tox -e py27" + sh "tox -e py36" + } + post { + always { + stash includes: 'results/py36.html', name: 'py36' + archiveArtifacts 'results/*' + junit 'results/*.xml' + } } } - stage('py36') { + stage('py27') { steps { - unstash 'workspace' - ansiColor('xterm') writeCapabilities(capabilities, 'capabilities.json') - sh "tox -e py36" + sh "tox -e py27" + } + post { + always { + stash includes: 'results/py27.html', name: 'py27' + archiveArtifacts 'results/*' + junit 'results/*.xml' + submitToActiveData('results/py27_raw.txt') + submitToTreeherder('fxapom', 'T', 'Tests', 'results/*', 'results/py27_tbpl.txt') + } } } } @@ -63,8 +69,27 @@ pipeline { } post { always { - stash includes: 'results/*', name: 'results' - archiveArtifacts 'results/*' + unstash 'py36' + unstash 'py27' + publishHTML(target: [ + allowMissing: false, + alwaysLinkToLastBuild: true, + keepAll: true, + reportDir: 'results', + reportFiles: "py36.html, py27.html", + reportName: 'HTML Report']) + } + changed { + ircNotification() + } + failure { + emailext( + attachLog: true, + attachmentsPattern: 'results/*.html', + body: '$BUILD_URL\n\n$FAILED_TESTS', + replyTo: '$DEFAULT_REPLYTO', + subject: '$DEFAULT_SUBJECT', + to: '$DEFAULT_RECIPIENTS') } } } diff --git a/axe_selenium_python/axe.py b/axe_selenium_python/axe.py index 7db6130..d76f4cb 100644 --- a/axe_selenium_python/axe.py +++ b/axe_selenium_python/axe.py @@ -40,7 +40,7 @@ def execute(self, context=None, options=None): """ Run axe against the current page. - :param context: which part of the page to analyze and/or what to exclude. + :param context: which page part(s) to analyze and/or what to exclude. :param options: dictionary of aXe options. """ template = 'return axe.run(%s).then(function(result){return result;});' diff --git a/axe_selenium_python/tests/conftest.py b/axe_selenium_python/tests/conftest.py index 99f345d..567c454 100644 --- a/axe_selenium_python/tests/conftest.py +++ b/axe_selenium_python/tests/conftest.py @@ -42,7 +42,7 @@ def pytest_html_results_table_row(report, cells): @pytest.mark.hookwrapper def pytest_runtest_makereport(item, call): - """Make HTML report. Use test function docstrings as description in report.""" + """Make HTML report using test-function docstrings as description.""" outcome = yield report = outcome.get_result() # add docstring to 'description' column From c937a5b2101e357bbcb22a12f4248fd7e4d2d97f Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 16:44:01 -0700 Subject: [PATCH 05/15] Use Dockerfile and agent --- Jenkinsfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e037988..2422ee8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,6 +30,9 @@ pipeline { } stages { stage('Lint') { + agent { + dockerfile true + } steps { sh "tox -e flake8" } @@ -37,6 +40,9 @@ pipeline { stage('Test') { parallel { stage('py36') { + agent { + dockerfile true + } steps { writeCapabilities(capabilities, 'capabilities.json') sh "tox -e py36" @@ -50,6 +56,9 @@ pipeline { } } stage('py27') { + agent { + dockerfile true + } steps { writeCapabilities(capabilities, 'capabilities.json') sh "tox -e py27" @@ -59,8 +68,6 @@ pipeline { stash includes: 'results/py27.html', name: 'py27' archiveArtifacts 'results/*' junit 'results/*.xml' - submitToActiveData('results/py27_raw.txt') - submitToTreeherder('fxapom', 'T', 'Tests', 'results/*', 'results/py27_tbpl.txt') } } } From 547877b15442a2541d000549238cdceab5270ae7 Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 16:54:57 -0700 Subject: [PATCH 06/15] Add pytest_xdist, so we can use -n=auto --- axe_selenium_python/tests/requirements/tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/axe_selenium_python/tests/requirements/tests.txt b/axe_selenium_python/tests/requirements/tests.txt index 2a6dadd..e6d48b2 100644 --- a/axe_selenium_python/tests/requirements/tests.txt +++ b/axe_selenium_python/tests/requirements/tests.txt @@ -2,3 +2,4 @@ pytest==3.4.2 pytest-selenium==1.12.0 pytest-html==1.16.1 pytest_base_url==1.4.1 +pytest_xdist==1.22.2 From 00c7a079efbe323846d6ebc55b924c3faa557623 Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 17:13:46 -0700 Subject: [PATCH 07/15] Restore results paths --- Jenkinsfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2422ee8..3b9907a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,7 +20,6 @@ pipeline { environment { PYTEST_PROCESSES = "${PYTEST_PROCESSES ?: "auto"}" PYTEST_ADDOPTS = - "-n=${PYTEST_PROCESSES} " + "--tb=short " + "--color=yes " + "--driver=SauceLabs " + @@ -78,14 +77,6 @@ pipeline { always { unstash 'py36' unstash 'py27' - publishHTML(target: [ - allowMissing: false, - alwaysLinkToLastBuild: true, - keepAll: true, - reportDir: 'results', - reportFiles: "py36.html, py27.html", - reportName: 'HTML Report']) - } changed { ircNotification() } From 1c69cb5c386a27193a6b733bbfb33c9b427917d1 Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 17:16:57 -0700 Subject: [PATCH 08/15] Remove PYTEST_PROCESSES, for now --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3b9907a..b7a5bf7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,6 @@ pipeline { timeout(time: 30, unit: 'MINUTES') } environment { - PYTEST_PROCESSES = "${PYTEST_PROCESSES ?: "auto"}" PYTEST_ADDOPTS = "--tb=short " + "--color=yes " + @@ -56,7 +55,7 @@ pipeline { } stage('py27') { agent { - dockerfile true + dockerfile true } steps { writeCapabilities(capabilities, 'capabilities.json') From 77b0c9f5810d43e750022fe4d69e0b1583397b0b Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 17:19:10 -0700 Subject: [PATCH 09/15] Restore removed brace --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index b7a5bf7..c37107c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,6 +76,7 @@ pipeline { always { unstash 'py36' unstash 'py27' + } changed { ircNotification() } From af2d2cfa289ada8c15b6620edfe7da242144a03e Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 17:37:54 -0700 Subject: [PATCH 10/15] True allowing missing files, for now --- Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index c37107c..3eebbe5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,6 +76,13 @@ pipeline { always { unstash 'py36' unstash 'py27' + publishHTML(target: [ + allowMissing: true, + alwaysLinkToLastBuild: true, + keepAll: true, + reportDir: 'results', + reportFiles: "py36.html, py27.html", + reportName: 'HTML Report']) } changed { ircNotification() From 4ce18b70bc36103232efbcbf79b73bfac47aeeb3 Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 17:50:55 -0700 Subject: [PATCH 11/15] Remove 'results' stashing and publishing, for now --- Jenkinsfile | 39 --------------------------------------- 1 file changed, 39 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3eebbe5..4b317d9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,13 +45,6 @@ pipeline { writeCapabilities(capabilities, 'capabilities.json') sh "tox -e py36" } - post { - always { - stash includes: 'results/py36.html', name: 'py36' - archiveArtifacts 'results/*' - junit 'results/*.xml' - } - } } stage('py27') { agent { @@ -61,40 +54,8 @@ pipeline { writeCapabilities(capabilities, 'capabilities.json') sh "tox -e py27" } - post { - always { - stash includes: 'results/py27.html', name: 'py27' - archiveArtifacts 'results/*' - junit 'results/*.xml' - } - } } } } } - post { - always { - unstash 'py36' - unstash 'py27' - publishHTML(target: [ - allowMissing: true, - alwaysLinkToLastBuild: true, - keepAll: true, - reportDir: 'results', - reportFiles: "py36.html, py27.html", - reportName: 'HTML Report']) - } - changed { - ircNotification() - } - failure { - emailext( - attachLog: true, - attachmentsPattern: 'results/*.html', - body: '$BUILD_URL\n\n$FAILED_TESTS', - replyTo: '$DEFAULT_REPLYTO', - subject: '$DEFAULT_SUBJECT', - to: '$DEFAULT_RECIPIENTS') - } - } } From c08f6a418c9299750b860b61c7a3c71f8ebb2314 Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 17:52:23 -0700 Subject: [PATCH 12/15] Remove pytest_xdist from reqs --- axe_selenium_python/tests/requirements/tests.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/axe_selenium_python/tests/requirements/tests.txt b/axe_selenium_python/tests/requirements/tests.txt index e6d48b2..2a6dadd 100644 --- a/axe_selenium_python/tests/requirements/tests.txt +++ b/axe_selenium_python/tests/requirements/tests.txt @@ -2,4 +2,3 @@ pytest==3.4.2 pytest-selenium==1.12.0 pytest-html==1.16.1 pytest_base_url==1.4.1 -pytest_xdist==1.22.2 From 579984e15647cb41048423406ee15f92d470dc9e Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 17:55:20 -0700 Subject: [PATCH 13/15] Swap out SauceLabs driver for Firefox --- tox.ini | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 4501b9b..efae7e4 100644 --- a/tox.ini +++ b/tox.ini @@ -8,9 +8,8 @@ passenv = HOME DISPLAY MOZ_HEADLESS PYTEST_ADDOPTS PYTEST_BASE_URL \ deps = -raxe_selenium_python/tests/requirements/tests.txt commands = pytest \ --verbose \ - --driver=SauceLabs \ - --base-url=https://mozillians.org \ - --capability browserName Firefox + --driver=Firefox {posargs} \ + --base-url=https://web-mozillians-staging.production.paas.mozilla.community \ [testenv:flake8] deps = -raxe_selenium_python/tests/requirements/flake8.txt From 439ccf684916ee93adb474a782b18f90ada7a66b Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 17:56:38 -0700 Subject: [PATCH 14/15] Remove bogus continuation line --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index efae7e4..1848962 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ deps = -raxe_selenium_python/tests/requirements/tests.txt commands = pytest \ --verbose \ --driver=Firefox {posargs} \ - --base-url=https://web-mozillians-staging.production.paas.mozilla.community \ + --base-url=https://web-mozillians-staging.production.paas.mozilla.community [testenv:flake8] deps = -raxe_selenium_python/tests/requirements/flake8.txt From 17830b7ac0f21484a8f2b414566ad19e4ca6ab8e Mon Sep 17 00:00:00 2001 From: Stephen Donner Date: Sat, 24 Mar 2018 17:59:57 -0700 Subject: [PATCH 15/15] Restore SauceLabs driver. Sigh. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1848962..d4cd4eb 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ passenv = HOME DISPLAY MOZ_HEADLESS PYTEST_ADDOPTS PYTEST_BASE_URL \ deps = -raxe_selenium_python/tests/requirements/tests.txt commands = pytest \ --verbose \ - --driver=Firefox {posargs} \ + --driver=SauceLabs \ --base-url=https://web-mozillians-staging.production.paas.mozilla.community [testenv:flake8]