Test Against Multiple Vault Versions (#274)

* Clarify desired CI coverage for Vault versions

* Update CI tooling to cover multiple Vault versions

* Add utils to help skip tests when needed

* Fix broken root_token logic from previous refactor

* Add skipIf calls where needed for different Vault vers

* Better ordering

* Bump dist to xenial for good measure

.travis.yml

@@ -1,24 +1,80 @@
-dist: trusty
+dist: xenial
 sudo: false
 language: python
 matrix:
   include:
-  - python: '2.7'
-    env: TOXENV=py27 VAULT_BRANCH=release
-  - python: '2.7'
-    env: TOXENV=py27-flake8 VAULT_BRANCH=release
-  - python: '3.6'
-    env: TOXENV=py36 VAULT_BRANCH=release
-  - python: '3.6'
-    env: TOXENV=py36-flake8 VAULT_BRANCH=release
-  - python: '3.6'
-    env: TOXENV=py36 VAULT_BRANCH=head
+    - name: 'Python v2.7, Vault v0.8.3 - Integration/Unit Tests'
+      python: '2.7'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=0.8.3
+    - name: 'Python v2.7, Vault v0.9.6 - Integration/Unit Tests'
+      python: '2.7'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=0.9.6
+    - name: 'Python v2.7, Vault v0.10.4 - Integration/Unit Tests'
+      python: '2.7'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=0.10.4
+    - name: 'Python v2.7, Vault v0.11.1 - Integration/Unit Tests'
+      python: '2.7'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=0.11.1
+    - name: 'Python v2.7, Vault HEAD ref - Integration/Unit Tests'
+      python: '2.7'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=HEAD
+    - name: 'Python v2.7 - Linting (flake8)'
+      python: '2.7'
+      env:
+        - TOXENV=py27-flake8
+    - name: 'Python v3.6, Vault v0.8.3 - Integration/Unit Tests'
+      python: '3.6'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=0.8.3
+    - name: 'Python v3.6, Vault v0.9.6 - Integration/Unit Tests'
+      python: '3.6'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=0.9.6
+    - name: 'Python v3.6, Vault v0.10.4 - Integration/Unit Tests'
+      python: '3.6'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=0.10.4
+    - name: 'Python v3.6, Vault v0.11.1 - Integration/Unit Tests'
+      python: '3.6'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=0.11.1
+    - name: 'Python v3.6, Vault HEAD ref - Integration/Unit Tests'
+      python: '3.6'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=HEAD
+    - name: 'Python v3.6 - Linting (flake8)'
+      python: '3.6'
+      env:
+        - TOXENV=py27-flake8
   allow_failures:
-  - python: '3.6'
-    env: TOXENV=py36 VAULT_BRANCH=head
+    - name: 'Python v2.7, Vault HEAD ref - Integration/Unit Tests'
+      python: '2.7'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=HEAD
+    - name: 'Python v3.6, Vault HEAD ref - Integration/Unit Tests'
+      python: '3.6'
+      env:
+        - TOXENV=py27
+        - VAULT_VERSION=HEAD
 install:
-- scripts/install-vault-${VAULT_BRANCH}.sh
-- pip install tox
+  - scripts/install-vault.sh ${VAULT_VERSION}
+  - pip install tox
 script:
-- export PATH=$HOME/bin:$PATH
-- make test
+  - export PATH=$HOME/bin:$PATH
+  - make test

README.md

@@ -4,7 +4,8 @@
 
 [![Travis CI](https://travis-ci.org/hvac/hvac.svg?branch=master)](https://travis-ci.org/hvac/hvac) [![codecov](https://codecov.io/gh/hvac/hvac/branch/master/graph/badge.svg)](https://codecov.io/gh/hvac/hvac) [![Documentation Status](https://readthedocs.org/projects/hvac/badge/)](https://hvac.readthedocs.io/en/latest/?badge=latest) [![PyPI version](https://badge.fury.io/py/hvac.svg)](https://badge.fury.io/py/hvac)
 
-Tested against Vault v0.1.2 and HEAD. Requires v0.1.2 or later.
+Tested against the latest release, HEAD ref, and 3 previous major versions (counting back from the latest release) of Vault. 
+Currently supports Vault v0.8.3 or later.
 
 ## Documentation
 

hvac/tests/integration_tests/api/auth/test_ldap.py

@@ -2,7 +2,7 @@
 from unittest import TestCase
 
 from ldap_test import LdapServer
-from parameterized import parameterized
+from parameterized import parameterized, param
 
 from hvac import exceptions
 from hvac.tests import utils
@@ -319,13 +319,36 @@ def test_delete_user(self, test_label, username, configure_first=True, raises=No
         )
 
     @parameterized.expand([
-        ('working creds with policy', LDAP_USER_NAME, LDAP_USER_PASSWORD, True),
-        ('working creds no membership', LDAP_USER_NAME, LDAP_USER_PASSWORD, False, exceptions.InvalidRequest,
-         'user is not a member of any authorized group'),
-        ('invalid creds', 'not_your_dude_pal', 'some other dudes password', False, exceptions.InvalidRequest,
-         'ldap operation failed'),
+        param(
+            label='working creds with policy'
+        ),
+        param(
+            label='invalid creds',
+            username='not_your_dude_pal',
+            password='some other dudes password',
+            attach_policy=False,
+            raises=exceptions.InvalidRequest,
+        ),
+        # The following two test cases cover either side of the associated changelog entry for LDAP auth here:
+        # https://github.com/hashicorp/vault/blob/master/CHANGELOG.md#0103-june-20th-2018
+        param(
+            label='working creds no membership with Vault version >= 0.10.3',
+            attach_policy=False,
+            skip_due_to_vault_version=utils.skip_if_vault_version_lt('0.10.3'),
+        ),
+        param(
+            label='working creds no membership with Vault version < 0.10.3',
+            attach_policy=False,
+            raises=exceptions.InvalidRequest,
+            exception_message='user is not a member of any authorized group',
+            skip_due_to_vault_version=utils.skip_if_vault_version_ge('0.10.3'),
+        ),
     ])
-    def test_login(self, test_label, username, password, attach_policy, raises=None, exception_message=''):
+    def test_login(self, label, username=LDAP_USER_NAME, password=LDAP_USER_PASSWORD, attach_policy=True, raises=None,
+                   exception_message='', skip_due_to_vault_version=False):
+        if skip_due_to_vault_version:
+            self.skipTest(reason='test case does not apply to Vault version under test')
+
         test_policy_name = 'test-ldap-policy'
         self.client.ldap.configure(
             url=self.mock_ldap_url,
@@ -361,10 +384,6 @@ def test_login(self, test_label, username, password, attach_policy, raises=None,
                 username=username,
                 password=password,
             )
-            self.assertEqual(
-                first=['default', test_policy_name],
-                second=login_response['auth']['policies']
-            )
             self.assertDictEqual(
                 d1=dict(username=username),
                 d2=login_response['auth']['metadata'],
@@ -373,3 +392,11 @@ def test_login(self, test_label, username, password, attach_policy, raises=None,
                 first=login_response['auth']['client_token'],
                 second=self.client.token,
             )
+            if attach_policy:
+                expected_policies = ['default', test_policy_name]
+            else:
+                expected_policies = ['default']
+            self.assertEqual(
+                first=expected_policies,
+                second=login_response['auth']['policies']
+            )

hvac/tests/integration_tests/api/secrets_engines/test_kv_v2.py

@@ -1,12 +1,14 @@
 import logging
 from unittest import TestCase
+from unittest import skipIf
 
 from parameterized import parameterized
 
 from hvac import exceptions
 from hvac.tests import utils
 
 
+@skipIf(utils.skip_if_vault_version_lt('0.10.0'), "KV version 2 secret engine not available before Vault version 0.10.0")
 class TestKvV2(utils.HvacIntegrationTestCase, TestCase):
     DEFAULT_MOUNT_POINT = 'kvv2'
 

hvac/tests/integration_tests/v1/test_integration.py

@@ -5,6 +5,8 @@
 from uuid import UUID
 
 from hvac import exceptions
+from unittest import skipIf
+
 from hvac.tests import utils
 
 
@@ -1125,10 +1127,10 @@ def test_start_generate_root_with_completion(self):
         # Assert our new root token is properly formed and authenticated
         self.client.token = new_root_token
         if self.client.is_authenticated():
-            self.root_token = new_root_token
+            self.manager.root_token = new_root_token
         else:
             # If our new token was unable to authenticate, set the test client's token back to the original value
-            self.client.token = self.root_token
+            self.client.token = self.manager.root_token
             self.fail('Unable to authenticate with the newly generated root token.')
 
     def test_start_generate_root_then_cancel(self):
@@ -1264,6 +1266,7 @@ def test_tune_auth_backend(self):
 
         self.client.disable_auth_backend(mount_point=test_mount_point)
 
+    @skipIf(utils.skip_if_vault_version('0.10.0'), "not supported in this vault version")
     def test_kv2_secret_backend(self):
         if 'test/' in self.client.list_secret_backends():
             self.client.disable_secret_backend('test')

hvac/tests/utils.py

@@ -1,11 +1,13 @@
 """Collection of classes and methods used by various hvac test cases."""
 import json
 import logging
+import operator
 import os
 import re
 import socket
 import subprocess
 import time
+from distutils.version import StrictVersion
 
 from hvac import Client
 
@@ -19,6 +21,31 @@
 logger = logging.getLogger(__name__)
 
 VERSION_REGEX = re.compile('Vault v([\d\.]+)')
+LATEST_VAULT_VERSION = '0.11.1'
+
+
+def get_installed_vault_version():
+    command = ['vault', '-version']
+    process = subprocess.Popen(args=command, stdout=subprocess.PIPE)
+    output, _ = process.communicate()
+    version = output.strip().split()[1].lstrip('v')
+    return version
+
+
+def skip_if_vault_version(supported_version, comparison=operator.lt):
+    current_version = os.getenv('VAULT_VERSION')
+    if current_version is None or current_version.lower() == 'head':
+        current_version = get_installed_vault_version()
+
+    return comparison(StrictVersion(current_version), StrictVersion(supported_version))
+
+
+def skip_if_vault_version_lt(supported_version):
+    return skip_if_vault_version(supported_version, comparison=operator.lt)
+
+
+def skip_if_vault_version_ge(supported_version):
+    return skip_if_vault_version(supported_version, comparison=operator.ge)
 
 
 def create_client(**kwargs):

scripts/install-vault.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+set -eux
+
+DEFAULT_VAULT_VERSION=0.11.1
+VAULT_VERSION=${1:-$DEFAULT_VAULT_VERSION}
+
+function build_and_install_vault_head_ref() {
+    mkdir -p $HOME/bin
+
+    eval "$(GIMME_GO_VERSION=1.10.3 gimme)"
+
+    export GOPATH=$HOME/go
+    mkdir $GOPATH
+
+    export PATH=$GOPATH/bin:$PATH
+
+    go get github.com/tools/godep
+    go get github.com/mitchellh/gox
+
+    git clone https://github.com/hashicorp/vault.git $GOPATH/src/github.com/hashicorp/vault
+    cd $GOPATH/src/github.com/hashicorp/vault
+    make dev
+
+    mv bin/vault $HOME/bin
+}
+
+function install_vault_release() {
+    mkdir -p $HOME/bin
+
+    cd /tmp
+
+    curl -sOL https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip
+    unzip vault_${VAULT_VERSION}_linux_amd64.zip
+    mv vault $HOME/bin
+}
+
+if [[ "$(tr [A-Z] [a-z] <<<"$VAULT_VERSION")" == "head" ]]; then
+    build_and_install_vault_head_ref
+else
+    install_vault_release
+fi

tox.ini

@@ -10,7 +10,7 @@ commands = nosetests -s --with-coverage --cover-package=hvac --cover-html {posar
            codecov
 deps = -rrequirements.txt
        -rrequirements-dev.txt
-passenv = CI TRAVIS TRAVIS_*
+passenv = CI TRAVIS TRAVIS_* VAULT_VERSION
 
 [testenv:py27-flake8]
 basepython = python2.7