Integrated integration testing infrastructure.

In other words: end-to-end, Org-based integration testing
will happen now if you pass a --org flag to pytest. Otherwise,
requests and responses will be mocked.

cumulusci/cli/tests/test_cci.py

@@ -62,22 +62,22 @@ def recursive_list_files(d="."):
 
 class TestCCI(unittest.TestCase):
     @classmethod
-    def setUpClass(self):
-        self.global_tempdir = tempfile.gettempdir()
-        self.tempdir = tempfile.mkdtemp()
-        self.environ_mock = mock.patch.dict(
+    def setUpClass(cls):
+        cls.global_tempdir = tempfile.gettempdir()
+        cls.tempdir = tempfile.mkdtemp()
+        cls.environ_mock = mock.patch.dict(
             os.environ, {"HOME": tempfile.mkdtemp(), "CUMULUSCI_KEY": ""}
         )
-        assert self.global_tempdir not in os.environ.get("HOME", "")
-        self.environ_mock.start()
-        assert self.global_tempdir in os.environ["HOME"]
+        assert cls.global_tempdir not in os.environ.get("HOME", "")
+        cls.environ_mock.start()
+        assert cls.global_tempdir in os.environ["HOME"]
 
     @classmethod
-    def tearDownClass(self):
-        assert self.global_tempdir in os.environ["HOME"]
-        self.environ_mock.stop()
-        shutil.rmtree(self.tempdir)
-        assert self.global_tempdir not in os.environ.get("HOME", "")
+    def tearDownClass(cls):
+        assert cls.global_tempdir in os.environ["HOME"]
+        cls.environ_mock.stop()
+        shutil.rmtree(cls.tempdir)
+        assert cls.global_tempdir not in os.environ.get("HOME", "")
 
     def test_get_installed_version(self):
         result = cci.get_installed_version()
@@ -299,7 +299,7 @@ def test_handle_click_exception(self, style, traceback, cci_open):
         cci_open.__enter__.return_value = mock.Mock()
 
         cci.handle_exception(click.ClickException("oops"), False, logfile_path)
-        style.call_args_list[0][0] == f"Error: oops"
+        style.call_args_list[0][0] == "Error: oops"
 
         os.remove(logfile_path)
 
@@ -318,7 +318,7 @@ def test_connection_exception_message(self, style):
         cci.connection_error_message()
         style.assert_called_once_with(
             (
-                f"We encountered an error with your internet connection. "
+                "We encountered an error with your internet connection. "
                 "Please check your connection and try the last cci command again."
             ),
             fg="red",

cumulusci/cli/tests/test_runtime.py

@@ -17,12 +17,13 @@
 class TestCliRuntime(unittest.TestCase):
     key = "1234567890abcdef"
 
-    @classmethod
-    def setUpClass(cls):
+    def setup_method(self, method):
         os.chdir(os.path.dirname(cumulusci.__file__))
+        self.environ_mock = mock.patch.dict(os.environ, {"CUMULUSCI_KEY": self.key})
+        self.environ_mock.start()
 
-    def setUp(self):
-        os.environ["CUMULUSCI_KEY"] = self.key
+    def teardown_method(self, method):
+        self.environ_mock.stop()
 
     def test_init(self):
         config = CliRuntime()

cumulusci/conftest.py

@@ -5,6 +5,7 @@
 
 from pytest import fixture
 from cumulusci.core.github import get_github_api
+from cumulusci.tests.pytest_plugins.pytest_sf_vcr import vcr_config
 
 
 @fixture(scope="session", autouse=True)
@@ -51,3 +52,6 @@ def _make_response(status):
         return MockHttpResponse(status)
 
     return _make_response
+
+
+vcr_config = fixture(vcr_config, scope="module")

cumulusci/tests/pytest_plugins/pytest_sf_orgconnect.py

@@ -0,0 +1,69 @@
+from pytest import fixture
+
+from cumulusci.cli.runtime import CliRuntime
+from cumulusci.salesforce_api.utils import get_simple_salesforce_connection
+from cumulusci.core.config import TaskConfig
+
+
+def pytest_addoption(parser, pluginmanager):
+    parser.addoption("--org", action="store", default=None, help="org to use")
+
+
+def sf_pytest_orgname(request):
+    return request.config.getoption("--org") or "qa"
+
+
+@fixture(scope="session")
+def runtime():
+    """Get the CumulusCI runtime for the current working directory."""
+    import os
+
+    print("YYY test_error_handling", os.getcwd())
+
+    return CliRuntime()
+
+
+@fixture(scope="session")
+def project_config(runtime):
+    """Get the project config for the current working directory."""
+    return runtime.project_config
+
+
+@fixture(scope="session")
+def org_config(request, runtime):
+    """Get an org config with an active access token.
+
+    Specify the org name using the --org option when running pytest.
+    Or else it will use your default CCI org.
+    """
+    org_name = sf_pytest_orgname(request)
+    assert org_name
+    org_name, org_config = runtime.get_org(org_name)
+    org_config.refresh_oauth_token(runtime.keychain)
+    return org_config
+
+
+@fixture
+def sf(request, project_config, org_config):
+    """Get a simple-salesforce client for org_config."""
+    sf = get_simple_salesforce_connection(project_config, org_config)
+    return sf
+
+
+@fixture
+def create_task(request, project_config, org_config):
+    """Get a task _factory_ which can be used to construct task instances.
+    """
+    session_project_config = project_config
+    session_org_config = org_config
+
+    def create_task(task_class, options=None, project_config=None, org_config=None):
+        project_config = project_config or session_project_config
+        org_config = org_config or session_org_config
+        options = options or {}
+
+        task_config = TaskConfig({"options": options})
+
+        return task_class(project_config, task_config, org_config)
+
+    return create_task

cumulusci/tests/pytest_plugins/pytest_sf_vcr.py

@@ -0,0 +1,58 @@
+import re
+
+from .pytest_sf_orgconnect import sf_pytest_orgname
+
+
+def sf_before_record_cb(request):
+    if request.body and "<sessionId>" in request.body.decode():
+        request.body = re.sub(
+            "<sessionId>.*</sessionId>",
+            "<sessionId>**Elided**</sessionId>",
+            request.body.decode(),
+        ).encode()
+    request.uri = re.sub(
+        "//.*.my.salesforce.com", "//orgname.salesforce.com", request.uri
+    )
+    request.uri = re.sub(
+        "//cs.*.salesforce.com", "//podname.salesforce.com", request.uri
+    )
+    request.uri = re.sub(
+        "Organization/00D3B000000F7hh", "Organization/ORGID", request.uri
+    )
+
+    request.headers = {"Request-Headers": "Elided"}
+
+    return request
+
+
+# junk_headers = ["Public-Key-Pins-Report-Only", ]
+def sf_before_record_response(response):
+    # for header in junk_headers:
+    #     if response["headers"].get(header):
+    #         del response["headers"][header]
+    response["headers"] = {"Response-Headers": "Elided"}
+    return response
+
+
+def vcr_config(request):
+    "Fixture for configuring VCR"
+
+    orgname = sf_pytest_orgname(request)
+
+    if orgname:
+        record_mode = "all"
+    else:
+        record_mode = "none"
+
+    return {
+        "record_mode": record_mode,
+        "decode_compressed_response": True,
+        "before_record_response": sf_before_record_response,
+        "before_record_request": sf_before_record_cb,
+        "filter_headers": [
+            "Authorization",
+            "Cookie",
+            "Public-Key-Pins-Report-Only",
+            "Last-Modified",
+        ],
+    }

cumulusci/tests/test_integration_infrastructure.py

@@ -0,0 +1,25 @@
+import pytest
+from cumulusci.tasks.salesforce.GetInstalledPackages import GetInstalledPackages
+from pathlib import Path
+
+
+class TestIntegrationInfrastructure:
+    "Test our two plugins for doing integration testing"
+
+    @pytest.mark.vcr()
+    def test_integration_tests(self, create_task):
+        task = create_task(GetInstalledPackages, {})
+        assert task() is not None
+
+    def test_file_was_created(self):
+        filename = (
+            Path(__file__).parent
+            / "cassettes/TestIntegrationInfrastructure.test_integration_tests.yaml"
+        )
+
+        assert filename.exists(), filename
+        with filename.open() as f:
+            data = f.read()
+            assert "Bearer 0" not in data
+            assert "Public-Key-Pins-Report-Only" not in data
+            assert "<sessionId>00" not in data

pytest.ini

@@ -1,3 +1,3 @@
 [pytest]
 testpaths = cumulusci
-addopts =  --typeguard-packages=cumulusci
+addopts =  --typeguard-packages=cumulusci  -p cumulusci.tests.pytest_plugins.pytest_sf_vcr -p cumulusci.tests.pytest_plugins.pytest_sf_orgconnect

requirements_dev.txt

@@ -6,10 +6,12 @@ coveralls==2.0.0
 flake8==3.8.2
 pre-commit==2.4.0
 pytest==5.4.3
+pytest-vcr==1.0.2
 responses==0.10.14
 Sphinx==3.1.0
 testfixtures==6.14.1
 tox==3.15.2
 twine==3.1.1
 typeguard==2.9.1
+vcrpy==4.0.2
 wheel==0.34.2