make functional test cassettes work in any order

README.md

@@ -292,16 +292,41 @@ To individually run the unit tests, run `make test` to run the suite in parallel
 
 Functional tests are run alone using `make test-functional` and otherwise will be ran along with `make check`.
 
-Some of the tests appear to get into a state that reliably causes subsequent tests to crash. Such tests have been isolated and are clearly marked. The Makefile is used to ensure we exercise them in a completely new process.
-Use the `qtbot` object to drive the UI. This is part of the [pytest-qt](https://pytest-qt.readthedocs.io/en/latest/) package.
+We use `qtbot`, bundled with the [pytest-qt](https://pytest-qt.readthedocs.io/en/latest/) package, for UI interaction within our functional tests. We use [vcrpy](https://vcrpy.readthedocs.io/en/latest/) to replay the original response from the test server. These responses are stored in the `tests/functional/cassettes` directory. Our test TOTP is set to `994892` and stored in casettes.
 
-When writing tests that require the user to log in, on first run of the test
-you must make sure the TOTP value in `conftest.py` is correct for the time at which the test is run.
-For any further run of the test, this doesn't need to be the case since [vcrpy](https://vcrpy.readthedocs.io/en/latest/)
-will replay the original response from the test server. These responses are
-stored in the cassettes directory and should be committed to the git
-repository. Before committing, set the TOTP value in the cassette back to the value we use across all functional tests: `994892`.
+#### Generating new cassettes
 
+Some changes may require generating new cassettes, such as modifications to API endpoints. If you see the following warning, you may need to generate a new cassette or all new cassettes, depending on the change you made:
+
+```
+Can't overwrite existing cassette ('<path-to-cassette-for-a-functional-test>') in your current record mode ('once').
+```
+
+To generate new cassettes, follow these instructions:
+
+1. Before generating new cassettes that will be used to replay server interaction, bypass TOTP verification so that we can use the hard-coded value of `123456` in `tests/conftest.py`. You can do this by applying the following diff to the securedrop server, which you will be running in the next step:
+       ```diff
+       diff --git a/securedrop/models.py b/securedrop/models.py
+       index dcd26bbaf..50bc491b4 100644
+       --- a/securedrop/models.py
+       +++ b/securedrop/models.py
+       @@ -649,6 +649,7 @@ class Journalist(db.Model):
+
+                try:
+                    user = Journalist.query.filter_by(username=username).one()
+       +            return user
+                except NoResultFound:
+                    raise InvalidUsernameException(
+                        "invalid username '{}'".format(username))
+       (END)
+       ```
+2. Start the dev server: `NUM_SOURCES=0 make dev`
+3. Set up the dev server with data required for functional tests to pass and run in any order:
+    - Create two new sources, each with one submission that contains both a file and a message. The message should be set to `this is the message`. The file should be called `hello.txt` and contain a single line of text: `hello`.
+3. Delete all the old cassettes by running `rm -r tests/functional/cassettes` or just delete the cassettes you wish to regenerate.
+4. Run `make test-functional` which will regenerate cassettes for any that are missing in the `tests/functional/cassettes` folder.
+
+Note: One of the functional tests deletes a source, so you may need to add it back in between test runs where you are generating new cassettes.
 
 ## Making a Release
 

tests/conftest.py

@@ -34,14 +34,17 @@
 # Modify cassettes to use the following TOTP code. For developing new tests,
 # you can modify this so you don't need to keep editing cassettes during
 # development.
-TOTP = "994892"
+TOTP = "123456"
 
 # Time (in milliseconds) to wait for these GUI elements to render.
 TIME_APP_START = 1000
+TIME_LOGIN = 10000
+TIME_LOGOUT = 10000
+TIME_SYNC = 10000
 TIME_CLICK_ACTION = 1000
 TIME_RENDER_SOURCE_LIST = 20000
 TIME_RENDER_CONV_VIEW = 1000
-TIME_SYNC = 10000
+TIME_RENDER_EXPORT_DIALOG = 1000
 TIME_FILE_DOWNLOAD = 5000
 
 
@@ -78,47 +81,75 @@ def homedir(i18n):
 
 
 @pytest.fixture(scope="function")
-def functional_test_logged_out_context(homedir, reply_status_codes, session, config):
+def functional_test_app_started_context(homedir, reply_status_codes, session, config, qtbot):
     """
-    Returns a tuple containing a Window instance and a Controller instance that
-    have been correctly set up and isolated from any other instances of the
-    application to be run in the test suite.
+    Returns a tuple containing the gui window and controller of a configured client. This should be
+    used to for tests that need to start from the login dialog before the main application window
+    is visible.
     """
-
     gui = Window()
-    # Configure test keys.
-    create_gpg_test_context(homedir)
+    create_gpg_test_context(homedir)  # Configure test keys
+    session_maker = make_session_maker(homedir)  # Configure and create the database
+    controller = Controller(HOSTNAME, gui, session_maker, homedir, False, False)
+    gui.setup(controller)  # Connect the gui to the controller
 
-    # Configure and create the database.
-    session_maker = make_session_maker(homedir)
+    def login_dialog_is_visible():
+        assert gui.login_dialog is not None
 
-    # Create the controller.
-    controller = Controller(HOSTNAME, gui, session_maker, homedir, False, False)
-    # Link the gui and controller together.
-    gui.controller = controller
-    # Et Voila...
-    return (gui, controller, homedir)
+    qtbot.waitUntil(login_dialog_is_visible, timeout=TIME_APP_START)
+
+    return (gui, controller)
 
 
 @pytest.fixture(scope="function")
-def functional_test_logged_in_context(functional_test_logged_out_context, qtbot):
+def functional_test_logged_in_context(functional_test_app_started_context, qtbot):
     """
-    Returns a tuple containing a Window and Controller instance that have been
-    correctly configured to work together, isolated from other runs of the
-    test suite and in a logged in state.
+    Returns a tuple containing the gui window and controller of a configured client after logging in
+    with our test user account.
     """
-    gui, controller, tempdir = functional_test_logged_out_context
-    gui.setup(controller)
+    gui, controller = functional_test_app_started_context
+
+    # Authenticate our test account and login in
     qtbot.keyClicks(gui.login_dialog.username_field, USERNAME)
+    qtbot.wait(TIME_CLICK_ACTION)
     qtbot.keyClicks(gui.login_dialog.password_field, PASSWORD)
     qtbot.keyClicks(gui.login_dialog.tfa_field, TOTP)
     qtbot.mouseClick(gui.login_dialog.submit, Qt.LeftButton)
+    qtbot.wait(TIME_CLICK_ACTION)
 
-    def wait_for_login():
+    def logged_in():
         assert gui.login_dialog is None
+        assert gui.isVisible()
+
+    qtbot.waitUntil(logged_in, timeout=TIME_LOGIN)
+
+    return (gui, controller)
+
+
+@pytest.fixture(scope="function")
+def functional_test_offline_context(functional_test_logged_in_context, qtbot):
+    """
+    Returns a tuple containing the gui window and controller of a configured client after making
+    sure we have sources from a sync before switching to offline mode.
+    """
+    gui, controller = functional_test_logged_in_context
+
+    # The controller begins a sync as soon as the user authentication is successful so we just
+    # need to wait for the length of a sync to ensure the local db is up to date with the server
+    qtbot.wait(TIME_SYNC)
+
+    # Trigger log out
+    # Note: The qtbot object cannot interact with QAction items (as used in the logout button/menu),
+    # so we programatically logout rather than using the GUI via qtbot
+    gui.left_pane.user_profile.user_button.menu.logout.trigger()
+
+    def check_login_button():
+        assert gui.left_pane.user_profile.login_button.isVisible()
+
+    # When the login button appears then we know we're now in offline mode
+    qtbot.waitUntil(check_login_button, timeout=TIME_LOGOUT)
 
-    qtbot.waitUntil(wait_for_login, timeout=10000)
-    return (gui, controller, homedir)
+    return (gui, controller)
 
 
 @pytest.fixture(scope="function")