fix(secrets): use literal newlines in multiline env vars

Docker, bash, etc. support multiline environment variables, by simply wrapping
the value in single quotes with newlines, e.g. in an .env file:

  multi_line_value='first line
  second line
  third line'

Resulting in the expected:

  $ echo "$multi_line_value"
  first line
  second line
  third line

Due to a quirk in VS Code's Python extension, multiline values are not parsed,
see https://code.visualstudio.com/docs/python/environments#_environment-variables

> ... Multiline values aren't supported ...

And more ongoing discussion at microsoft/vscode-python#18307

When running locally in e.g. Debug mode, and secrets are read dynamically from the environment, Python loses
the multiline value and we end up with:

  >> value = os.environ.get("multi_line_value")
  >> print(value)

  first line

This changes the samples and docs so literal newlines are added to the value of the environment variable in an .env file:

  multi_line_value='first line\nsecond line\nthird line'

But the initial value read by Python contains _escaped_ newline characters:

  first line\\nsecond line\\nthird line

Hence unescaping so that local secrets contain the actual newline character:

  first line\nsecond line\nthird line

.env.sample

@@ -2,66 +2,14 @@ testsecret=Hello from the local environment!
 auth_provider_client_id=benefits-oauth-client-id
 courtesy_card_verifier_api_auth_key=server-auth-token
 mobility_pass_verifier_api_auth_key=server-auth-token
-client_private_key='-----BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEA1pt0ZoOuPEVPJJS+5r884zcjZLkZZ2GcPwr79XOLDbOi46on
-Ca79kjRnhS0VUK96SwUPS0z9J5mDA5LSNL2RoxFb5QGaevnJY828NupzTNdUd0sY
-JK3kRjKUggHWuB55hwJcH/Dx7I3DNH4NL68UAlK+VjwJkfYPrhq/bl5z8ZiurvBa
-5C1mDxhFpcTZlCfxQoas7D1d+uPACF6mEMbQNd3RaIaSREO50NvNywXIIt/OmCiR
-qI7JtOcn4eyh1I4j9WtlbMhRJLfwPMAgY5epTsWcURmhVofF2wVoFbib3JGCfA7t
-z/gmP5YoEKnf/cumKmF3e9LrZb8zwm7bTHUViwIDAQABAoIBAQCIv0XMjNvZS9DC
-XoXGQtVpcxj6dXfaiDgnc7hZDubsNCr3JtT5NqgdIYdVNQUABNDIPNEiCkzFjuwM
-uuF2+dRzM/x6UCs/cSsCjXYBCCOwMwV/fjpEJQnwMQqwTLulVsXZYYeSUtXVBf/8
-0tVULRty34apLFhsyX30UtboXQdESfpmm5ZsqsZJlYljw+M7JxRMneQclI19y/ya
-hPWlfhLB9OffVEJXGaWx1NSYnKoCMKqE/+4krROr6V62xXaNyX6WtU6XiT7C6R5A
-PBxfhmoeFdVCF6a+Qq0v2fKThYoZnV4sn2q2An9YPfynFYnlgzdfnAFSejsqxQd0
-fxYLOtMBAoGBAP1jxjHDJngZ1N+ymw9MIpRgr3HeuMP5phiSTbY2tu9lPzQd+TMX
-fhr1bQh2Fd/vU0u7X0yPnTWtUrLlCdGnWPpXivx95GNGgUUIk2HStFdrRx+f2Qvk
-G8vtLgmSbjQ26UiHzxi9Wa0a41PWIA3TixkcFrS2X29Qc4yd6pVHmicfAoGBANjR
-Z8aaDkSKLkq5Nk1T7I0E1+mtPoH1tPV/FJClXjJrvfDuYHBeOyUpipZddnZuPGWA
-IW2tFIsMgJQtgpvgs52NFI7pQGJRUPK/fTG+Ycocxo78TkLr/RIj8Kj5brXsbZ9P
-3/WBX5GAISTSp1ab8xVgK/Tm07hGupKVqnY2lCAVAoGAIql0YjhE2ecGtLcU+Qm8
-LTnwpg4GjmBnNTNGSCfB7IuYEsQK489R49Qw3xhwM5rkdRajmbCHm+Eiz+/+4NwY
-kt5I1/NMu7vYUR40MwyEuPSm3Q+bvEGu/71pL8wFIUVlshNJ5CN60fA8qqo+5kVK
-4Ntzy7Kq6WpC9Dhh75vE3ZcCgYEAty99uXtxsJD6+aEwcvcENkUwUztPQ6ggAwci
-je9Z/cmwCj6s9mN3HzfQ4qgGrZsHpk4ycCK655xhilBFOIQJ3YRUKUaDYk4H0YDe
-Osf6gTP8wtQDH2GZSNlavLk5w7UFDYQD2b47y4fw+NaOEYvjPl0p5lmb6ebAPZb8
-FbKZRd0CgYBC1HTbA+zMEqDdY4MWJJLC6jZsjdxOGhzjrCtWcIWEGMDF7oDDEoix
-W3j2hwm4C6vaNkH9XX1dr5+q6gq8vJQdbYoExl22BGMiNbfI3+sLRk0zBYL//W6c
-tSREgR4EjosqQfbkceLJ2JT1wuNjInI0eR9H3cRugvlDTeWtbdJ5qA==
------END RSA PRIVATE KEY-----'
-client_public_key='-----BEGIN PUBLIC KEY-----
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1pt0ZoOuPEVPJJS+5r88
-4zcjZLkZZ2GcPwr79XOLDbOi46onCa79kjRnhS0VUK96SwUPS0z9J5mDA5LSNL2R
-oxFb5QGaevnJY828NupzTNdUd0sYJK3kRjKUggHWuB55hwJcH/Dx7I3DNH4NL68U
-AlK+VjwJkfYPrhq/bl5z8ZiurvBa5C1mDxhFpcTZlCfxQoas7D1d+uPACF6mEMbQ
-Nd3RaIaSREO50NvNywXIIt/OmCiRqI7JtOcn4eyh1I4j9WtlbMhRJLfwPMAgY5ep
-TsWcURmhVofF2wVoFbib3JGCfA7tz/gmP5YoEKnf/cumKmF3e9LrZb8zwm7bTHUV
-iwIDAQAB
------END PUBLIC KEY-----'
-mst_payment_processor_client_cert='-----BEGIN CERTIFICATE-----
-PEM DATA
------END CERTIFICATE-----'
-mst_payment_processor_client_cert_private_key='-----BEGIN RSA PRIVATE KEY-----
-PEM DATA
------END RSA PRIVATE KEY-----'
-mst_payment_processor_client_cert_root_ca='-----BEGIN CERTIFICATE-----
-PEM DATA
------END CERTIFICATE-----'
-sacrt_payment_processor_client_cert='-----BEGIN CERTIFICATE-----
-PEM DATA
------END CERTIFICATE-----'
-sacrt_payment_processor_client_cert_private_key='-----BEGIN RSA PRIVATE KEY-----
-PEM DATA
------END RSA PRIVATE KEY-----'
-sacrt_payment_processor_client_cert_root_ca='-----BEGIN CERTIFICATE-----
-PEM DATA
------END CERTIFICATE-----'
-sbmtd_payment_processor_client_cert='-----BEGIN CERTIFICATE-----
-PEM DATA
------END CERTIFICATE-----'
-sbmtd_payment_processor_client_cert_private_key='-----BEGIN RSA PRIVATE KEY-----
-PEM DATA
------END RSA PRIVATE KEY-----'
-sbmtd_payment_processor_client_cert_root_ca='-----BEGIN CERTIFICATE-----
-PEM DATA
------END CERTIFICATE-----'
+client_private_key='-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA1pt0ZoOuPEVPJJS+5r884zcjZLkZZ2GcPwr79XOLDbOi46on\nCa79kjRnhS0VUK96SwUPS0z9J5mDA5LSNL2RoxFb5QGaevnJY828NupzTNdUd0sY\nJK3kRjKUggHWuB55hwJcH/Dx7I3DNH4NL68UAlK+VjwJkfYPrhq/bl5z8ZiurvBa\n5C1mDxhFpcTZlCfxQoas7D1d+uPACF6mEMbQNd3RaIaSREO50NvNywXIIt/OmCiR\nqI7JtOcn4eyh1I4j9WtlbMhRJLfwPMAgY5epTsWcURmhVofF2wVoFbib3JGCfA7t\nz/gmP5YoEKnf/cumKmF3e9LrZb8zwm7bTHUViwIDAQABAoIBAQCIv0XMjNvZS9DC\nXoXGQtVpcxj6dXfaiDgnc7hZDubsNCr3JtT5NqgdIYdVNQUABNDIPNEiCkzFjuwM\nuuF2+dRzM/x6UCs/cSsCjXYBCCOwMwV/fjpEJQnwMQqwTLulVsXZYYeSUtXVBf/8\n0tVULRty34apLFhsyX30UtboXQdESfpmm5ZsqsZJlYljw+M7JxRMneQclI19y/ya\nhPWlfhLB9OffVEJXGaWx1NSYnKoCMKqE/+4krROr6V62xXaNyX6WtU6XiT7C6R5A\nPBxfhmoeFdVCF6a+Qq0v2fKThYoZnV4sn2q2An9YPfynFYnlgzdfnAFSejsqxQd0\nfxYLOtMBAoGBAP1jxjHDJngZ1N+ymw9MIpRgr3HeuMP5phiSTbY2tu9lPzQd+TMX\nfhr1bQh2Fd/vU0u7X0yPnTWtUrLlCdGnWPpXivx95GNGgUUIk2HStFdrRx+f2Qvk\nG8vtLgmSbjQ26UiHzxi9Wa0a41PWIA3TixkcFrS2X29Qc4yd6pVHmicfAoGBANjR\nZ8aaDkSKLkq5Nk1T7I0E1+mtPoH1tPV/FJClXjJrvfDuYHBeOyUpipZddnZuPGWA\nIW2tFIsMgJQtgpvgs52NFI7pQGJRUPK/fTG+Ycocxo78TkLr/RIj8Kj5brXsbZ9P\n3/WBX5GAISTSp1ab8xVgK/Tm07hGupKVqnY2lCAVAoGAIql0YjhE2ecGtLcU+Qm8\nLTnwpg4GjmBnNTNGSCfB7IuYEsQK489R49Qw3xhwM5rkdRajmbCHm+Eiz+/+4NwY\nkt5I1/NMu7vYUR40MwyEuPSm3Q+bvEGu/71pL8wFIUVlshNJ5CN60fA8qqo+5kVK\n4Ntzy7Kq6WpC9Dhh75vE3ZcCgYEAty99uXtxsJD6+aEwcvcENkUwUztPQ6ggAwci\nje9Z/cmwCj6s9mN3HzfQ4qgGrZsHpk4ycCK655xhilBFOIQJ3YRUKUaDYk4H0YDe\nOsf6gTP8wtQDH2GZSNlavLk5w7UFDYQD2b47y4fw+NaOEYvjPl0p5lmb6ebAPZb8\nFbKZRd0CgYBC1HTbA+zMEqDdY4MWJJLC6jZsjdxOGhzjrCtWcIWEGMDF7oDDEoix\nW3j2hwm4C6vaNkH9XX1dr5+q6gq8vJQdbYoExl22BGMiNbfI3+sLRk0zBYL//W6c\ntSREgR4EjosqQfbkceLJ2JT1wuNjInI0eR9H3cRugvlDTeWtbdJ5qA==\n-----END RSA PRIVATE KEY-----'
+client_public_key='-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1pt0ZoOuPEVPJJS+5r88\n4zcjZLkZZ2GcPwr79XOLDbOi46onCa79kjRnhS0VUK96SwUPS0z9J5mDA5LSNL2R\noxFb5QGaevnJY828NupzTNdUd0sYJK3kRjKUggHWuB55hwJcH/Dx7I3DNH4NL68U\nAlK+VjwJkfYPrhq/bl5z8ZiurvBa5C1mDxhFpcTZlCfxQoas7D1d+uPACF6mEMbQ\nNd3RaIaSREO50NvNywXIIt/OmCiRqI7JtOcn4eyh1I4j9WtlbMhRJLfwPMAgY5ep\nTsWcURmhVofF2wVoFbib3JGCfA7tz/gmP5YoEKnf/cumKmF3e9LrZb8zwm7bTHUV\niwIDAQAB\n-----END PUBLIC KEY-----'
+mst_payment_processor_client_cert='-----BEGIN CERTIFICATE-----\nPEM DATA\n-----END CERTIFICATE-----'
+mst_payment_processor_client_cert_private_key='-----BEGIN RSA PRIVATE KEY-----\nPEM DATA\n-----END RSA PRIVATE KEY-----'
+mst_payment_processor_client_cert_root_ca='-----BEGIN CERTIFICATE-----\nPEM DATA\n-----END CERTIFICATE-----'
+sacrt_payment_processor_client_cert='-----BEGIN CERTIFICATE-----\nPEM DATA\n-----END CERTIFICATE-----'
+sacrt_payment_processor_client_cert_private_key='-----BEGIN RSA PRIVATE KEY-----\nPEM DATA\n-----END RSA PRIVATE KEY-----'
+sacrt_payment_processor_client_cert_root_ca='-----BEGIN CERTIFICATE-----\nPEM DATA\n-----END CERTIFICATE-----'
+sbmtd_payment_processor_client_cert='-----BEGIN CERTIFICATE-----\nPEM DATA\n-----END CERTIFICATE-----'
+sbmtd_payment_processor_client_cert_private_key='-----BEGIN RSA PRIVATE KEY-----\nPEM DATA\n-----END RSA PRIVATE KEY-----'
+sbmtd_payment_processor_client_cert_root_ca='-----BEGIN CERTIFICATE-----\nPEM DATA\n-----END CERTIFICATE-----'

benefits/secrets.py

@@ -52,8 +52,15 @@ def get_secret_by_name(secret_name, client=None):
 
     if runtime_env == "local":
         logger.debug("Runtime environment is local, reading from environment instead of Azure KeyVault.")
+        # environment variable names cannot contain the hyphen character
+        # assume the variable name is the same but with underscores instead
         env_secret_name = secret_name.replace("-", "_")
-        return os.environ.get(env_secret_name)
+        secret_value = os.environ.get(env_secret_name)
+        # we have to replace literal newlines here with the actual newline character
+        # to support local environment variables values that span multiple lines (e.g. PEM keys/certs)
+        # because the VS Code Python extension doesn't support multiline environment variables
+        # https://code.visualstudio.com/docs/python/environments#_environment-variables
+        return secret_value.replace("\\n", "\n")
 
     elif client is None:
         # construct the KeyVault URL from the runtime environment

docs/configuration/environment-variables.md

@@ -6,6 +6,30 @@ The sections below outline in more detail the application environment variables
 
 See other topic pages in this section for more specific environment variable configurations.
 
+!!! warning "Multiline environment variables"
+
+    Although Docker, bash, etc. support multiline values directly in e.g. an .env file:
+
+    ```bash
+    multi_line_value='first line
+    second line
+    third line'
+    ```
+
+    The VS Code Python extension does not parse multiline values: https://code.visualstudio.com/docs/python/environments#_environment-variables
+
+    When specifying multiline values for local usage, use the literal newline character `\n` but maintain the single quote wrapper:
+
+    ```bash
+    multi_line_value='first line\nsecond line\third line'
+    ```
+
+    A quick bash script to convert direct multiline values to their literal newline character equivalent is:
+
+    ```bash
+    echo "${multi_line_value//$'\n'/\\n}"
+    ```
+
 ## Amplitude
 
 !!! tldr "Amplitude API docs"

tests/pytest/test_secrets.py

@@ -1,8 +1,8 @@
+import pytest
 from azure.core.exceptions import ClientAuthenticationError
 from django.core.exceptions import ValidationError
-import pytest
 
-from benefits.secrets import KEY_VAULT_URL, SecretNameValidator, NAME_VALIDATOR, get_secret_by_name
+from benefits.secrets import KEY_VAULT_URL, NAME_VALIDATOR, SecretNameValidator, get_secret_by_name
 
 
 @pytest.fixture(autouse=True)
@@ -138,10 +138,13 @@ def test_get_secret_by_name__unauthenticated_client__returns_None(mocker, runtim
     assert actual_value is None
 
 
-def test_get_secret_by_name__local__returns_environment_variable(mocker, settings, secret_name, secret_value):
+def test_get_secret_by_name__local__returns_environment_variable(mocker, settings, secret_name):
     settings.RUNTIME_ENVIRONMENT = lambda: "local"
 
-    env_spy = mocker.patch("benefits.secrets.os.environ.get", return_value=secret_value)
+    secret_value_literal_newlines = "the\\nsecret\\nvalue"
+    expected_secret_value = secret_value_literal_newlines.replace("\\n", "\n")
+
+    env_spy = mocker.patch("benefits.secrets.os.environ.get", return_value=secret_value_literal_newlines)
     env_secret_name = secret_name.replace("-", "_")
     client_cls = mocker.patch("benefits.secrets.SecretClient")
     client = client_cls.return_value
@@ -151,4 +154,4 @@ def test_get_secret_by_name__local__returns_environment_variable(mocker, setting
     client_cls.assert_not_called()
     client.get_secret.assert_not_called()
     env_spy.assert_called_once_with(env_secret_name)
-    assert actual_value == secret_value
+    assert actual_value == expected_secret_value