Merge pull request #7781 from ckan/7409-secret-consolidation-followup

[#7409] Secrets consolidation (followup from #7450)

changes/7781.migration

@@ -0,0 +1,12 @@
+Starting from CKAN 2.11, the :ref:`SECRET_KEY` configuration option is required to start CKAN. This is the secret token that is used by security related tasks by CKAN and its extensions. Previous CKAN versions relied on the ``beaker.session.secret`` config option for this. 
+The ``ckan generate config`` command generates a unique value for this option each time it generates a config file. Alternatively, you  can generate one manually with the following command::
+
+  python -c "import secrets; print(secrets.token_urlsafe(20))"
+
+Note that all the following secret configuration options will fallback to the ``SECRET_KEY`` value if not defined in your ini file:
+
+* :ref:`beaker.session.secret`
+* :ref:`beaker.session.validate_key`
+* :ref:`WTF_CSRF_SECRET_KEY`
+* :ref:`api_token.jwt.encode.secret`
+* :ref:`api_token.jwt.decode.secret`

ckan/config/config_declaration.yaml

@@ -84,8 +84,6 @@ groups:
         internal: true
       - key: PROPAGATE_EXCEPTIONS
         internal: true
-      - key: SECRET_KEY
-        internal: true
       - key: SEND_FILE_MAX_AGE_DEFAULT
         internal: true
       - key: SERVER_NAME
@@ -160,6 +158,23 @@ groups:
         validators: not_empty
         required: true
 
+      - key: SECRET_KEY
+        validators: not_empty
+        required: true
+        placeholder_callable: secrets:token_urlsafe
+        callable_args:
+          nbytes: 20
+        description: |
+          This is the secret token that is used by security related tasks by CKAN and its extensions.
+          ``ckan generate config`` generates a unique
+          value for this each time it generates a config file. Alternatively you can generate one with
+          the following command::
+
+            python -c "import secrets; print(secrets.token_urlsafe(20))"
+
+          When used in a cluster environment, the value must be the same on every machine.
+
+
       - key: ckan.legacy_route_mappings
         default: {}
         example: '{"home": "home.index", "about": "home.about", "search": "dataset.search"}'
@@ -251,16 +266,14 @@ groups:
         default: ckan
         description: Name of the cookie key used to save the session under.
       - key: beaker.session.secret
-        validators: not_empty
-        required: true
-        placeholder_callable: secrets:token_urlsafe
-        callable_args:
-          nbytes: 20
+        placeholder: string:%(SECRET_KEY)s
+        validators: configured_default("SECRET_KEY",None)
         description: |
           This is the secret token that the beaker library uses to hash the
           cookie sent to the client. `ckan generate config` generates a unique
           value for this each time it generates a config file. When used in a
           cluster environment, the value must be the same on every machine.
+          If not provided, the value of ``SECRET_KEY`` will be used.
       - key: beaker.session.auto
         type: bool
         default: False
@@ -317,13 +330,15 @@ groups:
       - key: beaker.session.validate_key
         validators: not_empty
         placeholder_callable: secrets:token_urlsafe
+        validators: configured_default("SECRET_KEY",None)
         callable_args:
           nbytes: 20
         description: |
           This is the secret token that is used to sign the local encrypted session.
           `ckan generate config` generates a unique
           value for this each time it generates a config file. When used in a
           cluster environment, the value must be the same on every machine.
+          If not provided, the value of ``SECRET_KEY`` will be used.
 
       - key: beaker.session.httponly
         type: bool
@@ -675,8 +690,11 @@ groups:
           this controls whether every view is protected by default.
 
       - key: WTF_CSRF_SECRET_KEY
-        description: Random data for generating secure tokens.
-        placeholder_callable: secrets:token_urlsafe
+        placeholder: string:%(SECRET_KEY)s
+        validators: configured_default("SECRET_KEY",None)
+        description: |
+          Random data for generating secure tokens.
+          If not provided, the value of ``SECRET_KEY`` will be used.
 
       - key: WTF_CSRF_METHODS
         type: list
@@ -782,7 +800,7 @@ groups:
         description: Number of bytes used to generate unique id for API Token.
 
       - key: api_token.jwt.encode.secret
-        placeholder: string:%(beaker.session.secret)s
+        placeholder: string:%(SECRET_KEY)s
         example: file:/path/to/private/key
         description: |
           A key suitable for the chosen algorithm(``api_token.jwt.algorithm``):
@@ -799,8 +817,10 @@ groups:
           * ``string:`` - Plain string, will be used as is.
           * ``file:`` - Path to file. Content of the file will be used as key.
 
+          If not provided, ``"string:" + SECRET_KEY`` is used.
+
       - key: api_token.jwt.decode.secret
-        placeholder: string:%(beaker.session.secret)s
+        placeholder: string:%(SECRET_KEY)s
         example: file:/path/to/public/key.pub
         description: |
           A key suitable for the chosen algorithm(``api_token.jwt.algorithm``):
@@ -817,6 +837,8 @@ groups:
           * ``string:`` - Plain string, will be used as is.
           * ``file:`` - Path to file. Content of the file will be used as key.
 
+          If not provided, ``"string:" + SECRET_KEY`` is used.
+
       - key: api_token.jwt.algorithm
         default: "HS256"
         example: RS256

ckan/config/middleware/flask_app.py

@@ -139,13 +139,6 @@ def make_flask_stack(conf: Union[Config, CKANConfig]) -> CKANApp:
 
     # Do all the Flask-specific stuff before adding other middlewares
 
-    # Secret key needed for flask-debug-toolbar and sessions
-    if not app.config.get('SECRET_KEY'):
-        app.config['SECRET_KEY'] = config.get('beaker.session.secret')
-    if not app.config.get('SECRET_KEY'):
-        raise RuntimeError(u'You must provide a value for the secret key'
-                           ' with the SECRET_KEY config option')
-
     root_path = config.get('ckan.root_path')
     if debug:
         from flask_debugtoolbar import DebugToolbarExtension

ckan/lib/api_token.py

@@ -19,7 +19,7 @@
 
 _config_encode_secret = u"api_token.jwt.encode.secret"
 _config_decode_secret = u"api_token.jwt.decode.secret"
-_config_secret_fallback = u"beaker.session.secret"
+_config_secret_fallback = u"SECRET_KEY"
 
 _config_algorithm = u"api_token.jwt.algorithm"
 

ckan/tests/config/test_environment.py

@@ -109,3 +109,22 @@ def test_config_from_envs_are_normalized(ckan_config):
     environment.update_config()
 
     assert ckan_config["smtp.starttls"] is False
+
+
+@pytest.mark.ckan_config("SECRET_KEY", "super_secret")
+@pytest.mark.ckan_config("beaker.session.secret", None)
+@pytest.mark.ckan_config("beaker.session.validate_key", None)
+@pytest.mark.ckan_config("WTF_CSRF_SECRET_KEY", None)
+def test_all_secrets_default_to_SECRET_KEY(ckan_config):
+
+    environment.update_config()
+
+    for key in [
+        "SECRET_KEY",
+        "beaker.session.secret",
+        "beaker.session.validate_key",
+        "WTF_CSRF_SECRET_KEY",
+    ]:
+        assert ckan_config[key] == "super_secret"
+
+    # Note: api_token.jwt.*.secret are tested in ckan/tests/lib/test_api_token.py

ckan/tests/config/test_middleware.py

@@ -78,10 +78,10 @@ def test_secret_key_is_used_if_present(app):
     assert app.flask_app.config[u"SECRET_KEY"] == u"super_secret_stuff"
 
 
-@pytest.mark.ckan_config(u"SECRET_KEY", None)
-def test_beaker_secret_is_used_by_default(app):
+@pytest.mark.ckan_config(u"SECRET_KEY", "some_secret")
+def test_SECRET_KEY_is_used_by_default(app):
     assert (
-        app.flask_app.config[u"SECRET_KEY"] == config[u"beaker.session.secret"]
+        app.flask_app.config[u"SECRET_KEY"] == "some_secret"
     )
 
 

ckan/tests/lib/test_api_token.py

@@ -0,0 +1,11 @@
+import pytest
+
+from ckan.lib.api_token import _get_secret
+
+
+@pytest.mark.ckan_config("SECRET_KEY", "super_secret")
+@pytest.mark.ckan_config("api_token.jwt.encode.secret", None)
+@pytest.mark.ckan_config("api_token.jwt.decode.secret", None)
+def test_secrets_default_to_SECRET_KEY():
+    assert _get_secret(True) == "super_secret"  # Encode
+    assert _get_secret(False) == "super_secret"  # Decode

test-core.ini

@@ -20,6 +20,7 @@ cache_dir = %(here)s/data
 debug = false
 testing = true
 
+SECRET_KEY = This_is_a_secret_or_is_it
 
 # Specify the Postgres database for SQLAlchemy to use
 sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_test
@@ -82,8 +83,8 @@ ckan.activity_streams_email_notifications = True
 ckan.tracking_enabled = true
 
 beaker.session.key = ckan
-beaker.session.secret = This_is_a_secret_or_is_it
-WTF_CSRF_SECRET_KEY = %(beaker.session.secret)s
+beaker.session.secret = %(SECRET_KEY)s
+WTF_CSRF_SECRET_KEY = %(SECRET_KEY)s
 
 ## background jobs
 ckan.jobs.timeout = 180