Fix unhashable issue with secrets.backend_kwargs and caching (apache#…

…25970) Resolves apache#25968
pdebelak · Aug 26, 2022 · aa87763 · aa87763
1 parent 876536e
commit aa87763
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 8 deletions.
diff --git a/airflow/configuration.py b/airflow/configuration.py
@@ -1545,19 +1545,18 @@ def get_custom_secret_backend() -> Optional[BaseSecretsBackend]:
     """Get Secret Backend if defined in airflow.cfg"""
     secrets_backend_cls = conf.getimport(section='secrets', key='backend')
     if secrets_backend_cls:
-        try:
-            backends: Any = conf.get(section='secrets', key='backend_kwargs', fallback='{}')
-            alternative_secrets_config_dict = json.loads(backends)
-        except JSONDecodeError:
-            alternative_secrets_config_dict = {}
-
-        return _custom_secrets_backend(secrets_backend_cls, **alternative_secrets_config_dict)
+        backends: Any = conf.get(section='secrets', key='backend_kwargs', fallback='{}')
+        return _custom_secrets_backend(secrets_backend_cls, backends)
     return None
 
 
 @functools.lru_cache(maxsize=2)
-def _custom_secrets_backend(secrets_backend_cls, **alternative_secrets_config_dict):
+def _custom_secrets_backend(secrets_backend_cls, backend_kwargs):
     """Separate function to create secrets backend instance to allow caching"""
+    try:
+        alternative_secrets_config_dict = json.loads(backend_kwargs)
+    except JSONDecodeError:
+        alternative_secrets_config_dict = {}
     return secrets_backend_cls(**alternative_secrets_config_dict)
 
 

diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py
@@ -1378,3 +1378,45 @@ def fake_read_secret(path, mount_point, version):
         assert 'fake_key' == test_conf.get('test', 'secret_key')
         mock_hvac.Client.assert_called_once()
         _custom_secrets_backend.cache_clear()
+
+    @mock.patch('airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend')
+    @conf_vars(
+        {
+            (
+                "secrets",
+                "backend",
+            ): "airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend",
+            (
+                "secrets",
+                "backend_kwargs",
+            ): '{"connections_prefix": "airflow-connections", '
+            '"gcp_keyfile_dict": {"keyfilearg1": "keyfileval1", "keyfilearg2": "keyfileval2"}}',
+        }
+    )
+    def test_config_from_secret_backend_allows_dict_arguments(self, mock_backend):
+        """Get Config Value from a Secret Backend with dict arguments"""
+        _custom_secrets_backend.cache_clear()
+
+        test_config = '''[test]
+sql_alchemy_conn_secret = sql_alchemy_conn
+'''
+        test_config_default = '''[test]
+sql_alchemy_conn = airflow
+'''
+
+        test_conf = AirflowConfigParser(default_config=parameterized_config(test_config_default))
+        test_conf.read_string(test_config)
+        test_conf.sensitive_config_values = test_conf.sensitive_config_values | {
+            ('test', 'sql_alchemy_conn'),
+        }
+
+        mock_backend.return_value.get_config.return_value = 'google_conf'
+
+        assert 'google_conf' == test_conf.get('test', 'sql_alchemy_conn')
+
+        mock_backend.assert_called_with(
+            connections_prefix='airflow-connections',
+            gcp_keyfile_dict={'keyfilearg1': 'keyfileval1', 'keyfilearg2': 'keyfileval2'},
+        )
+
+        _custom_secrets_backend.cache_clear()