Expand Up
@@ -16,7 +16,6 @@
ConnectorRegistry ,
CustomConnectorTemplateLoader ,
FileConnectorTemplateLoader ,
register_custom_functions ,
)
from fides .api .service .saas_request .saas_request_override_factory import (
SaaSRequestOverrideFactory ,
Expand Down
Expand Up
@@ -113,15 +112,13 @@ def replaceable_planet_express_zip(
self ,
replaceable_planet_express_config ,
planet_express_dataset ,
planet_express_functions ,
planet_express_icon ,
) -> BytesIO :
return create_zip_file (
{
"config.yml" : replaceable_planet_express_config ,
"dataset.yml" : planet_express_dataset ,
"icon.svg" : planet_express_icon ,
"functions.py" : planet_express_functions ,
}
)
Expand All
@@ -135,8 +132,6 @@ def non_replaceable_zendesk_zip(self, zendesk_config, zendesk_dataset) -> BytesI
)
def test_custom_connector_template_loader_no_templates (self ):
CONFIG .security .allow_custom_connector_functions = True
connector_templates = CustomConnectorTemplateLoader .get_connector_templates ()
assert connector_templates == {}
Expand All
@@ -148,143 +143,20 @@ def test_custom_connector_template_loader_invalid_template(
mock_all : MagicMock ,
planet_express_dataset ,
planet_express_icon ,
planet_express_functions ,
):
CONFIG .security .allow_custom_connector_functions = True
mock_all .return_value = [
CustomConnectorTemplate (
key = "planet_express" ,
name = "Planet Express" ,
config = "planet_express_config" ,
dataset = planet_express_dataset ,
icon = planet_express_icon ,
functions = planet_express_functions ,
)
]
# verify the custom functions aren't loaded if the template is invalid
connector_templates = CustomConnectorTemplateLoader .get_connector_templates ()
assert connector_templates == {}
with pytest .raises (NoSuchSaaSRequestOverrideException ):
SaaSRequestOverrideFactory .get_override (
"planet_express_user_access" , SaaSRequestType .READ
)
# assert the strategy was not registered
authentication_strategies = AuthenticationStrategy .get_strategies ()
assert "planet_express" not in
strategy .name for strategy in authentication_strategies
]
@mock .patch (
"fides.api.models.custom_connector_template.CustomConnectorTemplate.all"
)
def test_custom_connector_template_loader_invalid_functions (
self ,
mock_all : MagicMock ,
planet_express_config ,
planet_express_dataset ,
planet_express_icon ,
):
CONFIG .security .allow_custom_connector_functions = True
# save custom connector template to the database
mock_all .return_value = [
CustomConnectorTemplate (
key = "planet_express" ,
name = "Planet Express" ,
config = planet_express_config ,
dataset = planet_express_dataset ,
icon = planet_express_icon ,
functions = "planet_express_functions" ,
)
]
# verify nothing is loaded if the custom functions fail to load
connector_templates = CustomConnectorTemplateLoader .get_connector_templates ()
assert connector_templates == {}
@mock .patch (
"fides.api.models.custom_connector_template.CustomConnectorTemplate.all"
)
def test_custom_connector_template_loader_custom_connector_functions_disabled (
self ,
mock_all : MagicMock ,
planet_express_config ,
planet_express_dataset ,
planet_express_icon ,
planet_express_functions ,
):
CONFIG .security .allow_custom_connector_functions = False
mock_all .return_value = [
CustomConnectorTemplate (
key = "planet_express" ,
name = "Planet Express" ,
config = planet_express_config ,
dataset = planet_express_dataset ,
icon = planet_express_icon ,
functions = planet_express_functions ,
)
]
# load custom connector templates from the database
connector_templates = CustomConnectorTemplateLoader .get_connector_templates ()
assert connector_templates == {}
with pytest .raises (NoSuchSaaSRequestOverrideException ):
SaaSRequestOverrideFactory .get_override (
"planet_express_user_access" , SaaSRequestType .READ
)
# assert the strategy was not registered
authentication_strategies = AuthenticationStrategy .get_strategies ()
assert "planet_express" not in
strategy .name for strategy in authentication_strategies
]
@mock .patch (
"fides.api.models.custom_connector_template.CustomConnectorTemplate.all"
)
def test_custom_connector_template_loader_custom_connector_functions_disabled_custom_functions (
self ,
mock_all : MagicMock ,
planet_express_config ,
planet_express_dataset ,
planet_express_icon ,
):
"""
A connector template with no custom functions should still be loaded
even if allow_custom_connector_functions is set to false
"""
CONFIG .security .allow_custom_connector_functions = False
# save custom connector template to the database
mock_all .return_value = [
CustomConnectorTemplate (
key = "planet_express" ,
name = "Planet Express" ,
config = planet_express_config ,
dataset = planet_express_dataset ,
icon = planet_express_icon ,
functions = None ,
)
]
# load custom connector templates from the database
connector_templates = CustomConnectorTemplateLoader .get_connector_templates ()
assert connector_templates == {
"planet_express" : ConnectorTemplate (
config = planet_express_config ,
dataset = planet_express_dataset ,
icon = planet_express_icon ,
human_readable = "Planet Express" ,
)
}
@mock .patch (
"fides.api.models.custom_connector_template.CustomConnectorTemplate.all"
)
Expand All
@@ -294,18 +166,14 @@ def test_custom_connector_template_loader(
planet_express_config ,
planet_express_dataset ,
planet_express_icon ,
planet_express_functions ,
):
CONFIG .security .allow_custom_connector_functions = True
mock_all .return_value = [
CustomConnectorTemplate (
key = "planet_express" ,
name = "Planet Express" ,
config = planet_express_config ,
dataset = planet_express_dataset ,
icon = planet_express_icon ,
functions = planet_express_functions ,
)
]
Expand All
@@ -318,22 +186,10 @@ def test_custom_connector_template_loader(
config = planet_express_config ,
dataset = planet_express_dataset ,
icon = planet_express_icon ,
functions = planet_express_functions ,
human_readable = "Planet Express" ,
)
}
# assert the request override was registered
SaaSRequestOverrideFactory .get_override (
"planet_express_user_access" , SaaSRequestType .READ
)
# assert the strategy was registered
authentication_strategies = AuthenticationStrategy .get_strategies ()
assert "planet_express" in [
strategy .name for strategy in authentication_strategies
]
@mock .patch (
"fides.api.models.custom_connector_template.CustomConnectorTemplate.all"
)
Expand All
@@ -343,18 +199,14 @@ def test_loaders_have_separate_instances(
planet_express_config ,
planet_express_dataset ,
planet_express_icon ,
planet_express_functions ,
):
CONFIG .security .allow_custom_connector_functions = True
mock_all .return_value = [
CustomConnectorTemplate (
key = "planet_express" ,
name = "Planet Express" ,
config = planet_express_config ,
dataset = planet_express_dataset ,
icon = planet_express_icon ,
functions = planet_express_functions ,
)
]
Expand All
@@ -375,7 +227,6 @@ def test_custom_connector_save_template(
planet_express_config ,
planet_express_dataset ,
planet_express_icon ,
planet_express_functions ,
):
db = MagicMock ()
Expand All
@@ -386,7 +237,6 @@ def test_custom_connector_save_template(
{
"config.yml" : planet_express_config ,
"dataset.yml" : planet_express_dataset ,
"functions.py" : planet_express_functions ,
"icon.svg" : planet_express_icon ,
}
)
Expand All
@@ -401,37 +251,55 @@ def test_custom_connector_save_template(
{
"config.yml" : planet_express_config ,
"dataset.yml" : planet_express_dataset ,
"functions.py" : planet_express_functions ,
"icon.svg" : planet_express_icon ,
}
)
),
)
assert mock_create_or_update .call_count == 2
def test_custom_connector_template_loader_disallowed_modules (
@mock .patch (
"fides.api.models.custom_connector_template.CustomConnectorTemplate.create_or_update"
)
def test_custom_connector_save_template_with_functions (
self ,
mock_create_or_update : MagicMock ,
planet_express_config ,
planet_express_dataset ,
planet_express_functions ,
planet_express_icon ,
):
CONFIG .security .allow_custom_connector_functions = True
with pytest .raises (SyntaxError ) as exc :
CustomConnectorTemplateLoader .save_template (
MagicMock (),
ZipFile (
create_zip_file (
{
"config.yml" : planet_express_config ,
"dataset.yml" : planet_express_dataset ,
"functions.py" : "import os" ,
"icon.svg" : planet_express_icon ,
}
)
),
db = MagicMock ()
CustomConnectorTemplateLoader .save_template (
db ,
ZipFile (
create_zip_file (
{
"config.yml" : planet_express_config ,
"dataset.yml" : planet_express_dataset ,
"functions.py" : planet_express_functions ,
"icon.svg" : planet_express_icon ,
}
)
),
)
# assert the request override was ignored
with pytest .raises (NoSuchSaaSRequestOverrideException ) as exc :
SaaSRequestOverrideFactory .get_override (
"planet_express_user_access" , SaaSRequestType .UPDATE
)
assert "Import of 'os' module is not allowed." == str (exc .value )
assert (
f"Custom SaaS override 'planet_express_user_access' does not exist."
in str (exc .value )
)
# assert the strategy was ignored
authentication_strategies = AuthenticationStrategy .get_strategies ()
assert "planet_express" not in
strategy .name for strategy in authentication_strategies
]
@mock .patch (
"fides.api.models.custom_connector_template.CustomConnectorTemplate.delete"
Expand Down
Expand Up
@@ -614,16 +482,3 @@ def test_non_replaceable_template(
config_contents = mock_create_or_update .call_args .kwargs ["data" ]["config" ]
custom_config = load_config_from_string (config_contents )
assert custom_config ["version" ] == "0.0.0"
class TestRegisterCustomFunctions :
def test_function_loader (self ):
"""Verify that all override implementations can be loaded by RestrictedPython"""
overrides_path = "src/fides/api/service/saas_request/override_implementations"
for filename in os .listdir (overrides_path ):
if filename .endswith (".py" ) and filename != "__init__.py" :
file_path = os .path .join (overrides_path , filename )
with open (file_path , "r" ) as file :
register_custom_functions (file .read ())