From bc801fb9ecc7e9a59fb7c1a487f3a3a124444a9f Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Tue, 18 Jun 2024 10:40:04 -0400 Subject: [PATCH 1/8] Framework for fake NodeSettings, including making requests to GravyValet --- osf/external/gravy_valet/auth_helpers.py | 1 + osf/external/gravy_valet/compat.py | 87 +++++++++ osf/external/gravy_valet/gv_mocks.py | 123 ++++++++---- osf/external/gravy_valet/request_helpers.py | 202 ++++++++++++++++++++ osf_tests/test_gv_utils.py | 122 ++++++++---- 5 files changed, 463 insertions(+), 72 deletions(-) create mode 100644 osf/external/gravy_valet/compat.py create mode 100644 osf/external/gravy_valet/request_helpers.py diff --git a/osf/external/gravy_valet/auth_helpers.py b/osf/external/gravy_valet/auth_helpers.py index 5b41fbd977f..174a6098fbc 100644 --- a/osf/external/gravy_valet/auth_helpers.py +++ b/osf/external/gravy_valet/auth_helpers.py @@ -75,6 +75,7 @@ def make_gravy_valet_hmac_headers( request_method: str, body: typing.Union[str, bytes] = '', hmac_key: typing.Optional[str] = None, + additional_header: typing.Optional[dict] = None, requesting_user: typing.Optional[OSFUser] = None, requested_resource: typing.Optional[AbstractNode] = None ) -> dict: diff --git a/osf/external/gravy_valet/compat.py b/osf/external/gravy_valet/compat.py new file mode 100644 index 00000000000..59c7e41e2b4 --- /dev/null +++ b/osf/external/gravy_valet/compat.py @@ -0,0 +1,87 @@ +import enum + +import dataclasses + +from addons.box.apps import BoxAddonAppConfig +from osf.models import Node, OSFUser +from . import request_helpers as gv_requests + + +class _LegacyConfigsForImps(enum.Enum): + """Mapping from a GravyValet StorageImp name to the Addon config.""" + box = BoxAddonAppConfig + + +def make_fake_node_settings(gv_data, requested_resource, requesting_user): + service_wb_key = gv_data.get_nested_attribute( + attribute_path=('base_account', 'external_storage_service'), + attribute_name='wb_key' + ) + legacy_config = _LegacyConfigsForImps(service_wb_key) + return FakeNodeSettings( + config=FakeAddonConfig.from_legacy_config(legacy_config), + gv_id=gv_data.resource_id, + folder_id=gv_data.get_attribute('configured_root_id'), + configured_resource=requested_resource, + active_user=requesting_user, + ) + + +@dataclasses.dataclass +class FakeAddonConfig: + short_name: str + + @classmethod + def from_legacy_config(legacy_config): + return FakeAddonConfig( + short_name=legacy_config.short_name + ) + + +@dataclasses.dataclass +class FakeNodeSettings: + config: FakeAddonConfig + folder_id: str + gv_id: str + configured_resource: Node + active_user: OSFUser + _storage_config: dict | None + _credentials: dict | None + + @property + def short_name(self): + return self.config.short_name + + def serialize_waterbutler_credentials(self): + if self._credentials is None: + self._fetch_wb_config() + return self._credentials + + def serialize_waterbutler_settings(self): + # sufficient for box + return { + 'folder': self.folder_id, + 'service': self.short_name, + } + + def _fetch_wb_config(self): + result = gv_requests.get_waterbutler_config( + gv_addon_pk=self.gv_id, + requested_resource=self.configured_resource, + requesting_user=self.active_user + ) + self._credentials = result.get_attribute('credentials') + self._storage_config = result.get_attribute('settings') + + +@dataclasses.dataclass +class FakeUserSettings: + config: FakeAddonConfig + gv_id: str + + +def _get_service_name_from_gv_data(gv_data): + return gv_data.get_included_attribute('base_account.external_storage_service.wb_key') + +def _get_configured_folder_from_gv_data(gv_data): + return gv_data.get_attribute('root_folder') diff --git a/osf/external/gravy_valet/gv_mocks.py b/osf/external/gravy_valet/gv_mocks.py index 8b29c9f0c9a..8f6ca10880a 100644 --- a/osf/external/gravy_valet/gv_mocks.py +++ b/osf/external/gravy_valet/gv_mocks.py @@ -15,6 +15,8 @@ from website import settings +INCLUDE_REGEX = r'(\?include=(?P.+))?' + class MockGVError(Exception): def __init__(self, status_code, *args, **kwargs): @@ -22,7 +24,7 @@ def __init__(self, status_code, *args, **kwargs): super().__init__(*args, **kwargs) -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class _MockGVEntity: RESOURCE_TYPE: typing.ClassVar[str] @@ -30,7 +32,7 @@ class _MockGVEntity: @property def api_path(self): - return f'v1/{self.RESOURCE_TYPE}/{self.pk}/' + return f'v1/{self.RESOURCE_TYPE}/{self.pk}' def serialize(self): data = { @@ -54,13 +56,13 @@ def _serialize_links(self): return {'self': f'{settings.GRAVYVALET_URL}/{self.api_path}'} def _format_relationship_entry(self, relationship_path, related_type=None, related_pk=None): - relationship_api_path = f'{settings.GRAVYVALET_URL}/{self.api_path}{relationship_path}/' + relationship_api_path = f'{settings.GRAVYVALET_URL}/{self.api_path}/{relationship_path}' relationship_entry = {'links': {'related': relationship_api_path}} if related_type and related_pk: relationship_entry['data'] = {'type': related_type, 'id': related_pk} return relationship_entry -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class _MockUserReference(_MockGVEntity): RESOURCE_TYPE = 'user-references' @@ -73,7 +75,7 @@ def _serialize_relationships(self): accounts_relationship = self._format_relationship_entry(relationship_path='authorized_storage_accounts') return {'authorized_storage_accounts': accounts_relationship} -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class _MockResourceReference(_MockGVEntity): RESOURCE_TYPE = 'resource-references' @@ -86,7 +88,7 @@ def _serialize_relationships(self): configured_addons_relationship = self._format_relationship_entry(relationship_path='configured_storage_addons') return {'configured_storage_addons': configured_addons_relationship} -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class _MockAddonProvider(_MockGVEntity): RESOURCE_TYPE = 'external-storage-services' @@ -113,11 +115,11 @@ def _serialize_relationships(self): } -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class _MockAccount(_MockGVEntity): RESOURCE_TYPE = 'authorized-storage-accounts' - provider: _MockAddonProvider + external_storage_service: _MockAddonProvider account_owner_pk: int display_name: str = '' @@ -128,6 +130,7 @@ def _serialize_attributes(self): 'authorized_capabilities': ['ACCESS', 'UPDATE'], 'authorized_operation_names': ['get_root_items'], 'credentials_available': True, + 'imp_name': 'BLARG', } def _serialize_relationships(self): @@ -140,7 +143,7 @@ def _serialize_relationships(self): 'external_storage_service': self._format_relationship_entry( relationship_path='external_storage_service', related_type=_MockAddonProvider.RESOURCE_TYPE, - related_pk=self.provider.pk + related_pk=self.external_storage_service.pk ), 'configured_storage_addons': self._format_relationship_entry( relationship_path='configured_storage_addons' @@ -150,12 +153,12 @@ def _serialize_relationships(self): ), } -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class _MockAddon(_MockGVEntity): RESOURCE_TYPE = 'configured-storage-addons' resource_pk: int - account: _MockAccount + base_account: _MockAccount display_name: str = '' root_folder: str = '/' @@ -163,11 +166,12 @@ def _serialize_attributes(self): return { 'display_name': self.display_name, 'root_folder': self.root_folder, - 'max_upload_mb': self.account.provider.max_upload_mb, - 'max_concurrent_uploads': self.account.provider.max_concurrent_uploads, - 'icon_url': self.account.provider.icon_url, + 'max_upload_mb': self.base_account.external_storage_service.max_upload_mb, + 'max_concurrent_uploads': self.base_account.external_storage_service.max_concurrent_uploads, + 'icon_url': self.base_account.external_storage_service.icon_url, 'connected_capabilities': ['ACCESS'], 'connected_operation_names': ['get_root_items'], + 'imp_name': 'BLARG', } def _serialize_relationships(self): @@ -180,12 +184,12 @@ def _serialize_relationships(self): 'base_account': self._format_relationship_entry( relationship_path='base_account', related_type=_MockAccount.RESOURCE_TYPE, - related_pk=self.account.pk + related_pk=self.base_account.pk ), 'external_storage_service': self._format_relationship_entry( relationship_path='external_storage_service', related_type=_MockAddonProvider.RESOURCE_TYPE, - related_pk=self.account.provider.pk + related_pk=self.base_account.external_storage_service.pk ), 'connected_operations': self._format_relationship_entry( relationship_path='connected_operations' @@ -196,12 +200,12 @@ def _serialize_relationships(self): class MockGravyValet(): ROUTES = { - r'v1/user-references/((?P\d+)/|(\?filter\[user_uri\]=(?P[^&]+)))$': '_get_user', - r'v1/resource-references/((?P\d+)/|(\?filter\[resource_uri\]=(?P[^&]+)))$': '_get_resource', - r'v1/authorized-storage-accounts/(?P\d+)/$': '_get_account', - r'v1/configured-storage-addons/(?P\d+)/$': '_get_addon', - r'v1/user-references/(?P\d+)/authorized_storage_accounts/(\?include=(?P(\w+,)+))?$': '_get_user_accounts', - r'v1/resource-references/(?P\d+)/configured_storage_addons/(\?include=(?P(\w+,)+))?$': '_get_resource_addons', + r'v1/user-references(/(?P\d+)|(\?filter\[user_uri\]=(?P[^&]+)))$': '_get_user', + r'v1/resource-references(/(?P\d+)|(\?filter\[resource_uri\]=(?P[^&]+)))$': '_get_resource', + r'v1/authorized-storage-accounts/(?P\d+)$': '_get_account', + r'v1/configured-storage-addons/(?P\d+)$': '_get_addon', + r'v1/user-references/(?P\d+)/authorized_storage_accounts$': '_get_user_accounts', + r'v1/resource-references/(?P\d+)/configured_storage_addons$': '_get_resource_addons', } def __init__(self): @@ -273,7 +277,7 @@ def configure_mock_account( new_account = _MockAccount( pk=account_pk, account_owner_pk=user_pk, - provider=connected_provider, + external_storage_service=connected_provider, **account_attrs ) self._user_accounts.setdefault(user_pk, []).append(new_account) @@ -290,7 +294,7 @@ def configure_mock_addon( new_addon = _MockAddon( pk=addon_pk, resource_pk=resource_pk, - account=connected_account, + base_account=connected_account, **config_attrs ) self._resource_addons.setdefault(resource_pk, []).append(new_addon) @@ -316,7 +320,7 @@ def _route_request(self, request): # -> tuple[int, dict, str] status_code = 200 for route_expr, routed_func_name in self.ROUTES.items(): - url_regex = re.compile(f'{re.escape(settings.GRAVYVALET_URL)}/{route_expr}') + url_regex = re.compile(f'{re.escape(settings.GRAVYVALET_URL)}/{route_expr}{INCLUDE_REGEX}') route_match = url_regex.match(urllib.parse.unquote(request.url)) if route_match: func = getattr(self, routed_func_name) @@ -337,6 +341,7 @@ def _get_user( headers: dict, pk=None, # str | None user_uri=None, # str | None + include_param: str = '', ) -> str: if bool(pk) == bool(user_uri): raise MockGVError(HTTPStatus.BAD_REQUEST) @@ -355,7 +360,8 @@ def _get_user( return _format_response_body( data=_MockUserReference(pk=pk, uri=user_uri), - list_view=list_view + list_view=list_view, + include_param=include_param, ) def _get_resource( @@ -363,6 +369,7 @@ def _get_resource( headers: dict, pk=None, # str | None resource_uri=None, # str | None + include_param: str = '', ) -> str: if bool(pk) == bool(resource_uri): raise MockGVError(HTTPStatus.BAD_REQUEST) @@ -381,10 +388,16 @@ def _get_resource( return _format_response_body( data=_MockResourceReference(pk=pk, uri=resource_uri), - list_view=list_view + list_view=list_view, + include_param=include_param, ) - def _get_account(self, headers: dict, pk: str): # -> tuple[int, dict, str] + def _get_account( + self, + headers: dict, + pk: str, + include_param: str = '', + ) -> str: pk = int(pk) account = None for account in itertools.chain.from_iterable(self._user_accounts.values()): @@ -399,9 +412,17 @@ def _get_account(self, headers: dict, pk: str): # -> tuple[int, dict, str] user_uri = self._known_users[account.account_owner_pk] _validate_user(user_uri, headers) - return _format_response_body(data=account, list_view=False) + return _format_response_body( + data=account, + list_view=False, + include_param=include_param, + ) - def _get_addon(self, headers: dict, pk: str): # -> tuple[int, dict, str] + def _get_addon( + self, headers: dict, + pk: str, + include_param: str = '', + ) -> str: pk = int(pk) addon = None for addon in itertools.chain.from_iterable(self._resource_addons.values()): @@ -416,38 +437,51 @@ def _get_addon(self, headers: dict, pk: str): # -> tuple[int, dict, str] resource_uri = self._known_resources[addon.resource_pk] _validate_resource_access(resource_uri, headers) - return _format_response_body(data=addon, list_view=False) + return _format_response_body( + data=addon, + list_view=False, + include_param=include_param, + ) def _get_user_accounts( self, headers: dict, user_pk: str, - includes: str = None + include_param: str = '', ) -> str: user_pk = int(user_pk) if self.validate_headers: user_uri = self._known_users[user_pk] _validate_user(user_uri, headers) - return _format_response_body(data=self._user_accounts.get(user_pk, []), list_view=True) + return _format_response_body( + data=self._user_accounts.get(user_pk, []), + list_view=True, + include_param=include_param + ) def _get_resource_addons( self, headers: dict, resource_pk: str, - includes: str = None + include_param: str = '', ) -> str: resource_pk = int(resource_pk) if self.validate_headers: resource_uri = self._known_resources[resource_pk] _validate_resource_access(resource_uri, headers) - return _format_response_body(data=self._resource_addons.get(resource_pk, []), list_view=True) + return _format_response_body( + data=self._resource_addons.get(resource_pk, []), + include_param=include_param, + list_view=True, + ) def _format_response_body( data, # _MockGVEntity | list[_MockGVEntity] list_view: bool = False, + include_param='', ) -> str: """Returns the expected (status, headers, json) tuple expected by callbacks for MockRequest.""" if list_view: @@ -458,11 +492,28 @@ def _format_response_body( serialized_data = data.serialize() response_dict = { - 'data': serialized_data + 'data': serialized_data, } + if include_param: + response_dict['included'] = _format_includes(data, include_param.split(',')) return json.dumps(response_dict) +def _format_includes(data, includes): + included_data = set() + if not isinstance(data, typing.Iterable): + data = (data,) + for entry in data: + for included_path in includes: + included_members = included_path.split('.') + source_object = entry + for member in included_members: + included_entry = getattr(source_object, member) + included_data.add(included_entry.serialize()) + source_object = included_entry + return json.dumps(list(included_data)) + + def _get_nested_count(d): # dict[Any, Any] -> int: """Get the total number of entries from a dictionary with lists for values.""" return sum(map(len, d.values())) diff --git a/osf/external/gravy_valet/request_helpers.py b/osf/external/gravy_valet/request_helpers.py new file mode 100644 index 00000000000..7d5d7554e11 --- /dev/null +++ b/osf/external/gravy_valet/request_helpers.py @@ -0,0 +1,202 @@ +from urllib.parse import urlencode, urljoin, urlparse, urlunparse + +import dataclasses +import requests + +from . import auth_helpers +from website import settings + + +# Use urljoin here to handle inconsistent use of trailing slash +API_BASE = urljoin(settings.GRAVYVALET_URL, 'v1/') + +# {{placeholder}} format allows f-string to return a formatable string +ACCOUNT_ENDPOINT = f'{API_BASE}authorized-storage-accounts/{{pk}}' +ADDON_ENDPOINT = f'{API_BASE}configured-storage-addons/{{pk}}' +WB_CONFIG_ENDPOINT = f'{ADDON_ENDPOINT}/waterbutler-config' + +USER_FILTER_ENDPOINT = f'{API_BASE}user-references?filter[user_uri]={{uri}}' +USER_DETAIL_ENDPOINT = f'{API_BASE}user-references/{{pk}}' + +RESOURCE_FILTER_ENDPOINT = f'{API_BASE}resource-references?filter[resource_uri]={{uri}}' +RESOURCE_DETAIL_ENDPOINT = f'{API_BASE}resource-references/{{pk}}' + +ACCOUNT_EXTERNAL_SERVICE_PATH = 'external_storage_service' +ADDON_EXTERNAL_SERVICE_PATH = 'base_account.external_storage_service' + + +def get_account(gv_account_pk, requesting_user): # -> JSONAPIResult + return get_gv_result( + endpoint_url=ACCOUNT_ENDPOINT.format(pk=gv_account_pk), + requesting_user=requesting_user, + params={'include': ACCOUNT_EXTERNAL_SERVICE_PATH}, + ) + + +def get_addon(gv_addon_pk, requested_resource, requesting_user): # -> JSONAPIResult + return get_gv_result( + endpoint=ADDON_ENDPOINT.format(pk=gv_addon_pk), + requesting_user=requesting_user, + requested_resource=requested_resource, + params={'include': ADDON_EXTERNAL_SERVICE_PATH}, + ) + + +def iterate_accounts_for_user(requesting_user): # -> typing.Iterator[JSONAPIResult] + user_result = get_gv_result( + endpoint_url=USER_FILTER_ENDPOINT.format(uri=requesting_user.get_semantic_iri()), + requesting_user=requesting_user, + ) + if not user_result: + return None + yield from iterate_gv_results( + endpoint_url=user_result.get_related_link('authorized_storage_accounts'), + requesting_user=requesting_user, + params={'include': ACCOUNT_EXTERNAL_SERVICE_PATH}, + ) + + +def iterate_addons_for_resource(requested_resource, requesting_user): # -> typing.Iterator[JSONAPIResult] + resource_result = get_gv_result( + endpoint_url=RESOURCE_FILTER_ENDPOINT.format(uri=requested_resource.get_semantic_iri()), + requesting_user=requesting_user, + requested_resource=requested_resource, + ) + if not resource_result: + return None + yield from iterate_gv_results( + endpoint_url=resource_result.get_related_link('configured_storage_addons'), + requesting_user=requesting_user, + requested_resource=requested_resource, + params={'include': ADDON_EXTERNAL_SERVICE_PATH} + ) + + +def get_waterbutler_config(gv_addon_pk, requested_resource, requesting_user): # -> JSONAPIResult + return get_gv_result( + endpoint=WB_CONFIG_ENDPOINT.format(pk=gv_addon_pk), + requesting_user=requesting_user, + requested_resource=requested_resource + ) + + +def get_gv_result(**kwargs): # -> JSONAPIResult + '''Return an expected single result from GravyValet, kwargs must match _make_gv_request.''' + response = _make_gv_request(**kwargs) + response_json = response.json() + if not response['data']: + return None + included_entities_lookup = _format_included_entities(response_json.get('included', [])) + return JSONAPIResult(response_json['data'], included_entities_lookup) + + +def iterate_gv_results(**kwargs): # -> typing.Iterator[JSONAPIResult] + '''Iterate through multiple extected results from GravyValet, kwargs must match _make_gv_request.''' + response_json = _make_gv_request(**kwargs).json() + if not response_json['data']: + return # empty iterator + included_entities_lookup = _format_included_entities(response_json.get('included', [])) + yield from (JSONAPIResult(entry, included_entities_lookup) for entry in response_json['data']) + + +def _make_gv_request( + endpoint_url: str, + requesting_user, + requested_resource=None, + request_method='GET', + params: dict = None +): + full_url = urlunparse(urlparse(endpoint_url)._replace(query=urlencode(params))) + auth_headers = auth_helpers.make_gravy_valet_hmac_headers( + request_url=full_url, + request_method=request_method, + additional_headers=auth_helpers._make_permissions_headers( + requesting_user=requesting_user, + requested_resource=requested_resource + ) + ) + response = requests.get(full_url, headers=auth_headers, params=params) + if not response.ok: + # log error to Sentry + pass + return response + + +def _format_included_entities(included_entities_json): + included_entities_by_type_and_id = { + (entity['type'], entity['id']): JSONAPIResult(entity) + for entity in included_entities_json + } + for entity in included_entities_by_type_and_id.values(): + entity._extract_included_relationships(included_entities_by_type_and_id) + return included_entities_by_type_and_id + + +class JSONAPIResult: + + resource_type: str + resource_id: str + _attributes: dict + _relationships: dict # [str, JSONAPIRelationship] + _includes: dict # [str, JSONAPIResult] + + def __init__(self, result_entry: dict, included_entities_lookup: dict = None): + self.resource_type = result_entry['type'] + self.resource_id = result_entry['id'] + self._attributes = dict(result_entry['attributes']) + self._related_links, self._related_ids = _extract_relationships(result_entry['relationships']) + if included_entities_lookup: + self._includes = self._extract_included_relationships(included_entities_lookup) + + def get_attribute(self, attribute_name): + return self._attributes.get(attribute_name) + + def get_related_id(self, related_type): + return self._related_ids.get(related_type) + + def get_related_link(self, relationship_name): + return self._related_links.get(relationship_name) + + def get_included_member(self, relationship_name): + return self._includes.get(relationship_name) + + def get_included_attribute(self, include_path: list, attribute_name: str): + related_object = self + for relationship_name in include_path: + related_object = related_object.get_included_member(relationship_name) + return related_object.get_attribute(attribute_name) + + def extract_included_relationships(self, included_entities_lookup): + for relationship_entry in self._relationships.values(): + if relationship_entry.related_id is None: + continue + included_entity = included_entities_lookup.get( + (relationship_entry.related_type, relationship_entry.related_id) + ) + if included_entity: + self._includes[relationship_entry.relationship_name] = included_entity + + +@dataclasses.dataclass +class JSONAPIRelationship: + relationship_name: str + related_link: str + related_type: str = None + related_id: str = None + + +def _extract_relationships(jsonapi_relationships_data): + relationships_by_name = {} + for relationship_name, relationship_entry in jsonapi_relationships_data.items(): + related_data = relationship_entry.get('data', {}) + related_type = related_data.get('type') + related_id = related_data.get('id') + related_link = relationship_entry['links']['related'] + relationships_by_name[relationship_name] = JSONAPIRelationship( + relationship_name=relationship_name, + related_link=related_link, + related_type=related_type, + related_id=related_id + ) + + return relationships_by_name diff --git a/osf_tests/test_gv_utils.py b/osf_tests/test_gv_utils.py index 15436b689bd..adf88683150 100644 --- a/osf_tests/test_gv_utils.py +++ b/osf_tests/test_gv_utils.py @@ -1,14 +1,17 @@ +import logging import pytest import requests from http import HTTPStatus from osf.external.gravy_valet import ( auth_helpers as gv_auth, - gv_mocks + gv_mocks, + request_helpers as gv_requests ) from osf_tests import factories from website.settings import GRAVYVALET_URL +logger = logging.getLogger(__name__) @pytest.mark.django_db class TestMockGV: @@ -65,7 +68,10 @@ def addon_three(self, project_two, account_one, mock_gv): return mock_gv.configure_mock_addon(project_two, account_one) def test_user_route__pk(self, mock_gv, test_user, account_one): - gv_user_detail_url = f'{GRAVYVALET_URL}/v1/user-references/{account_one.account_owner_pk}/' + gv_user_detail_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=account_one.account_owner_pk) + logger.critical('DEBUG DEBUG DEBUG') + logger.critical(gv_user_detail_url) + logger.critical('\n\n\n') with mock_gv.run_mock(): resp = requests.get(gv_user_detail_url) assert resp.status_code == HTTPStatus.OK @@ -73,13 +79,13 @@ def test_user_route__pk(self, mock_gv, test_user, account_one): assert json_data['id'] == account_one.account_owner_pk assert json_data['attributes']['user_uri'] == test_user.get_semantic_iri() assert json_data['links']['self'] == gv_user_detail_url - expected_accounts_link = f'{gv_user_detail_url}authorized_storage_accounts/' + expected_accounts_link = f'{gv_user_detail_url}/authorized_storage_accounts' retrieved_accounts_link = json_data['relationships']['authorized_storage_accounts']['links']['related'] assert retrieved_accounts_link == expected_accounts_link def test_user_route__filter(self, mock_gv, test_user, account_one): - gv_user_detail_url = f'{GRAVYVALET_URL}/v1/user-references/{account_one.account_owner_pk}/' - gv_user_filtered_list_url = f'{GRAVYVALET_URL}/v1/user-references/?filter[user_uri]={test_user.get_semantic_iri()}' + gv_user_detail_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=account_one.account_owner_pk) + gv_user_filtered_list_url = gv_requests.USER_FILTER_ENDPOINT.format(uri=test_user.get_semantic_iri()) with mock_gv.run_mock(): detail_resp = requests.get(gv_user_detail_url) filtered_list_resp = requests.get(gv_user_filtered_list_url) @@ -87,7 +93,7 @@ def test_user_route__filter(self, mock_gv, test_user, account_one): assert filtered_list_resp.json()['data'][0] == detail_resp.json()['data'] def test_user_route__accounts_link(self, mock_gv, account_one, account_two, account_three): - gv_user_detail_url = f'{GRAVYVALET_URL}/v1/user-references/{account_one.account_owner_pk}/' + gv_user_detail_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=account_one.account_owner_pk) with mock_gv.run_mock(): user_resp = requests.get(gv_user_detail_url) accounts_resp = requests.get( @@ -102,7 +108,7 @@ def test_user_route__accounts_link(self, mock_gv, account_one, account_two, acco assert serialized_account == expected_accounts_by_pk[serialized_account['id']].serialize() def test_resource_route__pk(self, mock_gv, project_one, addon_one): - gv_resource_detail_url = f'{GRAVYVALET_URL}/v1/resource-references/{addon_one.resource_pk}/' + gv_resource_detail_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=addon_one.resource_pk) with mock_gv.run_mock(): resp = requests.get(gv_resource_detail_url) assert resp.status_code == HTTPStatus.OK @@ -110,13 +116,13 @@ def test_resource_route__pk(self, mock_gv, project_one, addon_one): assert json_data['id'] == addon_one.resource_pk assert json_data['attributes']['resource_uri'] == project_one.get_semantic_iri() assert json_data['links']['self'] == gv_resource_detail_url - expected_addons_link = f'{gv_resource_detail_url}configured_storage_addons/' + expected_addons_link = f'{gv_resource_detail_url}/configured_storage_addons' retrieved_addons_link = json_data['relationships']['configured_storage_addons']['links']['related'] assert retrieved_addons_link == expected_addons_link def test_resource_route__filter(self, mock_gv, project_one, addon_one): - gv_resource_detail_url = f'{GRAVYVALET_URL}/v1/resource-references/{addon_one.resource_pk}/' - gv_resource_filtered_list_url = f'{GRAVYVALET_URL}/v1/resource-references/?filter[resource_uri]={project_one.get_semantic_iri()}' + gv_resource_detail_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=addon_one.resource_pk) + gv_resource_filtered_list_url = gv_requests.RESOURCE_FILTER_ENDPOINT.format(uri=project_one.get_semantic_iri()) with mock_gv.run_mock(): detail_resp = requests.get(gv_resource_detail_url) filtered_list_resp = requests.get(gv_resource_filtered_list_url) @@ -125,7 +131,7 @@ def test_resource_route__filter(self, mock_gv, project_one, addon_one): def test_resource_route__addons_link(self, mock_gv, addon_one, addon_two, addon_three): # addon three is the only one connected to project two - gv_resource_detail_url = f'{GRAVYVALET_URL}/v1/resource-references/{addon_three.resource_pk}/' + gv_resource_detail_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=addon_three.resource_pk) with mock_gv.run_mock(): resource_resp = requests.get(gv_resource_detail_url) addons_resp = requests.get( @@ -137,25 +143,26 @@ def test_resource_route__addons_link(self, mock_gv, addon_one, addon_two, addon_ assert json_data[0] == addon_three.serialize() def test_account_route(self, mock_gv, account_one): - gv_account_detail_url = f'{GRAVYVALET_URL}/v1/authorized-storage-accounts/{account_one.pk}/' + gv_account_detail_url = gv_requests.ACCOUNT_ENDPOINT.format(pk=account_one.pk) with mock_gv.run_mock(): resp = requests.get(gv_account_detail_url) assert resp.status_code == HTTPStatus.OK json_data = resp.json()['data'] assert json_data['id'] == account_one.pk assert json_data['relationships']['account_owner']['data']['id'] == account_one.account_owner_pk - assert json_data['relationships']['external_storage_service']['data']['id'] == account_one.provider.pk + assert json_data['relationships']['external_storage_service']['data']['id'] == account_one.external_storage_service.pk def test_addon_route(self, mock_gv, addon_one): - gv_addon_detail_url = f'{GRAVYVALET_URL}/v1/configured-storage-addons/{addon_one.pk}/' + gv_addon_detail_url = gv_requests.ADDON_ENDPOINT.format(pk=addon_one.pk) with mock_gv.run_mock(): resp = requests.get(gv_addon_detail_url) assert resp.status_code == HTTPStatus.OK json_data = resp.json()['data'] assert json_data['id'] == addon_one.pk assert json_data['relationships']['authorized_resource']['data']['id'] == addon_one.resource_pk - assert json_data['relationships']['base_account']['data']['id'] == addon_one.account.pk - assert json_data['relationships']['external_storage_service']['data']['id'] == addon_one.account.provider.pk + assert json_data['relationships']['base_account']['data']['id'] == addon_one.base_account.pk + assert json_data['relationships']['external_storage_service']['data']['id'] == addon_one.base_account.external_storage_service.pk + @pytest.mark.django_db class TestHMACValidation: @@ -190,7 +197,7 @@ def configured_addon(self, mock_gv, resource, external_account): return mock_gv.configure_mock_addon(resource, external_account) def test_validate_headers__bad_key(self, mock_gv, contributor, external_account): - request_url = f'{GRAVYVALET_URL}/v1/user-references/{external_account.account_owner_pk}/' + request_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=external_account.account_owner_pk) auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', @@ -202,14 +209,16 @@ def test_validate_headers__bad_key(self, mock_gv, contributor, external_account) assert resp.status_code == HTTPStatus.FORBIDDEN def test_validate_headers__missing_headers(self, mock_gv, contributor, external_account): - request_url = f'{GRAVYVALET_URL}/v1/user-references/{external_account.account_owner_pk}/' + request_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=external_account.account_owner_pk) + request_url = f'{GRAVYVALET_URL}/v1/user-references/{external_account.account_owner_pk}' with mock_gv.run_mock(): resp = requests.get(request_url) assert resp.status_code == HTTPStatus.UNAUTHORIZED - @pytest.mark.parametrize('subpath', ['', 'authorized_storage_accounts/']) + @pytest.mark.parametrize('subpath', ['', '/authorized_storage_accounts']) def test_validate_user__success(self, mock_gv, contributor, external_account, subpath): - request_url = f'{GRAVYVALET_URL}/v1/user-references/{external_account.account_owner_pk}/{subpath}' + base_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=external_account.account_owner_pk) + request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', @@ -219,9 +228,10 @@ def test_validate_user__success(self, mock_gv, contributor, external_account, su resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.OK - @pytest.mark.parametrize('subpath', ['', 'authorized_storage_accounts/']) + @pytest.mark.parametrize('subpath', ['', '/authorized_storage_accounts']) def test_validate_user__wrong_user(self, mock_gv, noncontributor, external_account, subpath): - request_url = f'{GRAVYVALET_URL}/v1/user-references/{external_account.account_owner_pk}/{subpath}' + base_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=external_account.account_owner_pk) + request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', @@ -231,9 +241,10 @@ def test_validate_user__wrong_user(self, mock_gv, noncontributor, external_accou resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.FORBIDDEN - @pytest.mark.parametrize('subpath', ['', 'authorized_storage_accounts/']) + @pytest.mark.parametrize('subpath', ['', '/authorized_storage_accounts']) def test_validate_user__no_user(self, mock_gv, external_account, subpath): - request_url = f'{GRAVYVALET_URL}/v1/user-references/{external_account.account_owner_pk}/{subpath}' + base_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=external_account.account_owner_pk) + request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', @@ -242,9 +253,10 @@ def test_validate_user__no_user(self, mock_gv, external_account, subpath): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.UNAUTHORIZED - @pytest.mark.parametrize('subpath', ['', 'configured_storage_addons/']) + @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) def test_validate_resource__success(self, mock_gv, contributor, resource, configured_addon, subpath): - request_url = f'{GRAVYVALET_URL}/v1/resource-references/{configured_addon.resource_pk}/{subpath}' + base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) + request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', @@ -255,9 +267,10 @@ def test_validate_resource__success(self, mock_gv, contributor, resource, config resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.OK - @pytest.mark.parametrize('subpath', ['', 'configured_storage_addons/']) + @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) def test_validate_resource__wrong_resource(self, mock_gv, contributor, configured_addon, subpath): - request_url = f'{GRAVYVALET_URL}/v1/resource-references/{configured_addon.resource_pk}/{subpath}' + base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) + request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', @@ -268,11 +281,12 @@ def test_validate_resource__wrong_resource(self, mock_gv, contributor, configure resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.BAD_REQUEST - @pytest.mark.parametrize('subpath', ['', 'configured_storage_addons/']) + @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) def test_validate_resource__noncontributor__public_resource(self, mock_gv, noncontributor, resource, configured_addon, subpath): resource.is_public = True resource.save() - request_url = f'{GRAVYVALET_URL}/v1/resource-references/{configured_addon.resource_pk}/{subpath}' + base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) + request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', @@ -283,11 +297,12 @@ def test_validate_resource__noncontributor__public_resource(self, mock_gv, nonco resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.OK - @pytest.mark.parametrize('subpath', ['', 'configured_storage_addons/']) + @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) def test_validate_resource__noncontributor__private_resource(self, mock_gv, noncontributor, resource, configured_addon, subpath): resource.is_public = False resource.save() - request_url = f'{GRAVYVALET_URL}/v1/resource-references/{configured_addon.resource_pk}/{subpath}' + base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) + request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', @@ -298,11 +313,12 @@ def test_validate_resource__noncontributor__private_resource(self, mock_gv, nonc resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.FORBIDDEN - @pytest.mark.parametrize('subpath', ['', 'configured_storage_addons/']) + @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) def test_validate_resource__unauthenticated_user__public_resource(self, mock_gv, resource, configured_addon, subpath): resource.is_public = True resource.save() - request_url = f'{GRAVYVALET_URL}/v1/resource-references/{configured_addon.resource_pk}/{subpath}' + base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) + request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', @@ -312,11 +328,12 @@ def test_validate_resource__unauthenticated_user__public_resource(self, mock_gv, resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.OK - @pytest.mark.parametrize('subpath', ['', 'configured_storage_addons/']) + @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) def test_validate_resource__unauthenticated_user__private_resource(self, mock_gv, resource, configured_addon, subpath): resource.is_public = False resource.save() - request_url = f'{GRAVYVALET_URL}/v1/resource-references/{configured_addon.resource_pk}/{subpath}' + base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) + request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', @@ -325,3 +342,36 @@ def test_validate_resource__unauthenticated_user__private_resource(self, mock_gv with mock_gv.run_mock(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.UNAUTHORIZED + + +@pytest.mark.django_db +class TestRequestHelpers: + + @pytest.fixture + def mock_gv(self): + #validate_headers == True by default + return gv_mocks.MockGravyValet() + + @pytest.fixture + def contributor(self): + return factories.AuthUserFactory() + + @pytest.fixture + def noncontributor(self): + return factories.AuthUserFactory() + + @pytest.fixture + def resource(self, contributor): + return factories.ProjectFactory(creator=contributor) + + @pytest.fixture + def external_service(self, mock_gv): + return mock_gv.configure_mock_provider('blarg') + + @pytest.fixture + def external_account(self, mock_gv, contributor, external_service): + return mock_gv.configure_mock_account(contributor, external_service.name) + + @pytest.fixture + def configured_addon(self, mock_gv, resource, external_account): + return mock_gv.configure_mock_addon(resource, external_account) From 900891bd64f2f251ead42144b46f5a1c32899ca2 Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Tue, 18 Jun 2024 11:07:52 -0400 Subject: [PATCH 2/8] Fix outdated references to `_relationship_links` and `_relationsihp_ids`, better documentation --- osf/external/gravy_valet/request_helpers.py | 57 ++++++++++++++------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/osf/external/gravy_valet/request_helpers.py b/osf/external/gravy_valet/request_helpers.py index 7d5d7554e11..b2e876f2975 100644 --- a/osf/external/gravy_valet/request_helpers.py +++ b/osf/external/gravy_valet/request_helpers.py @@ -25,7 +25,8 @@ ADDON_EXTERNAL_SERVICE_PATH = 'base_account.external_storage_service' -def get_account(gv_account_pk, requesting_user): # -> JSONAPIResult +def get_account(gv_account_pk, requesting_user): # -> JSONAPIResultEntry + '''Return a JSONAPIResultEntry representing a known AuthorizedStorageAccount.''' return get_gv_result( endpoint_url=ACCOUNT_ENDPOINT.format(pk=gv_account_pk), requesting_user=requesting_user, @@ -33,7 +34,8 @@ def get_account(gv_account_pk, requesting_user): # -> JSONAPIResult ) -def get_addon(gv_addon_pk, requested_resource, requesting_user): # -> JSONAPIResult +def get_addon(gv_addon_pk, requested_resource, requesting_user): # -> JSONAPIResultEntry + '''Return a JSONAPIResultEntry representing a known ConfiguredStorageAddon.''' return get_gv_result( endpoint=ADDON_ENDPOINT.format(pk=gv_addon_pk), requesting_user=requesting_user, @@ -42,7 +44,8 @@ def get_addon(gv_addon_pk, requested_resource, requesting_user): # -> JSONAPIRe ) -def iterate_accounts_for_user(requesting_user): # -> typing.Iterator[JSONAPIResult] +def iterate_accounts_for_user(requesting_user): # -> typing.Iterator[JSONAPIResultEntry] + '''Returns an iterator of JSONAPIResultEntries representing all of the AuthorizedStorageAccounts for a user.''' user_result = get_gv_result( endpoint_url=USER_FILTER_ENDPOINT.format(uri=requesting_user.get_semantic_iri()), requesting_user=requesting_user, @@ -56,7 +59,8 @@ def iterate_accounts_for_user(requesting_user): # -> typing.Iterator[JSONAPIRes ) -def iterate_addons_for_resource(requested_resource, requesting_user): # -> typing.Iterator[JSONAPIResult] +def iterate_addons_for_resource(requested_resource, requesting_user): # -> typing.Iterator[JSONAPIResultEntry] + '''Returns an iterator of JSONAPIResultEntires representing all of the ConfiguredStorageAddons for a resource.''' resource_result = get_gv_result( endpoint_url=RESOURCE_FILTER_ENDPOINT.format(uri=requested_resource.get_semantic_iri()), requesting_user=requesting_user, @@ -72,7 +76,7 @@ def iterate_addons_for_resource(requested_resource, requesting_user): # -> typi ) -def get_waterbutler_config(gv_addon_pk, requested_resource, requesting_user): # -> JSONAPIResult +def get_waterbutler_config(gv_addon_pk, requested_resource, requesting_user): # -> JSONAPIResultEntry return get_gv_result( endpoint=WB_CONFIG_ENDPOINT.format(pk=gv_addon_pk), requesting_user=requesting_user, @@ -80,23 +84,30 @@ def get_waterbutler_config(gv_addon_pk, requested_resource, requesting_user): # ) -def get_gv_result(**kwargs): # -> JSONAPIResult - '''Return an expected single result from GravyValet, kwargs must match _make_gv_request.''' +def get_gv_result(**kwargs): # -> JSONAPIResultEntry + '''Processes the result of a request to a GravyValet detail endpoint into a JSONAPIResultEntry. + + kwargs must match _make_gv_request + ''' response = _make_gv_request(**kwargs) response_json = response.json() if not response['data']: return None included_entities_lookup = _format_included_entities(response_json.get('included', [])) - return JSONAPIResult(response_json['data'], included_entities_lookup) + return JSONAPIResultEntry(response_json['data'], included_entities_lookup) + + +def iterate_gv_results(**kwargs): # -> typing.Iterator[JSONAPIResultEntry] + '''Processes the result of a request to GravyValet list endpoint into a generator of JSONAPIResultEntires. + kwargs must match _make_gv_request + ''' -def iterate_gv_results(**kwargs): # -> typing.Iterator[JSONAPIResult] - '''Iterate through multiple extected results from GravyValet, kwargs must match _make_gv_request.''' response_json = _make_gv_request(**kwargs).json() if not response_json['data']: return # empty iterator included_entities_lookup = _format_included_entities(response_json.get('included', [])) - yield from (JSONAPIResult(entry, included_entities_lookup) for entry in response_json['data']) + yield from (JSONAPIResultEntry(entry, included_entities_lookup) for entry in response_json['data']) def _make_gv_request( @@ -106,6 +117,7 @@ def _make_gv_request( request_method='GET', params: dict = None ): + '''Generates HMAC-Signed auth headers and makes a request to GravyValet, returning the result.''' full_url = urlunparse(urlparse(endpoint_url)._replace(query=urlencode(params))) auth_headers = auth_helpers.make_gravy_valet_hmac_headers( request_url=full_url, @@ -123,8 +135,13 @@ def _make_gv_request( def _format_included_entities(included_entities_json): + '''Processes all entries of a JSONAPI `include` element into JSONAPIResultEntries. + + Returns a dictionary of JSONAPIResultEntires keyed by type and id for easy lookup + and linking. Also links these entires in the case of nested Includes. + ''' included_entities_by_type_and_id = { - (entity['type'], entity['id']): JSONAPIResult(entity) + (entity['type'], entity['id']): JSONAPIResultEntry(entity) for entity in included_entities_json } for entity in included_entities_by_type_and_id.values(): @@ -132,30 +149,31 @@ def _format_included_entities(included_entities_json): return included_entities_by_type_and_id -class JSONAPIResult: - +class JSONAPIResultEntry: resource_type: str resource_id: str _attributes: dict + # JSONAPIRelationships keyed by relationship name _relationships: dict # [str, JSONAPIRelationship] - _includes: dict # [str, JSONAPIResult] + # Included JSONAPIResultEntries, keyed by relationship name + _includes: dict # [str, JSONAPIResultEntry] def __init__(self, result_entry: dict, included_entities_lookup: dict = None): self.resource_type = result_entry['type'] self.resource_id = result_entry['id'] self._attributes = dict(result_entry['attributes']) - self._related_links, self._related_ids = _extract_relationships(result_entry['relationships']) + self._relationships = _extract_relationships(result_entry['relationships']) if included_entities_lookup: self._includes = self._extract_included_relationships(included_entities_lookup) def get_attribute(self, attribute_name): return self._attributes.get(attribute_name) - def get_related_id(self, related_type): - return self._related_ids.get(related_type) + def get_related_id(self, relationship_name): + return self._relationships(relationship_name).related_id def get_related_link(self, relationship_name): - return self._related_links.get(relationship_name) + return self._relationships[relationship_name].related_link def get_included_member(self, relationship_name): return self._includes.get(relationship_name) @@ -186,6 +204,7 @@ class JSONAPIRelationship: def _extract_relationships(jsonapi_relationships_data): + '''Converts the `relationship entrie from a JSONAPI into a dict of JSONAPIRelationships keyed by name.''' relationships_by_name = {} for relationship_name, relationship_entry in jsonapi_relationships_data.items(): related_data = relationship_entry.get('data', {}) From 6b8b975b94f9345b7d27f11c405b8672eab7d38e Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Tue, 18 Jun 2024 11:11:52 -0400 Subject: [PATCH 3/8] mock -> fake --- osf/external/gravy_valet/gv_mocks.py | 553 --------------------------- osf_tests/test_gv_utils.py | 152 ++++---- 2 files changed, 76 insertions(+), 629 deletions(-) delete mode 100644 osf/external/gravy_valet/gv_mocks.py diff --git a/osf/external/gravy_valet/gv_mocks.py b/osf/external/gravy_valet/gv_mocks.py deleted file mode 100644 index 8f6ca10880a..00000000000 --- a/osf/external/gravy_valet/gv_mocks.py +++ /dev/null @@ -1,553 +0,0 @@ -import contextlib -import itertools -import json -import typing -import re -import urllib.parse -from http import HTTPStatus - -import dataclasses # backport -import responses - -from . import auth_helpers -from osf.models import OSFUser, AbstractNode -from osf.utils import permissions as osf_permissions -from website import settings - - -INCLUDE_REGEX = r'(\?include=(?P.+))?' - -class MockGVError(Exception): - - def __init__(self, status_code, *args, **kwargs): - self.status_code = status_code - super().__init__(*args, **kwargs) - - -@dataclasses.dataclass(frozen=True) -class _MockGVEntity: - - RESOURCE_TYPE: typing.ClassVar[str] - pk: int - - @property - def api_path(self): - return f'v1/{self.RESOURCE_TYPE}/{self.pk}' - - def serialize(self): - data = { - 'type': self.RESOURCE_TYPE, - 'id': self.pk, - 'attributes': self._serialize_attributes(), - 'links': self._serialize_links(), - } - relationships = self._serialize_relationships() - if relationships: - data['relationships'] = relationships - return data - - def _serialize_attributes(self): - ... - - def _serialize_relationships(self): - ... - - def _serialize_links(self): - return {'self': f'{settings.GRAVYVALET_URL}/{self.api_path}'} - - def _format_relationship_entry(self, relationship_path, related_type=None, related_pk=None): - relationship_api_path = f'{settings.GRAVYVALET_URL}/{self.api_path}/{relationship_path}' - relationship_entry = {'links': {'related': relationship_api_path}} - if related_type and related_pk: - relationship_entry['data'] = {'type': related_type, 'id': related_pk} - return relationship_entry - -@dataclasses.dataclass(frozen=True) -class _MockUserReference(_MockGVEntity): - - RESOURCE_TYPE = 'user-references' - uri: str - - def _serialize_attributes(self): - return {'user_uri': self.uri} - - def _serialize_relationships(self): - accounts_relationship = self._format_relationship_entry(relationship_path='authorized_storage_accounts') - return {'authorized_storage_accounts': accounts_relationship} - -@dataclasses.dataclass(frozen=True) -class _MockResourceReference(_MockGVEntity): - - RESOURCE_TYPE = 'resource-references' - uri: str - - def _serialize_attributes(self): - return {'resource_uri': self.uri} - - def _serialize_relationships(self): - configured_addons_relationship = self._format_relationship_entry(relationship_path='configured_storage_addons') - return {'configured_storage_addons': configured_addons_relationship} - -@dataclasses.dataclass(frozen=True) -class _MockAddonProvider(_MockGVEntity): - - RESOURCE_TYPE = 'external-storage-services' - name: str - max_upload_mb: int = 2**10 - max_concurrent_uploads: int = -5 - icon_url: str = 'vetted-url-for-icon.png' - - def _serialize_attributes(self): - return { - 'name': self.name, - 'max_upload_mb': self.max_upload_mb, - 'max_concurrent_uploads': self.max_concurrent_uploads, - 'configurable_api_root': False, - 'terms_of_service_features': [], - 'icon_url': self.icon_url, - } - - def _serialize_relationships(self): - return { - 'addon_imp': self._format_relationship_entry( - relationship_path='addon_imp', related_type='addon-imps', related_pk=1 - ) - } - - -@dataclasses.dataclass(frozen=True) -class _MockAccount(_MockGVEntity): - - RESOURCE_TYPE = 'authorized-storage-accounts' - external_storage_service: _MockAddonProvider - account_owner_pk: int - display_name: str = '' - - def _serialize_attributes(self): - return { - 'display_name': self.display_name, - 'authorized_scopes': ['all_of_the_scopes'], - 'authorized_capabilities': ['ACCESS', 'UPDATE'], - 'authorized_operation_names': ['get_root_items'], - 'credentials_available': True, - 'imp_name': 'BLARG', - } - - def _serialize_relationships(self): - return { - 'account_owner': self._format_relationship_entry( - relationship_path='account_owner', - related_type=_MockUserReference.RESOURCE_TYPE, - related_pk=self.account_owner_pk - ), - 'external_storage_service': self._format_relationship_entry( - relationship_path='external_storage_service', - related_type=_MockAddonProvider.RESOURCE_TYPE, - related_pk=self.external_storage_service.pk - ), - 'configured_storage_addons': self._format_relationship_entry( - relationship_path='configured_storage_addons' - ), - 'authorized_operations': self._format_relationship_entry( - relationship_path='authorized_operations' - ), - } - -@dataclasses.dataclass(frozen=True) -class _MockAddon(_MockGVEntity): - - RESOURCE_TYPE = 'configured-storage-addons' - resource_pk: int - base_account: _MockAccount - display_name: str = '' - root_folder: str = '/' - - def _serialize_attributes(self): - return { - 'display_name': self.display_name, - 'root_folder': self.root_folder, - 'max_upload_mb': self.base_account.external_storage_service.max_upload_mb, - 'max_concurrent_uploads': self.base_account.external_storage_service.max_concurrent_uploads, - 'icon_url': self.base_account.external_storage_service.icon_url, - 'connected_capabilities': ['ACCESS'], - 'connected_operation_names': ['get_root_items'], - 'imp_name': 'BLARG', - } - - def _serialize_relationships(self): - return { - 'authorized_resource': self._format_relationship_entry( - relationship_path='authorized_resource', - related_type=_MockResourceReference.RESOURCE_TYPE, - related_pk=self.resource_pk - ), - 'base_account': self._format_relationship_entry( - relationship_path='base_account', - related_type=_MockAccount.RESOURCE_TYPE, - related_pk=self.base_account.pk - ), - 'external_storage_service': self._format_relationship_entry( - relationship_path='external_storage_service', - related_type=_MockAddonProvider.RESOURCE_TYPE, - related_pk=self.base_account.external_storage_service.pk - ), - 'connected_operations': self._format_relationship_entry( - relationship_path='connected_operations' - ), - } - - -class MockGravyValet(): - - ROUTES = { - r'v1/user-references(/(?P\d+)|(\?filter\[user_uri\]=(?P[^&]+)))$': '_get_user', - r'v1/resource-references(/(?P\d+)|(\?filter\[resource_uri\]=(?P[^&]+)))$': '_get_resource', - r'v1/authorized-storage-accounts/(?P\d+)$': '_get_account', - r'v1/configured-storage-addons/(?P\d+)$': '_get_addon', - r'v1/user-references/(?P\d+)/authorized_storage_accounts$': '_get_user_accounts', - r'v1/resource-references/(?P\d+)/configured_storage_addons$': '_get_resource_addons', - } - - def __init__(self): - self._clear_mappings() - self._validate_headers = True - - @property - def validate_headers(self) -> bool: - return self._validate_headers - - @validate_headers.setter - def validate_headers(self, value: bool): - if not isinstance(value, bool): - raise ValueError('validate_headers must be a boolean value') - self._validate_headers = value - - def _clear_mappings(self, include_providers: bool = True): - """Reset all configured users/resources/acounts/addons and, optionally, providers.""" - if include_providers: - # Mapping from _MockAddonProvider name to _MockAddonProvider - self._known_providers = {} - # Bidirectional mapping between user uri and mock "pk" - self._known_users = {} - # Bidirectional mapping between resource uri and mock "pk" - self._known_resources = {} - # Mapping from user "pk" to _MockAccounts for the user - self._user_accounts = {} - # Mapping from resource "pk" to _MockAddons "configured" on the resource - self._resource_addons = {} - - def _get_or_create_user_entry(self, user: OSFUser): - user_uri = user.get_semantic_iri() - user_pk = self._known_users.get(user_uri) - if not user_pk: - user_pk = len(self._known_users) + 1 - self._known_users[user_uri] = user_pk - self._known_users[user_pk] = user_uri - return user_uri, user_pk - - def _get_or_create_resource_entry(self, resource: AbstractNode): - resource_uri = resource.get_semantic_iri() - resource_pk = self._known_resources.get(resource_uri) - if not resource_pk: - resource_pk = len(self._known_resources) + 1 - self._known_resources[resource_uri] = resource_pk - self._known_resources[resource_pk] = resource_uri - return resource_uri, resource_pk - - def configure_mock_provider(self, provider_name: str, **service_attrs) -> _MockAddonProvider: - known_provider = self._known_providers.get(provider_name) - provider_pk = known_provider.pk if known_provider else len(self._known_providers) + 1 - new_provider = _MockAddonProvider( - name=provider_name, - pk=provider_pk, - **service_attrs - ) - self._known_providers[provider_name] = new_provider - return new_provider - - def configure_mock_account( - self, - user: OSFUser, - addon_name: str, - **account_attrs - ) -> _MockAccount: - user_uri, user_pk = self._get_or_create_user_entry(user) - account_pk = _get_nested_count(self._user_accounts) + 1 - connected_provider = self._known_providers[addon_name] - new_account = _MockAccount( - pk=account_pk, - account_owner_pk=user_pk, - external_storage_service=connected_provider, - **account_attrs - ) - self._user_accounts.setdefault(user_pk, []).append(new_account) - return new_account - - def configure_mock_addon( - self, - resource: AbstractNode, - connected_account: _MockAccount, - **config_attrs - ) -> _MockAddon: - resource_uri, resource_pk = self._get_or_create_resource_entry(resource) - addon_pk = _get_nested_count(self._resource_addons) + 1 - new_addon = _MockAddon( - pk=addon_pk, - resource_pk=resource_pk, - base_account=connected_account, - **config_attrs - ) - self._resource_addons.setdefault(resource_pk, []).append(new_addon) - return new_addon - - @contextlib.contextmanager - def run_mock(self): - with responses.RequestsMock() as requests_mock: - requests_mock.add_callback( - responses.GET, - re.compile(f'{re.escape(settings.GRAVYVALET_URL)}.*'), - callback=self._route_request, - content_type='application/json', - ) - yield requests_mock - - def _route_request(self, request): # -> tuple[int, dict, str] - if self.validate_headers: - try: - _validate_request(request) - except MockGVError as e: - return (e.status_code, {}, '') - - status_code = 200 - for route_expr, routed_func_name in self.ROUTES.items(): - url_regex = re.compile(f'{re.escape(settings.GRAVYVALET_URL)}/{route_expr}{INCLUDE_REGEX}') - route_match = url_regex.match(urllib.parse.unquote(request.url)) - if route_match: - func = getattr(self, routed_func_name) - try: - body = func(headers=request.headers, **route_match.groupdict()) - except KeyError: # entity lookup failed somewhere - status_code = HTTPStatus.NOT_FOUND - body = '' - except MockGVError as e: - status_code = e.status_code - body = '' - return (status_code, {}, body) - - return (HTTPStatus.NOT_FOUND, {}, '') - - def _get_user( - self, - headers: dict, - pk=None, # str | None - user_uri=None, # str | None - include_param: str = '', - ) -> str: - if bool(pk) == bool(user_uri): - raise MockGVError(HTTPStatus.BAD_REQUEST) - - # if passed the user_uri, call came through list endpoint with filter - if user_uri: - list_view = True - pk = self._known_users[user_uri] - else: - list_view = False - pk = int(pk) - user_uri = self._known_users[pk] - - if self.validate_headers: - _validate_user(user_uri, headers) - - return _format_response_body( - data=_MockUserReference(pk=pk, uri=user_uri), - list_view=list_view, - include_param=include_param, - ) - - def _get_resource( - self, - headers: dict, - pk=None, # str | None - resource_uri=None, # str | None - include_param: str = '', - ) -> str: - if bool(pk) == bool(resource_uri): - raise MockGVError(HTTPStatus.BAD_REQUEST) - - # if passed the resource_uri, call came through list endpoint with filter - if resource_uri: - list_view = True - pk = self._known_resources[resource_uri] - else: - list_view = False - pk = int(pk) - resource_uri = self._known_resources[pk] - - if self.validate_headers: - _validate_resource_access(resource_uri, headers) - - return _format_response_body( - data=_MockResourceReference(pk=pk, uri=resource_uri), - list_view=list_view, - include_param=include_param, - ) - - def _get_account( - self, - headers: dict, - pk: str, - include_param: str = '', - ) -> str: - pk = int(pk) - account = None - for account in itertools.chain.from_iterable(self._user_accounts.values()): - if account.pk == pk: - account = account - break - - if not account: - raise MockGVError(HTTPStatus.NOT_FOUND) - - if self.validate_headers: - user_uri = self._known_users[account.account_owner_pk] - _validate_user(user_uri, headers) - - return _format_response_body( - data=account, - list_view=False, - include_param=include_param, - ) - - def _get_addon( - self, headers: dict, - pk: str, - include_param: str = '', - ) -> str: - pk = int(pk) - addon = None - for addon in itertools.chain.from_iterable(self._resource_addons.values()): - if addon.pk == pk: - addon = addon - break - - if not addon: - raise MockGVError(HTTPStatus.NOT_FOUND) - - if self.validate_headers: - resource_uri = self._known_resources[addon.resource_pk] - _validate_resource_access(resource_uri, headers) - - return _format_response_body( - data=addon, - list_view=False, - include_param=include_param, - ) - - def _get_user_accounts( - self, - headers: dict, - user_pk: str, - include_param: str = '', - ) -> str: - user_pk = int(user_pk) - if self.validate_headers: - user_uri = self._known_users[user_pk] - _validate_user(user_uri, headers) - - return _format_response_body( - data=self._user_accounts.get(user_pk, []), - list_view=True, - include_param=include_param - ) - - def _get_resource_addons( - self, - headers: dict, - resource_pk: str, - include_param: str = '', - ) -> str: - resource_pk = int(resource_pk) - if self.validate_headers: - resource_uri = self._known_resources[resource_pk] - _validate_resource_access(resource_uri, headers) - - return _format_response_body( - data=self._resource_addons.get(resource_pk, []), - include_param=include_param, - list_view=True, - ) - - -def _format_response_body( - data, # _MockGVEntity | list[_MockGVEntity] - list_view: bool = False, - include_param='', -) -> str: - """Returns the expected (status, headers, json) tuple expected by callbacks for MockRequest.""" - if list_view: - if not isinstance(data, list): - data = [data] - serialized_data = [entry.serialize() for entry in data] - else: - serialized_data = data.serialize() - - response_dict = { - 'data': serialized_data, - } - if include_param: - response_dict['included'] = _format_includes(data, include_param.split(',')) - return json.dumps(response_dict) - - -def _format_includes(data, includes): - included_data = set() - if not isinstance(data, typing.Iterable): - data = (data,) - for entry in data: - for included_path in includes: - included_members = included_path.split('.') - source_object = entry - for member in included_members: - included_entry = getattr(source_object, member) - included_data.add(included_entry.serialize()) - source_object = included_entry - return json.dumps(list(included_data)) - - -def _get_nested_count(d): # dict[Any, Any] -> int: - """Get the total number of entries from a dictionary with lists for values.""" - return sum(map(len, d.values())) - - -def _validate_request(request): - try: - auth_helpers.validate_signed_headers(request) - except ValueError: - error_code = ( - HTTPStatus.FORBIDDEN - if request.headers.get(auth_helpers.USER_HEADER) - else HTTPStatus.UNAUTHORIZED - ) - raise MockGVError(error_code) - - -def _validate_user(requested_user_uri, headers): - requesting_user_uri = headers.get(auth_helpers.USER_HEADER) - if requesting_user_uri is None: - raise MockGVError(HTTPStatus.UNAUTHORIZED) - if requesting_user_uri != requested_user_uri: - raise MockGVError(HTTPStatus.FORBIDDEN) - - -def _validate_resource_access(requested_resource_uri, headers): - headers_requested_resource = headers.get(auth_helpers.RESOURCE_HEADER) - # generously assume malformed request on mismatch between headers and request - if not headers_requested_resource or headers_requested_resource != requested_resource_uri: - raise MockGVError(HTTPStatus.BAD_REQUEST) - requesting_user_uri = headers.get(auth_helpers.USER_HEADER) - permission_denied_error_code = ( - HTTPStatus.FORBIDDEN if requesting_user_uri else HTTPStatus.UNAUTHORIZED - ) - resource_permissions = headers.get(auth_helpers.PERMISSIONS_HEADER, '').split(';') - if osf_permissions.READ not in resource_permissions: - raise MockGVError(permission_denied_error_code) diff --git a/osf_tests/test_gv_utils.py b/osf_tests/test_gv_utils.py index adf88683150..b10113b2ef0 100644 --- a/osf_tests/test_gv_utils.py +++ b/osf_tests/test_gv_utils.py @@ -5,7 +5,7 @@ from osf.external.gravy_valet import ( auth_helpers as gv_auth, - gv_mocks, + gv_fakes, request_helpers as gv_requests ) from osf_tests import factories @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) @pytest.mark.django_db -class TestMockGV: +class TestFakeGV: @pytest.fixture def test_user(self): @@ -29,50 +29,50 @@ def project_two(self, test_user): return factories.ProjectFactory(creator=test_user) @pytest.fixture - def mock_gv(self, test_user, project_one, project_two): - mock_gv = gv_mocks.MockGravyValet() - mock_gv.validate_headers = False - return mock_gv + def fake_gv(self, test_user, project_one, project_two): + fake_gv = gv_fakes.FakeGravyValet() + fake_gv.validate_headers = False + return fake_gv @pytest.fixture - def provider_one(self, mock_gv): - return mock_gv.configure_mock_provider(provider_name='foo') + def provider_one(self, fake_gv): + return fake_gv.configure_fake_provider(provider_name='foo') @pytest.fixture - def provider_two(self, mock_gv): - return mock_gv.configure_mock_provider(provider_name='bar') + def provider_two(self, fake_gv): + return fake_gv.configure_fake_provider(provider_name='bar') @pytest.fixture - def account_one(self, test_user, provider_one, mock_gv): - return mock_gv.configure_mock_account(test_user, provider_one.name) + def account_one(self, test_user, provider_one, fake_gv): + return fake_gv.configure_fake_account(test_user, provider_one.name) @pytest.fixture - def account_two(self, test_user, provider_two, mock_gv): - return mock_gv.configure_mock_account(test_user, provider_two.name) + def account_two(self, test_user, provider_two, fake_gv): + return fake_gv.configure_fake_account(test_user, provider_two.name) @pytest.fixture - def account_three(self, provider_one, mock_gv): + def account_three(self, provider_one, fake_gv): account_owner = factories.AuthUserFactory() - return mock_gv.configure_mock_account(account_owner, provider_one.name) + return fake_gv.configure_fake_account(account_owner, provider_one.name) @pytest.fixture - def addon_one(self, project_one, account_one, mock_gv): - return mock_gv.configure_mock_addon(project_one, account_one) + def addon_one(self, project_one, account_one, fake_gv): + return fake_gv.configure_fake_addon(project_one, account_one) @pytest.fixture - def addon_two(self, project_one, account_two, mock_gv): - return mock_gv.configure_mock_addon(project_one, account_two) + def addon_two(self, project_one, account_two, fake_gv): + return fake_gv.configure_fake_addon(project_one, account_two) @pytest.fixture - def addon_three(self, project_two, account_one, mock_gv): - return mock_gv.configure_mock_addon(project_two, account_one) + def addon_three(self, project_two, account_one, fake_gv): + return fake_gv.configure_fake_addon(project_two, account_one) - def test_user_route__pk(self, mock_gv, test_user, account_one): + def test_user_route__pk(self, fake_gv, test_user, account_one): gv_user_detail_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=account_one.account_owner_pk) logger.critical('DEBUG DEBUG DEBUG') logger.critical(gv_user_detail_url) logger.critical('\n\n\n') - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(gv_user_detail_url) assert resp.status_code == HTTPStatus.OK json_data = resp.json()['data'] @@ -83,18 +83,18 @@ def test_user_route__pk(self, mock_gv, test_user, account_one): retrieved_accounts_link = json_data['relationships']['authorized_storage_accounts']['links']['related'] assert retrieved_accounts_link == expected_accounts_link - def test_user_route__filter(self, mock_gv, test_user, account_one): + def test_user_route__filter(self, fake_gv, test_user, account_one): gv_user_detail_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=account_one.account_owner_pk) gv_user_filtered_list_url = gv_requests.USER_FILTER_ENDPOINT.format(uri=test_user.get_semantic_iri()) - with mock_gv.run_mock(): + with fake_gv.run_fake(): detail_resp = requests.get(gv_user_detail_url) filtered_list_resp = requests.get(gv_user_filtered_list_url) assert filtered_list_resp.status_code == HTTPStatus.OK assert filtered_list_resp.json()['data'][0] == detail_resp.json()['data'] - def test_user_route__accounts_link(self, mock_gv, account_one, account_two, account_three): + def test_user_route__accounts_link(self, fake_gv, account_one, account_two, account_three): gv_user_detail_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=account_one.account_owner_pk) - with mock_gv.run_mock(): + with fake_gv.run_fake(): user_resp = requests.get(gv_user_detail_url) accounts_resp = requests.get( user_resp.json()['data']['relationships']['authorized_storage_accounts']['links']['related'] @@ -107,9 +107,9 @@ def test_user_route__accounts_link(self, mock_gv, account_one, account_two, acco for serialized_account in json_data: assert serialized_account == expected_accounts_by_pk[serialized_account['id']].serialize() - def test_resource_route__pk(self, mock_gv, project_one, addon_one): + def test_resource_route__pk(self, fake_gv, project_one, addon_one): gv_resource_detail_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=addon_one.resource_pk) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(gv_resource_detail_url) assert resp.status_code == HTTPStatus.OK json_data = resp.json()['data'] @@ -120,19 +120,19 @@ def test_resource_route__pk(self, mock_gv, project_one, addon_one): retrieved_addons_link = json_data['relationships']['configured_storage_addons']['links']['related'] assert retrieved_addons_link == expected_addons_link - def test_resource_route__filter(self, mock_gv, project_one, addon_one): + def test_resource_route__filter(self, fake_gv, project_one, addon_one): gv_resource_detail_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=addon_one.resource_pk) gv_resource_filtered_list_url = gv_requests.RESOURCE_FILTER_ENDPOINT.format(uri=project_one.get_semantic_iri()) - with mock_gv.run_mock(): + with fake_gv.run_fake(): detail_resp = requests.get(gv_resource_detail_url) filtered_list_resp = requests.get(gv_resource_filtered_list_url) assert filtered_list_resp.status_code == HTTPStatus.OK assert filtered_list_resp.json()['data'][0] == detail_resp.json()['data'] - def test_resource_route__addons_link(self, mock_gv, addon_one, addon_two, addon_three): + def test_resource_route__addons_link(self, fake_gv, addon_one, addon_two, addon_three): # addon three is the only one connected to project two gv_resource_detail_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=addon_three.resource_pk) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resource_resp = requests.get(gv_resource_detail_url) addons_resp = requests.get( resource_resp.json()['data']['relationships']['configured_storage_addons']['links']['related'] @@ -142,9 +142,9 @@ def test_resource_route__addons_link(self, mock_gv, addon_one, addon_two, addon_ assert len(json_data) == 1 assert json_data[0] == addon_three.serialize() - def test_account_route(self, mock_gv, account_one): + def test_account_route(self, fake_gv, account_one): gv_account_detail_url = gv_requests.ACCOUNT_ENDPOINT.format(pk=account_one.pk) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(gv_account_detail_url) assert resp.status_code == HTTPStatus.OK json_data = resp.json()['data'] @@ -152,9 +152,9 @@ def test_account_route(self, mock_gv, account_one): assert json_data['relationships']['account_owner']['data']['id'] == account_one.account_owner_pk assert json_data['relationships']['external_storage_service']['data']['id'] == account_one.external_storage_service.pk - def test_addon_route(self, mock_gv, addon_one): + def test_addon_route(self, fake_gv, addon_one): gv_addon_detail_url = gv_requests.ADDON_ENDPOINT.format(pk=addon_one.pk) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(gv_addon_detail_url) assert resp.status_code == HTTPStatus.OK json_data = resp.json()['data'] @@ -168,9 +168,9 @@ def test_addon_route(self, mock_gv, addon_one): class TestHMACValidation: @pytest.fixture - def mock_gv(self): + def fake_gv(self): #validate_headers == True by default - return gv_mocks.MockGravyValet() + return gv_fakes.FakeGravyValet() @pytest.fixture def contributor(self): @@ -185,18 +185,18 @@ def resource(self, contributor): return factories.ProjectFactory(creator=contributor) @pytest.fixture - def external_service(self, mock_gv): - return mock_gv.configure_mock_provider('blarg') + def external_service(self, fake_gv): + return fake_gv.configure_fake_provider('blarg') @pytest.fixture - def external_account(self, mock_gv, contributor, external_service): - return mock_gv.configure_mock_account(contributor, external_service.name) + def external_account(self, fake_gv, contributor, external_service): + return fake_gv.configure_fake_account(contributor, external_service.name) @pytest.fixture - def configured_addon(self, mock_gv, resource, external_account): - return mock_gv.configure_mock_addon(resource, external_account) + def configured_addon(self, fake_gv, resource, external_account): + return fake_gv.configure_fake_addon(resource, external_account) - def test_validate_headers__bad_key(self, mock_gv, contributor, external_account): + def test_validate_headers__bad_key(self, fake_gv, contributor, external_account): request_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=external_account.account_owner_pk) auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, @@ -204,19 +204,19 @@ def test_validate_headers__bad_key(self, mock_gv, contributor, external_account) requesting_user=contributor, hmac_key='bad key' ) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.FORBIDDEN - def test_validate_headers__missing_headers(self, mock_gv, contributor, external_account): + def test_validate_headers__missing_headers(self, fake_gv, contributor, external_account): request_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=external_account.account_owner_pk) request_url = f'{GRAVYVALET_URL}/v1/user-references/{external_account.account_owner_pk}' - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url) assert resp.status_code == HTTPStatus.UNAUTHORIZED @pytest.mark.parametrize('subpath', ['', '/authorized_storage_accounts']) - def test_validate_user__success(self, mock_gv, contributor, external_account, subpath): + def test_validate_user__success(self, fake_gv, contributor, external_account, subpath): base_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=external_account.account_owner_pk) request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( @@ -224,12 +224,12 @@ def test_validate_user__success(self, mock_gv, contributor, external_account, su request_method='GET', requesting_user=contributor ) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.OK @pytest.mark.parametrize('subpath', ['', '/authorized_storage_accounts']) - def test_validate_user__wrong_user(self, mock_gv, noncontributor, external_account, subpath): + def test_validate_user__wrong_user(self, fake_gv, noncontributor, external_account, subpath): base_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=external_account.account_owner_pk) request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( @@ -237,24 +237,24 @@ def test_validate_user__wrong_user(self, mock_gv, noncontributor, external_accou request_method='GET', requesting_user=noncontributor, ) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.FORBIDDEN @pytest.mark.parametrize('subpath', ['', '/authorized_storage_accounts']) - def test_validate_user__no_user(self, mock_gv, external_account, subpath): + def test_validate_user__no_user(self, fake_gv, external_account, subpath): base_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=external_account.account_owner_pk) request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', ) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.UNAUTHORIZED @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) - def test_validate_resource__success(self, mock_gv, contributor, resource, configured_addon, subpath): + def test_validate_resource__success(self, fake_gv, contributor, resource, configured_addon, subpath): base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( @@ -263,12 +263,12 @@ def test_validate_resource__success(self, mock_gv, contributor, resource, config requesting_user=contributor, requested_resource=resource, ) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.OK @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) - def test_validate_resource__wrong_resource(self, mock_gv, contributor, configured_addon, subpath): + def test_validate_resource__wrong_resource(self, fake_gv, contributor, configured_addon, subpath): base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) request_url = f'{base_url}{subpath}' auth_headers = gv_auth.make_gravy_valet_hmac_headers( @@ -277,12 +277,12 @@ def test_validate_resource__wrong_resource(self, mock_gv, contributor, configure requesting_user=contributor, requested_resource=factories.ProjectFactory(creator=contributor), ) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.BAD_REQUEST @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) - def test_validate_resource__noncontributor__public_resource(self, mock_gv, noncontributor, resource, configured_addon, subpath): + def test_validate_resource__noncontributor__public_resource(self, fake_gv, noncontributor, resource, configured_addon, subpath): resource.is_public = True resource.save() base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) @@ -293,12 +293,12 @@ def test_validate_resource__noncontributor__public_resource(self, mock_gv, nonco requesting_user=noncontributor, requested_resource=resource, ) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.OK @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) - def test_validate_resource__noncontributor__private_resource(self, mock_gv, noncontributor, resource, configured_addon, subpath): + def test_validate_resource__noncontributor__private_resource(self, fake_gv, noncontributor, resource, configured_addon, subpath): resource.is_public = False resource.save() base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) @@ -309,12 +309,12 @@ def test_validate_resource__noncontributor__private_resource(self, mock_gv, nonc requesting_user=noncontributor, requested_resource=resource, ) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.FORBIDDEN @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) - def test_validate_resource__unauthenticated_user__public_resource(self, mock_gv, resource, configured_addon, subpath): + def test_validate_resource__unauthenticated_user__public_resource(self, fake_gv, resource, configured_addon, subpath): resource.is_public = True resource.save() base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) @@ -324,12 +324,12 @@ def test_validate_resource__unauthenticated_user__public_resource(self, mock_gv, request_method='GET', requested_resource=resource, ) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.OK @pytest.mark.parametrize('subpath', ['', '/configured_storage_addons']) - def test_validate_resource__unauthenticated_user__private_resource(self, mock_gv, resource, configured_addon, subpath): + def test_validate_resource__unauthenticated_user__private_resource(self, fake_gv, resource, configured_addon, subpath): resource.is_public = False resource.save() base_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=configured_addon.resource_pk) @@ -339,7 +339,7 @@ def test_validate_resource__unauthenticated_user__private_resource(self, mock_gv request_method='GET', requested_resource=resource, ) - with mock_gv.run_mock(): + with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) assert resp.status_code == HTTPStatus.UNAUTHORIZED @@ -348,9 +348,9 @@ def test_validate_resource__unauthenticated_user__private_resource(self, mock_gv class TestRequestHelpers: @pytest.fixture - def mock_gv(self): + def fake_gv(self): #validate_headers == True by default - return gv_mocks.MockGravyValet() + return gv_fakes.FakeGravyValet() @pytest.fixture def contributor(self): @@ -365,13 +365,13 @@ def resource(self, contributor): return factories.ProjectFactory(creator=contributor) @pytest.fixture - def external_service(self, mock_gv): - return mock_gv.configure_mock_provider('blarg') + def external_service(self, fake_gv): + return fake_gv.configure_fake_provider('blarg') @pytest.fixture - def external_account(self, mock_gv, contributor, external_service): - return mock_gv.configure_mock_account(contributor, external_service.name) + def external_account(self, fake_gv, contributor, external_service): + return fake_gv.configure_fake_account(contributor, external_service.name) @pytest.fixture - def configured_addon(self, mock_gv, resource, external_account): - return mock_gv.configure_mock_addon(resource, external_account) + def configured_addon(self, fake_gv, resource, external_account): + return fake_gv.configure_fake_addon(resource, external_account) From c3616ad1d7e9803ad44e77e592ca3158a8e42897 Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Tue, 18 Jun 2024 11:12:34 -0400 Subject: [PATCH 4/8] add "gv_fakes" --- osf/external/gravy_valet/gv_fakes.py | 553 +++++++++++++++++++++++++++ 1 file changed, 553 insertions(+) create mode 100644 osf/external/gravy_valet/gv_fakes.py diff --git a/osf/external/gravy_valet/gv_fakes.py b/osf/external/gravy_valet/gv_fakes.py new file mode 100644 index 00000000000..44dba32bd80 --- /dev/null +++ b/osf/external/gravy_valet/gv_fakes.py @@ -0,0 +1,553 @@ +import contextlib +import itertools +import json +import typing +import re +import urllib.parse +from http import HTTPStatus + +import dataclasses # backport +import responses + +from . import auth_helpers +from osf.models import OSFUser, AbstractNode +from osf.utils import permissions as osf_permissions +from website import settings + + +INCLUDE_REGEX = r'(\?include=(?P.+))?' + +class FakeGVError(Exception): + + def __init__(self, status_code, *args, **kwargs): + self.status_code = status_code + super().__init__(*args, **kwargs) + + +@dataclasses.dataclass(frozen=True) +class _FakeGVEntity: + + RESOURCE_TYPE: typing.ClassVar[str] + pk: int + + @property + def api_path(self): + return f'v1/{self.RESOURCE_TYPE}/{self.pk}' + + def serialize(self): + data = { + 'type': self.RESOURCE_TYPE, + 'id': self.pk, + 'attributes': self._serialize_attributes(), + 'links': self._serialize_links(), + } + relationships = self._serialize_relationships() + if relationships: + data['relationships'] = relationships + return data + + def _serialize_attributes(self): + ... + + def _serialize_relationships(self): + ... + + def _serialize_links(self): + return {'self': f'{settings.GRAVYVALET_URL}/{self.api_path}'} + + def _format_relationship_entry(self, relationship_path, related_type=None, related_pk=None): + relationship_api_path = f'{settings.GRAVYVALET_URL}/{self.api_path}/{relationship_path}' + relationship_entry = {'links': {'related': relationship_api_path}} + if related_type and related_pk: + relationship_entry['data'] = {'type': related_type, 'id': related_pk} + return relationship_entry + +@dataclasses.dataclass(frozen=True) +class _FakeUserReference(_FakeGVEntity): + + RESOURCE_TYPE = 'user-references' + uri: str + + def _serialize_attributes(self): + return {'user_uri': self.uri} + + def _serialize_relationships(self): + accounts_relationship = self._format_relationship_entry(relationship_path='authorized_storage_accounts') + return {'authorized_storage_accounts': accounts_relationship} + +@dataclasses.dataclass(frozen=True) +class _FakeResourceReference(_FakeGVEntity): + + RESOURCE_TYPE = 'resource-references' + uri: str + + def _serialize_attributes(self): + return {'resource_uri': self.uri} + + def _serialize_relationships(self): + configured_addons_relationship = self._format_relationship_entry(relationship_path='configured_storage_addons') + return {'configured_storage_addons': configured_addons_relationship} + +@dataclasses.dataclass(frozen=True) +class _FakeAddonProvider(_FakeGVEntity): + + RESOURCE_TYPE = 'external-storage-services' + name: str + max_upload_mb: int = 2**10 + max_concurrent_uploads: int = -5 + icon_url: str = 'vetted-url-for-icon.png' + + def _serialize_attributes(self): + return { + 'name': self.name, + 'max_upload_mb': self.max_upload_mb, + 'max_concurrent_uploads': self.max_concurrent_uploads, + 'configurable_api_root': False, + 'terms_of_service_features': [], + 'icon_url': self.icon_url, + } + + def _serialize_relationships(self): + return { + 'addon_imp': self._format_relationship_entry( + relationship_path='addon_imp', related_type='addon-imps', related_pk=1 + ) + } + + +@dataclasses.dataclass(frozen=True) +class _FakeAccount(_FakeGVEntity): + + RESOURCE_TYPE = 'authorized-storage-accounts' + external_storage_service: _FakeAddonProvider + account_owner_pk: int + display_name: str = '' + + def _serialize_attributes(self): + return { + 'display_name': self.display_name, + 'authorized_scopes': ['all_of_the_scopes'], + 'authorized_capabilities': ['ACCESS', 'UPDATE'], + 'authorized_operation_names': ['get_root_items'], + 'credentials_available': True, + 'imp_name': 'BLARG', + } + + def _serialize_relationships(self): + return { + 'account_owner': self._format_relationship_entry( + relationship_path='account_owner', + related_type=_FakeUserReference.RESOURCE_TYPE, + related_pk=self.account_owner_pk + ), + 'external_storage_service': self._format_relationship_entry( + relationship_path='external_storage_service', + related_type=_FakeAddonProvider.RESOURCE_TYPE, + related_pk=self.external_storage_service.pk + ), + 'configured_storage_addons': self._format_relationship_entry( + relationship_path='configured_storage_addons' + ), + 'authorized_operations': self._format_relationship_entry( + relationship_path='authorized_operations' + ), + } + +@dataclasses.dataclass(frozen=True) +class _FakeAddon(_FakeGVEntity): + + RESOURCE_TYPE = 'configured-storage-addons' + resource_pk: int + base_account: _FakeAccount + display_name: str = '' + root_folder: str = '/' + + def _serialize_attributes(self): + return { + 'display_name': self.display_name, + 'root_folder': self.root_folder, + 'max_upload_mb': self.base_account.external_storage_service.max_upload_mb, + 'max_concurrent_uploads': self.base_account.external_storage_service.max_concurrent_uploads, + 'icon_url': self.base_account.external_storage_service.icon_url, + 'connected_capabilities': ['ACCESS'], + 'connected_operation_names': ['get_root_items'], + 'imp_name': 'BLARG', + } + + def _serialize_relationships(self): + return { + 'authorized_resource': self._format_relationship_entry( + relationship_path='authorized_resource', + related_type=_FakeResourceReference.RESOURCE_TYPE, + related_pk=self.resource_pk + ), + 'base_account': self._format_relationship_entry( + relationship_path='base_account', + related_type=_FakeAccount.RESOURCE_TYPE, + related_pk=self.base_account.pk + ), + 'external_storage_service': self._format_relationship_entry( + relationship_path='external_storage_service', + related_type=_FakeAddonProvider.RESOURCE_TYPE, + related_pk=self.base_account.external_storage_service.pk + ), + 'connected_operations': self._format_relationship_entry( + relationship_path='connected_operations' + ), + } + + +class FakeGravyValet(): + + ROUTES = { + r'v1/user-references(/(?P\d+)|(\?filter\[user_uri\]=(?P[^&]+)))$': '_get_user', + r'v1/resource-references(/(?P\d+)|(\?filter\[resource_uri\]=(?P[^&]+)))$': '_get_resource', + r'v1/authorized-storage-accounts/(?P\d+)$': '_get_account', + r'v1/configured-storage-addons/(?P\d+)$': '_get_addon', + r'v1/user-references/(?P\d+)/authorized_storage_accounts$': '_get_user_accounts', + r'v1/resource-references/(?P\d+)/configured_storage_addons$': '_get_resource_addons', + } + + def __init__(self): + self._clear_mappings() + self._validate_headers = True + + @property + def validate_headers(self) -> bool: + return self._validate_headers + + @validate_headers.setter + def validate_headers(self, value: bool): + if not isinstance(value, bool): + raise ValueError('validate_headers must be a boolean value') + self._validate_headers = value + + def _clear_mappings(self, include_providers: bool = True): + """Reset all configured users/resources/acounts/addons and, optionally, providers.""" + if include_providers: + # Mapping from _FakeAddonProvider name to _FakeAddonProvider + self._known_providers = {} + # Bidirectional mapping between user uri and fake "pk" + self._known_users = {} + # Bidirectional mapping between resource uri and fake "pk" + self._known_resources = {} + # Mapping from user "pk" to _FakeAccounts for the user + self._user_accounts = {} + # Mapping from resource "pk" to _FakeAddons "configured" on the resource + self._resource_addons = {} + + def _get_or_create_user_entry(self, user: OSFUser): + user_uri = user.get_semantic_iri() + user_pk = self._known_users.get(user_uri) + if not user_pk: + user_pk = len(self._known_users) + 1 + self._known_users[user_uri] = user_pk + self._known_users[user_pk] = user_uri + return user_uri, user_pk + + def _get_or_create_resource_entry(self, resource: AbstractNode): + resource_uri = resource.get_semantic_iri() + resource_pk = self._known_resources.get(resource_uri) + if not resource_pk: + resource_pk = len(self._known_resources) + 1 + self._known_resources[resource_uri] = resource_pk + self._known_resources[resource_pk] = resource_uri + return resource_uri, resource_pk + + def configure_fake_provider(self, provider_name: str, **service_attrs) -> _FakeAddonProvider: + known_provider = self._known_providers.get(provider_name) + provider_pk = known_provider.pk if known_provider else len(self._known_providers) + 1 + new_provider = _FakeAddonProvider( + name=provider_name, + pk=provider_pk, + **service_attrs + ) + self._known_providers[provider_name] = new_provider + return new_provider + + def configure_fake_account( + self, + user: OSFUser, + addon_name: str, + **account_attrs + ) -> _FakeAccount: + user_uri, user_pk = self._get_or_create_user_entry(user) + account_pk = _get_nested_count(self._user_accounts) + 1 + connected_provider = self._known_providers[addon_name] + new_account = _FakeAccount( + pk=account_pk, + account_owner_pk=user_pk, + external_storage_service=connected_provider, + **account_attrs + ) + self._user_accounts.setdefault(user_pk, []).append(new_account) + return new_account + + def configure_fake_addon( + self, + resource: AbstractNode, + connected_account: _FakeAccount, + **config_attrs + ) -> _FakeAddon: + resource_uri, resource_pk = self._get_or_create_resource_entry(resource) + addon_pk = _get_nested_count(self._resource_addons) + 1 + new_addon = _FakeAddon( + pk=addon_pk, + resource_pk=resource_pk, + base_account=connected_account, + **config_attrs + ) + self._resource_addons.setdefault(resource_pk, []).append(new_addon) + return new_addon + + @contextlib.contextmanager + def run_fake(self): + with responses.RequestsMock() as requests_mock: + requests_mock.add_callback( + responses.GET, + re.compile(f'{re.escape(settings.GRAVYVALET_URL)}.*'), + callback=self._route_request, + content_type='application/json', + ) + yield requests_mock + + def _route_request(self, request): # -> tuple[int, dict, str] + if self.validate_headers: + try: + _validate_request(request) + except FakeGVError as e: + return (e.status_code, {}, '') + + status_code = 200 + for route_expr, routed_func_name in self.ROUTES.items(): + url_regex = re.compile(f'{re.escape(settings.GRAVYVALET_URL)}/{route_expr}{INCLUDE_REGEX}') + route_match = url_regex.match(urllib.parse.unquote(request.url)) + if route_match: + func = getattr(self, routed_func_name) + try: + body = func(headers=request.headers, **route_match.groupdict()) + except KeyError: # entity lookup failed somewhere + status_code = HTTPStatus.NOT_FOUND + body = '' + except FakeGVError as e: + status_code = e.status_code + body = '' + return (status_code, {}, body) + + return (HTTPStatus.NOT_FOUND, {}, '') + + def _get_user( + self, + headers: dict, + pk=None, # str | None + user_uri=None, # str | None + include_param: str = '', + ) -> str: + if bool(pk) == bool(user_uri): + raise FakeGVError(HTTPStatus.BAD_REQUEST) + + # if passed the user_uri, call came through list endpoint with filter + if user_uri: + list_view = True + pk = self._known_users[user_uri] + else: + list_view = False + pk = int(pk) + user_uri = self._known_users[pk] + + if self.validate_headers: + _validate_user(user_uri, headers) + + return _format_response_body( + data=_FakeUserReference(pk=pk, uri=user_uri), + list_view=list_view, + include_param=include_param, + ) + + def _get_resource( + self, + headers: dict, + pk=None, # str | None + resource_uri=None, # str | None + include_param: str = '', + ) -> str: + if bool(pk) == bool(resource_uri): + raise FakeGVError(HTTPStatus.BAD_REQUEST) + + # if passed the resource_uri, call came through list endpoint with filter + if resource_uri: + list_view = True + pk = self._known_resources[resource_uri] + else: + list_view = False + pk = int(pk) + resource_uri = self._known_resources[pk] + + if self.validate_headers: + _validate_resource_access(resource_uri, headers) + + return _format_response_body( + data=_FakeResourceReference(pk=pk, uri=resource_uri), + list_view=list_view, + include_param=include_param, + ) + + def _get_account( + self, + headers: dict, + pk: str, + include_param: str = '', + ) -> str: + pk = int(pk) + account = None + for account in itertools.chain.from_iterable(self._user_accounts.values()): + if account.pk == pk: + account = account + break + + if not account: + raise FakeGVError(HTTPStatus.NOT_FOUND) + + if self.validate_headers: + user_uri = self._known_users[account.account_owner_pk] + _validate_user(user_uri, headers) + + return _format_response_body( + data=account, + list_view=False, + include_param=include_param, + ) + + def _get_addon( + self, headers: dict, + pk: str, + include_param: str = '', + ) -> str: + pk = int(pk) + addon = None + for addon in itertools.chain.from_iterable(self._resource_addons.values()): + if addon.pk == pk: + addon = addon + break + + if not addon: + raise FakeGVError(HTTPStatus.NOT_FOUND) + + if self.validate_headers: + resource_uri = self._known_resources[addon.resource_pk] + _validate_resource_access(resource_uri, headers) + + return _format_response_body( + data=addon, + list_view=False, + include_param=include_param, + ) + + def _get_user_accounts( + self, + headers: dict, + user_pk: str, + include_param: str = '', + ) -> str: + user_pk = int(user_pk) + if self.validate_headers: + user_uri = self._known_users[user_pk] + _validate_user(user_uri, headers) + + return _format_response_body( + data=self._user_accounts.get(user_pk, []), + list_view=True, + include_param=include_param + ) + + def _get_resource_addons( + self, + headers: dict, + resource_pk: str, + include_param: str = '', + ) -> str: + resource_pk = int(resource_pk) + if self.validate_headers: + resource_uri = self._known_resources[resource_pk] + _validate_resource_access(resource_uri, headers) + + return _format_response_body( + data=self._resource_addons.get(resource_pk, []), + include_param=include_param, + list_view=True, + ) + + +def _format_response_body( + data, # _FakeGVEntity | list[_FakeGVEntity] + list_view: bool = False, + include_param='', +) -> str: + """Returns the expected (status, headers, json) tuple expected by callbacks for FakeRequest.""" + if list_view: + if not isinstance(data, list): + data = [data] + serialized_data = [entry.serialize() for entry in data] + else: + serialized_data = data.serialize() + + response_dict = { + 'data': serialized_data, + } + if include_param: + response_dict['included'] = _format_includes(data, include_param.split(',')) + return json.dumps(response_dict) + + +def _format_includes(data, includes): + included_data = set() + if not isinstance(data, typing.Iterable): + data = (data,) + for entry in data: + for included_path in includes: + included_members = included_path.split('.') + source_object = entry + for member in included_members: + included_entry = getattr(source_object, member) + included_data.add(included_entry.serialize()) + source_object = included_entry + return json.dumps(list(included_data)) + + +def _get_nested_count(d): # dict[Any, Any] -> int: + """Get the total number of entries from a dictionary with lists for values.""" + return sum(map(len, d.values())) + + +def _validate_request(request): + try: + auth_helpers.validate_signed_headers(request) + except ValueError: + error_code = ( + HTTPStatus.FORBIDDEN + if request.headers.get(auth_helpers.USER_HEADER) + else HTTPStatus.UNAUTHORIZED + ) + raise FakeGVError(error_code) + + +def _validate_user(requested_user_uri, headers): + requesting_user_uri = headers.get(auth_helpers.USER_HEADER) + if requesting_user_uri is None: + raise FakeGVError(HTTPStatus.UNAUTHORIZED) + if requesting_user_uri != requested_user_uri: + raise FakeGVError(HTTPStatus.FORBIDDEN) + + +def _validate_resource_access(requested_resource_uri, headers): + headers_requested_resource = headers.get(auth_helpers.RESOURCE_HEADER) + # generously assume malformed request on mismatch between headers and request + if not headers_requested_resource or headers_requested_resource != requested_resource_uri: + raise FakeGVError(HTTPStatus.BAD_REQUEST) + requesting_user_uri = headers.get(auth_helpers.USER_HEADER) + permission_denied_error_code = ( + HTTPStatus.FORBIDDEN if requesting_user_uri else HTTPStatus.UNAUTHORIZED + ) + resource_permissions = headers.get(auth_helpers.PERMISSIONS_HEADER, '').split(';') + if osf_permissions.READ not in resource_permissions: + raise FakeGVError(permission_denied_error_code) From dbecc91d479f0ea6979aea5d988c3f535e23321f Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Thu, 20 Jun 2024 13:40:00 -0400 Subject: [PATCH 5/8] Add tests, make them work, clean up everything --- osf/external/gravy_valet/auth_helpers.py | 16 +-- osf/external/gravy_valet/compat.py | 73 ++++++---- osf/external/gravy_valet/gv_fakes.py | 32 +++-- osf/external/gravy_valet/request_helpers.py | 43 +++--- osf_tests/test_gv_utils.py | 151 +++++++++++++++++--- 5 files changed, 231 insertions(+), 84 deletions(-) diff --git a/osf/external/gravy_valet/auth_helpers.py b/osf/external/gravy_valet/auth_helpers.py index 174a6098fbc..5b4e3f8fc7b 100644 --- a/osf/external/gravy_valet/auth_helpers.py +++ b/osf/external/gravy_valet/auth_helpers.py @@ -55,7 +55,10 @@ def _get_signed_components( return signed_segments, signed_headers -def _make_permissions_headers(requesting_user=None, requested_resource=None): +def make_permissions_headers( + requesting_user: typing.Optional[OSFUser] = None, + requested_resource: typing.Optional[AbstractNode] = None +) -> dict: osf_permissions_headers = {} if requesting_user: osf_permissions_headers[USER_HEADER] = requesting_user.get_semantic_iri() @@ -75,17 +78,12 @@ def make_gravy_valet_hmac_headers( request_method: str, body: typing.Union[str, bytes] = '', hmac_key: typing.Optional[str] = None, - additional_header: typing.Optional[dict] = None, - requesting_user: typing.Optional[OSFUser] = None, - requested_resource: typing.Optional[AbstractNode] = None + additional_headers: typing.Optional[dict] = None, ) -> dict: - osf_permissions_headers = {} - if requesting_user or requested_resource: - osf_permissions_headers = _make_permissions_headers(requesting_user, requested_resource) - + additional_headers = additional_headers or {} signed_string_segments, signed_headers = _get_signed_components( - request_url, request_method, body, **osf_permissions_headers + request_url, request_method, body, **additional_headers ) signature = _sign_message( message='\n'.join(signed_string_segments), hmac_key=hmac_key diff --git a/osf/external/gravy_valet/compat.py b/osf/external/gravy_valet/compat.py index 59c7e41e2b4..55c80456a98 100644 --- a/osf/external/gravy_valet/compat.py +++ b/osf/external/gravy_valet/compat.py @@ -7,58 +7,83 @@ from . import request_helpers as gv_requests -class _LegacyConfigsForImps(enum.Enum): +class _LegacyConfigsForWBKey(enum.Enum): """Mapping from a GravyValet StorageImp name to the Addon config.""" box = BoxAddonAppConfig -def make_fake_node_settings(gv_data, requested_resource, requesting_user): - service_wb_key = gv_data.get_nested_attribute( +def make_generic_node_settings(gv_addon_data, requested_resource, requesting_user): + service_wb_key = gv_addon_data.get_included_attribute( attribute_path=('base_account', 'external_storage_service'), attribute_name='wb_key' ) - legacy_config = _LegacyConfigsForImps(service_wb_key) - return FakeNodeSettings( - config=FakeAddonConfig.from_legacy_config(legacy_config), - gv_id=gv_data.resource_id, - folder_id=gv_data.get_attribute('configured_root_id'), + legacy_config = _LegacyConfigsForWBKey(service_wb_key) + return EphemeralNodeSettings( + config=EphemeralAddonConfig.from_legacy_config(legacy_config), + gv_id=gv_addon_data.resource_id, + folder_id=gv_addon_data.get_attribute('configured_root_id'), configured_resource=requested_resource, active_user=requesting_user, ) +def make_generic_user_settings(gv_account_data, requesting_user): + service_wb_key = gv_account_data.get_included_attribute( + attribute_path=('external_storage_service'), + attribute_name='wb_key' + ) + legacy_config = _LegacyConfigsForWBKey(service_wb_key) + return EphemeralUserSettings( + config=EphemeralAddonConfig.from_legacy_config(legacy_config), + gv_id=gv_account_data.resource_id, + active_user=requesting_user, + ) + @dataclasses.dataclass -class FakeAddonConfig: +class EphemeralAddonConfig: + '''Minimalist dataclass for storing the actually used properties of an AddonConfig''' + name: str + label: str short_name: str + full_name: str @classmethod def from_legacy_config(legacy_config): - return FakeAddonConfig( - short_name=legacy_config.short_name + return EphemeralAddonConfig( + name=legacy_config.name, + label=legacy_config.label, + full_name=legacy_config.full_name, + short_name=legacy_config.short_name, ) @dataclasses.dataclass -class FakeNodeSettings: - config: FakeAddonConfig +class EphemeralNodeSettings: + '''Minimalist dataclass for storing/translating the actually used properties of NodeSettings.''' + config: EphemeralAddonConfig folder_id: str gv_id: str - configured_resource: Node - active_user: OSFUser _storage_config: dict | None _credentials: dict | None + # These are needed in order to make further requests for credentials + configured_resource: Node + active_user: OSFUser + @property def short_name(self): return self.config.short_name def serialize_waterbutler_credentials(self): + # sufficient for most OAuth services, including Box + # TODO: Define per-service translation (or common schemes) if self._credentials is None: self._fetch_wb_config() return self._credentials def serialize_waterbutler_settings(self): - # sufficient for box + # sufficient for Box + # TODO: Define per-service translation (or common schemes) return { 'folder': self.folder_id, 'service': self.short_name, @@ -75,13 +100,13 @@ def _fetch_wb_config(self): @dataclasses.dataclass -class FakeUserSettings: - config: FakeAddonConfig +class EphemeralUserSettings: + '''Minimalist dataclass for storing the actually used properties of UserSettings.''' + config: EphemeralAddonConfig gv_id: str + # This is needed to support making further requests + active_user: OSFUser - -def _get_service_name_from_gv_data(gv_data): - return gv_data.get_included_attribute('base_account.external_storage_service.wb_key') - -def _get_configured_folder_from_gv_data(gv_data): - return gv_data.get_attribute('root_folder') + @property + def short_name(self): + return self.config.short_name diff --git a/osf/external/gravy_valet/gv_fakes.py b/osf/external/gravy_valet/gv_fakes.py index 44dba32bd80..7a0a60bd3f3 100644 --- a/osf/external/gravy_valet/gv_fakes.py +++ b/osf/external/gravy_valet/gv_fakes.py @@ -1,8 +1,9 @@ import contextlib import itertools import json -import typing +import logging import re +import typing import urllib.parse from http import HTTPStatus @@ -15,7 +16,9 @@ from website import settings -INCLUDE_REGEX = r'(\?include=(?P.+))?' +logger = logging.getLogger(__name__) + +INCLUDE_REGEX = r'(\?include=(?P.+))' class FakeGVError(Exception): @@ -96,15 +99,17 @@ class _FakeAddonProvider(_FakeGVEntity): max_upload_mb: int = 2**10 max_concurrent_uploads: int = -5 icon_url: str = 'vetted-url-for-icon.png' + wb_key: str = None def _serialize_attributes(self): return { - 'name': self.name, + 'display_name': self.name, 'max_upload_mb': self.max_upload_mb, 'max_concurrent_uploads': self.max_concurrent_uploads, 'configurable_api_root': False, 'terms_of_service_features': [], 'icon_url': self.icon_url, + 'wb_key': self.wb_key or self.name } def _serialize_relationships(self): @@ -200,12 +205,12 @@ def _serialize_relationships(self): class FakeGravyValet(): ROUTES = { - r'v1/user-references(/(?P\d+)|(\?filter\[user_uri\]=(?P[^&]+)))$': '_get_user', - r'v1/resource-references(/(?P\d+)|(\?filter\[resource_uri\]=(?P[^&]+)))$': '_get_resource', - r'v1/authorized-storage-accounts/(?P\d+)$': '_get_account', - r'v1/configured-storage-addons/(?P\d+)$': '_get_addon', - r'v1/user-references/(?P\d+)/authorized_storage_accounts$': '_get_user_accounts', - r'v1/resource-references/(?P\d+)/configured_storage_addons$': '_get_resource_addons', + r'v1/user-references(/(?P\d+)|(\?filter\[user_uri\]=(?P[^&]+)))': '_get_user', + r'v1/resource-references(/(?P\d+)|(\?filter\[resource_uri\]=(?P[^&]+)))': '_get_resource', + r'v1/authorized-storage-accounts/(?P\d+)': '_get_account', + r'v1/configured-storage-addons/(?P\d+)': '_get_addon', + r'v1/user-references/(?P\d+)/authorized_storage_accounts': '_get_user_accounts', + r'v1/resource-references/(?P\d+)/configured_storage_addons': '_get_resource_addons', } def __init__(self): @@ -320,13 +325,14 @@ def _route_request(self, request): # -> tuple[int, dict, str] status_code = 200 for route_expr, routed_func_name in self.ROUTES.items(): - url_regex = re.compile(f'{re.escape(settings.GRAVYVALET_URL)}/{route_expr}{INCLUDE_REGEX}') + url_regex = re.compile(f'{re.escape(settings.GRAVYVALET_URL)}/{route_expr}({INCLUDE_REGEX}|$)') route_match = url_regex.match(urllib.parse.unquote(request.url)) if route_match: func = getattr(self, routed_func_name) try: body = func(headers=request.headers, **route_match.groupdict()) except KeyError: # entity lookup failed somewhere + logger.critical('BAD LOOKUP') status_code = HTTPStatus.NOT_FOUND body = '' except FakeGVError as e: @@ -334,6 +340,7 @@ def _route_request(self, request): # -> tuple[int, dict, str] body = '' return (status_code, {}, body) + logger.critical('route not found') return (HTTPStatus.NOT_FOUND, {}, '') def _get_user( @@ -406,6 +413,7 @@ def _get_account( break if not account: + logger.critical('Account not found') raise FakeGVError(HTTPStatus.NOT_FOUND) if self.validate_headers: @@ -509,9 +517,9 @@ def _format_includes(data, includes): source_object = entry for member in included_members: included_entry = getattr(source_object, member) - included_data.add(included_entry.serialize()) + included_data.add(included_entry) source_object = included_entry - return json.dumps(list(included_data)) + return [included_entity.serialize() for included_entity in included_data] def _get_nested_count(d): # dict[Any, Any] -> int: diff --git a/osf/external/gravy_valet/request_helpers.py b/osf/external/gravy_valet/request_helpers.py index b2e876f2975..3e2e11c494c 100644 --- a/osf/external/gravy_valet/request_helpers.py +++ b/osf/external/gravy_valet/request_helpers.py @@ -1,11 +1,13 @@ from urllib.parse import urlencode, urljoin, urlparse, urlunparse +import logging import dataclasses import requests from . import auth_helpers from website import settings +logger = logging.getLogger(__name__) # Use urljoin here to handle inconsistent use of trailing slash API_BASE = urljoin(settings.GRAVYVALET_URL, 'v1/') @@ -15,10 +17,10 @@ ADDON_ENDPOINT = f'{API_BASE}configured-storage-addons/{{pk}}' WB_CONFIG_ENDPOINT = f'{ADDON_ENDPOINT}/waterbutler-config' -USER_FILTER_ENDPOINT = f'{API_BASE}user-references?filter[user_uri]={{uri}}' +USER_LIST_ENDPOINT = f'{API_BASE}user-references' USER_DETAIL_ENDPOINT = f'{API_BASE}user-references/{{pk}}' -RESOURCE_FILTER_ENDPOINT = f'{API_BASE}resource-references?filter[resource_uri]={{uri}}' +RESOURCE_LIST_ENDPOINT = f'{API_BASE}resource-references' RESOURCE_DETAIL_ENDPOINT = f'{API_BASE}resource-references/{{pk}}' ACCOUNT_EXTERNAL_SERVICE_PATH = 'external_storage_service' @@ -37,7 +39,7 @@ def get_account(gv_account_pk, requesting_user): # -> JSONAPIResultEntry def get_addon(gv_addon_pk, requested_resource, requesting_user): # -> JSONAPIResultEntry '''Return a JSONAPIResultEntry representing a known ConfiguredStorageAddon.''' return get_gv_result( - endpoint=ADDON_ENDPOINT.format(pk=gv_addon_pk), + endpoint_url=ADDON_ENDPOINT.format(pk=gv_addon_pk), requesting_user=requesting_user, requested_resource=requested_resource, params={'include': ADDON_EXTERNAL_SERVICE_PATH}, @@ -47,8 +49,9 @@ def get_addon(gv_addon_pk, requested_resource, requesting_user): # -> JSONAPIRe def iterate_accounts_for_user(requesting_user): # -> typing.Iterator[JSONAPIResultEntry] '''Returns an iterator of JSONAPIResultEntries representing all of the AuthorizedStorageAccounts for a user.''' user_result = get_gv_result( - endpoint_url=USER_FILTER_ENDPOINT.format(uri=requesting_user.get_semantic_iri()), + endpoint_url=USER_LIST_ENDPOINT, requesting_user=requesting_user, + params={'filter[user_uri]': requesting_user.get_semantic_iri()}, ) if not user_result: return None @@ -62,9 +65,10 @@ def iterate_accounts_for_user(requesting_user): # -> typing.Iterator[JSONAPIRes def iterate_addons_for_resource(requested_resource, requesting_user): # -> typing.Iterator[JSONAPIResultEntry] '''Returns an iterator of JSONAPIResultEntires representing all of the ConfiguredStorageAddons for a resource.''' resource_result = get_gv_result( - endpoint_url=RESOURCE_FILTER_ENDPOINT.format(uri=requested_resource.get_semantic_iri()), + endpoint_url=RESOURCE_LIST_ENDPOINT, requesting_user=requesting_user, requested_resource=requested_resource, + params={'filter[resource_uri]': requested_resource.get_semantic_iri()}, ) if not resource_result: return None @@ -78,23 +82,25 @@ def iterate_addons_for_resource(requested_resource, requesting_user): # -> typi def get_waterbutler_config(gv_addon_pk, requested_resource, requesting_user): # -> JSONAPIResultEntry return get_gv_result( - endpoint=WB_CONFIG_ENDPOINT.format(pk=gv_addon_pk), + endpoint_url=WB_CONFIG_ENDPOINT.format(pk=gv_addon_pk), requesting_user=requesting_user, requested_resource=requested_resource ) def get_gv_result(**kwargs): # -> JSONAPIResultEntry - '''Processes the result of a request to a GravyValet detail endpoint into a JSONAPIResultEntry. + '''Processes the result of a request to a GravyValet detail endpoint into a single JSONAPIResultEntry. kwargs must match _make_gv_request ''' - response = _make_gv_request(**kwargs) - response_json = response.json() - if not response['data']: + response_json = _make_gv_request(**kwargs).json() + if not response_json['data']: return None + data = response_json['data'] + if isinstance(data, list): + data = data[0] # Assume filtered list endpoint included_entities_lookup = _format_included_entities(response_json.get('included', [])) - return JSONAPIResultEntry(response_json['data'], included_entities_lookup) + return JSONAPIResultEntry(data, included_entities_lookup) def iterate_gv_results(**kwargs): # -> typing.Iterator[JSONAPIResultEntry] @@ -102,12 +108,12 @@ def iterate_gv_results(**kwargs): # -> typing.Iterator[JSONAPIResultEntry] kwargs must match _make_gv_request ''' - response_json = _make_gv_request(**kwargs).json() if not response_json['data']: return # empty iterator included_entities_lookup = _format_included_entities(response_json.get('included', [])) - yield from (JSONAPIResultEntry(entry, included_entities_lookup) for entry in response_json['data']) + for entry in response_json['data']: + yield JSONAPIResultEntry(entry, included_entities_lookup) def _make_gv_request( @@ -122,12 +128,12 @@ def _make_gv_request( auth_headers = auth_helpers.make_gravy_valet_hmac_headers( request_url=full_url, request_method=request_method, - additional_headers=auth_helpers._make_permissions_headers( + additional_headers=auth_helpers.make_permissions_headers( requesting_user=requesting_user, - requested_resource=requested_resource + requested_resource=requested_resource, ) ) - response = requests.get(full_url, headers=auth_headers, params=params) + response = requests.get(endpoint_url, headers=auth_headers, params=params) if not response.ok: # log error to Sentry pass @@ -145,7 +151,7 @@ def _format_included_entities(included_entities_json): for entity in included_entities_json } for entity in included_entities_by_type_and_id.values(): - entity._extract_included_relationships(included_entities_by_type_and_id) + entity.extract_included_relationships(included_entities_by_type_and_id) return included_entities_by_type_and_id @@ -163,8 +169,9 @@ def __init__(self, result_entry: dict, included_entities_lookup: dict = None): self.resource_id = result_entry['id'] self._attributes = dict(result_entry['attributes']) self._relationships = _extract_relationships(result_entry['relationships']) + self._includes = {} if included_entities_lookup: - self._includes = self._extract_included_relationships(included_entities_lookup) + self.extract_included_relationships(included_entities_lookup) def get_attribute(self, attribute_name): return self._attributes.get(attribute_name) diff --git a/osf_tests/test_gv_utils.py b/osf_tests/test_gv_utils.py index b10113b2ef0..bbb973a77af 100644 --- a/osf_tests/test_gv_utils.py +++ b/osf_tests/test_gv_utils.py @@ -69,9 +69,6 @@ def addon_three(self, project_two, account_one, fake_gv): def test_user_route__pk(self, fake_gv, test_user, account_one): gv_user_detail_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=account_one.account_owner_pk) - logger.critical('DEBUG DEBUG DEBUG') - logger.critical(gv_user_detail_url) - logger.critical('\n\n\n') with fake_gv.run_fake(): resp = requests.get(gv_user_detail_url) assert resp.status_code == HTTPStatus.OK @@ -85,10 +82,13 @@ def test_user_route__pk(self, fake_gv, test_user, account_one): def test_user_route__filter(self, fake_gv, test_user, account_one): gv_user_detail_url = gv_requests.USER_DETAIL_ENDPOINT.format(pk=account_one.account_owner_pk) - gv_user_filtered_list_url = gv_requests.USER_FILTER_ENDPOINT.format(uri=test_user.get_semantic_iri()) + gv_user_list_url = gv_requests.USER_LIST_ENDPOINT with fake_gv.run_fake(): detail_resp = requests.get(gv_user_detail_url) - filtered_list_resp = requests.get(gv_user_filtered_list_url) + filtered_list_resp = requests.get( + gv_user_list_url, + params={'filter[user_uri]': test_user.get_semantic_iri()} + ) assert filtered_list_resp.status_code == HTTPStatus.OK assert filtered_list_resp.json()['data'][0] == detail_resp.json()['data'] @@ -122,10 +122,13 @@ def test_resource_route__pk(self, fake_gv, project_one, addon_one): def test_resource_route__filter(self, fake_gv, project_one, addon_one): gv_resource_detail_url = gv_requests.RESOURCE_DETAIL_ENDPOINT.format(pk=addon_one.resource_pk) - gv_resource_filtered_list_url = gv_requests.RESOURCE_FILTER_ENDPOINT.format(uri=project_one.get_semantic_iri()) + gv_resource_list_url = gv_requests.RESOURCE_LIST_ENDPOINT with fake_gv.run_fake(): detail_resp = requests.get(gv_resource_detail_url) - filtered_list_resp = requests.get(gv_resource_filtered_list_url) + filtered_list_resp = requests.get( + gv_resource_list_url, + params={'filter[resource_uri]': project_one.get_semantic_iri()}, + ) assert filtered_list_resp.status_code == HTTPStatus.OK assert filtered_list_resp.json()['data'][0] == detail_resp.json()['data'] @@ -201,8 +204,10 @@ def test_validate_headers__bad_key(self, fake_gv, contributor, external_account) auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', - requesting_user=contributor, - hmac_key='bad key' + hmac_key='bad key', + additional_headers=gv_auth.make_permissions_headers( + requesting_user=contributor + ), ) with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) @@ -222,7 +227,9 @@ def test_validate_user__success(self, fake_gv, contributor, external_account, su auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', - requesting_user=contributor + additional_headers=gv_auth.make_permissions_headers( + requesting_user=contributor + ), ) with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) @@ -235,7 +242,9 @@ def test_validate_user__wrong_user(self, fake_gv, noncontributor, external_accou auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', - requesting_user=noncontributor, + additional_headers=gv_auth.make_permissions_headers( + requesting_user=noncontributor, + ) ) with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) @@ -260,8 +269,10 @@ def test_validate_resource__success(self, fake_gv, contributor, resource, config auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', - requesting_user=contributor, - requested_resource=resource, + additional_headers=gv_auth.make_permissions_headers( + requesting_user=contributor, + requested_resource=resource, + ), ) with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) @@ -274,8 +285,10 @@ def test_validate_resource__wrong_resource(self, fake_gv, contributor, configure auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', - requesting_user=contributor, - requested_resource=factories.ProjectFactory(creator=contributor), + additional_headers=gv_auth.make_permissions_headers( + requesting_user=contributor, + requested_resource=factories.ProjectFactory(creator=contributor), + ), ) with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) @@ -290,8 +303,10 @@ def test_validate_resource__noncontributor__public_resource(self, fake_gv, nonco auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', - requesting_user=noncontributor, - requested_resource=resource, + additional_headers=gv_auth.make_permissions_headers( + requesting_user=noncontributor, + requested_resource=resource, + ), ) with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) @@ -306,8 +321,10 @@ def test_validate_resource__noncontributor__private_resource(self, fake_gv, nonc auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', - requesting_user=noncontributor, - requested_resource=resource, + additional_headers=gv_auth.make_permissions_headers( + requesting_user=noncontributor, + requested_resource=resource, + ), ) with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) @@ -322,7 +339,9 @@ def test_validate_resource__unauthenticated_user__public_resource(self, fake_gv, auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', - requested_resource=resource, + additional_headers=gv_auth.make_permissions_headers( + requested_resource=resource + ), ) with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) @@ -337,7 +356,9 @@ def test_validate_resource__unauthenticated_user__private_resource(self, fake_gv auth_headers = gv_auth.make_gravy_valet_hmac_headers( request_url=request_url, request_method='GET', - requested_resource=resource, + additional_headers=gv_auth.make_permissions_headers( + requested_resource=resource, + ) ) with fake_gv.run_fake(): resp = requests.get(request_url, headers=auth_headers) @@ -375,3 +396,91 @@ def external_account(self, fake_gv, contributor, external_service): @pytest.fixture def configured_addon(self, fake_gv, resource, external_account): return fake_gv.configure_fake_addon(resource, external_account) + + def test_get_account(self, contributor, external_account, external_service, fake_gv): + with fake_gv.run_fake(): + result = gv_requests.get_account( + gv_account_pk=external_account.pk, + requesting_user=contributor + ) + retrieved_id = result.resource_id + assert retrieved_id == external_account.pk + retrieved_account_name = result.get_attribute('display_name') + assert retrieved_account_name == external_account.display_name + retrieved_service_name = result.get_included_attribute( + include_path=['external_storage_service'], + attribute_name='display_name' + ) + assert retrieved_service_name == external_service.name + + def test_get_addon(self, resource, contributor, configured_addon, external_account, external_service, fake_gv): + with fake_gv.run_fake(): + result = gv_requests.get_addon( + gv_addon_pk=external_account.pk, + requested_resource=resource, + requesting_user=contributor, + ) + retrieved_id = result.resource_id + assert retrieved_id == configured_addon.pk + retrieved_addon_name = result.get_attribute('display_name') + assert retrieved_addon_name == configured_addon.display_name + retrieved_service_name = result.get_included_attribute( + include_path=['base_account', 'external_storage_service'], + attribute_name='display_name' + ) + assert retrieved_service_name == external_service.name + + def test_get_user_accounts(self, contributor, fake_gv, external_service): + expected_account_count = 5 + expected_accounts = { + account.pk: account for account in ( + fake_gv.configure_fake_account(contributor, external_service.name) + for _ in range(expected_account_count) + ) + } + # unrelated account, will KeyError below if returned in request results + fake_gv.configure_fake_account(factories.UserFactory(), external_service.name) + with fake_gv.run_fake(): + accounts_iterator = gv_requests.iterate_accounts_for_user( + requesting_user=contributor + ) + # Need to keep this in the context manager, as generator does not fire request until first access to data + for retrieved_account in accounts_iterator: + configured_account = expected_accounts.pop(retrieved_account.resource_id) + assert retrieved_account.get_attribute('display_name') == configured_account.display_name + assert retrieved_account.get_included_attribute( + include_path=['external_storage_service'], + attribute_name='display_name' + ) == external_service.name + + assert not expected_accounts # all accounts popped + + def test_get_resource_addons(self, resource, contributor, fake_gv): + service_one = fake_gv.configure_fake_provider('argle') + service_two = fake_gv.configure_fake_provider('bargle') + account_one = fake_gv.configure_fake_account(contributor, service_one.name) + account_two = fake_gv.configure_fake_account(contributor, service_two.name) + account_three = fake_gv.configure_fake_account(contributor, service_one.name) + addon_one = fake_gv.configure_fake_addon(resource, account_one) + addon_two = fake_gv.configure_fake_addon(resource, account_two) + addon_three = fake_gv.configure_fake_addon(resource, account_three) + # Unrelated Addon, will KeyError below if retured in request results + fake_gv.configure_fake_addon(factories.ProjectFactory(creator=contributor), account_one) + + expected_addons = {addon.pk: addon for addon in [addon_one, addon_two, addon_three]} + with fake_gv.run_fake(): + addons_iterator = gv_requests.iterate_addons_for_resource( + requested_resource=resource, + requesting_user=contributor, + ) + # Need to keep this in the context manager, as generator does not fire request until first access to data + for retrieved_addon in addons_iterator: + configured_addon = expected_addons.pop(retrieved_addon.resource_id) + assert retrieved_addon.get_attribute('display_name') == configured_addon.display_name + assert retrieved_addon.get_nested_member('base_account').resource_id == configured_addon.base_account.pk + assert retrieved_addon.get_included_attribute( + include_path=['base_account', 'external_storage_service'], + attribute_name='display_name' + ) == configured_addon.base_account.external_storage_service.name + + assert not expected_addons # all addons popped From fbca992d4e91b2299b9898c785a9b95b4e667a95 Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Thu, 20 Jun 2024 13:53:26 -0400 Subject: [PATCH 6/8] Tests for translations --- .../{compat.py => translations.py} | 4 +- osf_tests/test_gv_utils.py | 58 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) rename osf/external/gravy_valet/{compat.py => translations.py} (95%) diff --git a/osf/external/gravy_valet/compat.py b/osf/external/gravy_valet/translations.py similarity index 95% rename from osf/external/gravy_valet/compat.py rename to osf/external/gravy_valet/translations.py index 55c80456a98..86adfe8c384 100644 --- a/osf/external/gravy_valet/compat.py +++ b/osf/external/gravy_valet/translations.py @@ -12,7 +12,7 @@ class _LegacyConfigsForWBKey(enum.Enum): box = BoxAddonAppConfig -def make_generic_node_settings(gv_addon_data, requested_resource, requesting_user): +def make_ephemeral_node_settings(gv_addon_data, requested_resource, requesting_user): service_wb_key = gv_addon_data.get_included_attribute( attribute_path=('base_account', 'external_storage_service'), attribute_name='wb_key' @@ -26,7 +26,7 @@ def make_generic_node_settings(gv_addon_data, requested_resource, requesting_use active_user=requesting_user, ) -def make_generic_user_settings(gv_account_data, requesting_user): +def make_ephemeral_user_settings(gv_account_data, requesting_user): service_wb_key = gv_account_data.get_included_attribute( attribute_path=('external_storage_service'), attribute_name='wb_key' diff --git a/osf_tests/test_gv_utils.py b/osf_tests/test_gv_utils.py index bbb973a77af..3f5d4af756d 100644 --- a/osf_tests/test_gv_utils.py +++ b/osf_tests/test_gv_utils.py @@ -5,6 +5,7 @@ from osf.external.gravy_valet import ( auth_helpers as gv_auth, + translations, gv_fakes, request_helpers as gv_requests ) @@ -484,3 +485,60 @@ def test_get_resource_addons(self, resource, contributor, fake_gv): ) == configured_addon.base_account.external_storage_service.name assert not expected_addons # all addons popped + + +@pytest.mark.django_db +class TestEphemeralSettings: + + @pytest.fixture + def fake_gv(self): + return gv_fakes.FakeGravyValet() + + @pytest.fixture + def fake_box(self, fake_gv): + return fake_gv.configure_fake_provider('box') + + @pytest.fixture + def contributor(self): + return factories.AuthUserFactory() + + @pytest.fixture + def project(self, contributor): + return factories.ProjectFactory(creator=contributor) + + @pytest.fixture + def fake_box_account(self, fake_gv, fake_box, contributor): + return fake_gv.configure_fake_account(contributor, fake_box.name) + + @pytest.fixture + def fake_box_addon(self, fake_gv, project, fake_box_account): + return fake_gv.configure_fake_addon(project, fake_box_account) + + def test_make_fake_user_settings(self, contributor, fake_box_account, fake_gv): + with fake_gv.run_fake(): + account_data = gv_requests.get_account( + gv_account_id=fake_box_account.pk, + requesting_user=contributor, + ) + ephemeral_config = translations.make_epehemral_user_settings(account_data, requesting_user=contributor) + assert ephemeral_config.short_name == 'box' + assert ephemeral_config.gv_id == fake_box_account.pk + assert ephemeral_config.config.name == 'addons.box' + + def testmake_fake_node_settings(self, contributor, project, fake_box_addon, fake_gv): + with fake_gv.run_fake(): + addon_data = gv_requests.get_addon( + gv_addon_id=fake_box_addon.pk, + requesting_user=contributor, + requested_resource=project, + ) + ephemeral_config = translations.make_epehemral_node_settings( + addon_data, requesting_user=contributor, requested_resource=project + ) + assert ephemeral_config.short_name == 'box' + assert ephemeral_config.gv_id == fake_box_addon.pk + assert ephemeral_config.config.name == 'addons.box' + assert ephemeral_config.serialize_waterbutler_settings == { + 'folder': fake_box_addon.root_folder, + 'service': 'box' + } From efafb0fa455d2aaa391e795dfeeee3fa09397335 Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Thu, 20 Jun 2024 14:40:53 -0400 Subject: [PATCH 7/8] Tests for ephemeral configs --- osf/external/gravy_valet/request_helpers.py | 3 +- osf/external/gravy_valet/translations.py | 43 +++++++++++---------- osf_tests/test_gv_utils.py | 24 ++++++------ 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/osf/external/gravy_valet/request_helpers.py b/osf/external/gravy_valet/request_helpers.py index 3e2e11c494c..8bc4f21dc53 100644 --- a/osf/external/gravy_valet/request_helpers.py +++ b/osf/external/gravy_valet/request_helpers.py @@ -3,6 +3,7 @@ import logging import dataclasses import requests +import typing from . import auth_helpers from website import settings @@ -185,7 +186,7 @@ def get_related_link(self, relationship_name): def get_included_member(self, relationship_name): return self._includes.get(relationship_name) - def get_included_attribute(self, include_path: list, attribute_name: str): + def get_included_attribute(self, include_path: typing.Iterator, attribute_name: str): related_object = self for relationship_name in include_path: related_object = related_object.get_included_member(relationship_name) diff --git a/osf/external/gravy_valet/translations.py b/osf/external/gravy_valet/translations.py index 86adfe8c384..312a4d4996a 100644 --- a/osf/external/gravy_valet/translations.py +++ b/osf/external/gravy_valet/translations.py @@ -12,29 +12,30 @@ class _LegacyConfigsForWBKey(enum.Enum): box = BoxAddonAppConfig -def make_ephemeral_node_settings(gv_addon_data, requested_resource, requesting_user): - service_wb_key = gv_addon_data.get_included_attribute( - attribute_path=('base_account', 'external_storage_service'), +def make_ephemeral_user_settings(gv_account_data, requesting_user): + service_wb_key = gv_account_data.get_included_attribute( + include_path=('external_storage_service', ), attribute_name='wb_key' ) - legacy_config = _LegacyConfigsForWBKey(service_wb_key) - return EphemeralNodeSettings( + legacy_config = _LegacyConfigsForWBKey[service_wb_key].value + return EphemeralUserSettings( config=EphemeralAddonConfig.from_legacy_config(legacy_config), - gv_id=gv_addon_data.resource_id, - folder_id=gv_addon_data.get_attribute('configured_root_id'), - configured_resource=requested_resource, + gv_id=gv_account_data.resource_id, active_user=requesting_user, ) -def make_ephemeral_user_settings(gv_account_data, requesting_user): - service_wb_key = gv_account_data.get_included_attribute( - attribute_path=('external_storage_service'), + +def make_ephemeral_node_settings(gv_addon_data, requested_resource, requesting_user): + service_wb_key = gv_addon_data.get_included_attribute( + include_path=('base_account', 'external_storage_service'), attribute_name='wb_key' ) - legacy_config = _LegacyConfigsForWBKey(service_wb_key) - return EphemeralUserSettings( + legacy_config = _LegacyConfigsForWBKey[service_wb_key].value + return EphemeralNodeSettings( config=EphemeralAddonConfig.from_legacy_config(legacy_config), - gv_id=gv_account_data.resource_id, + gv_id=gv_addon_data.resource_id, + folder_id=gv_addon_data.get_attribute('root_folder'), + configured_resource=requested_resource, active_user=requesting_user, ) @@ -48,8 +49,8 @@ class EphemeralAddonConfig: full_name: str @classmethod - def from_legacy_config(legacy_config): - return EphemeralAddonConfig( + def from_legacy_config(cls, legacy_config): + return cls( name=legacy_config.name, label=legacy_config.label, full_name=legacy_config.full_name, @@ -63,27 +64,28 @@ class EphemeralNodeSettings: config: EphemeralAddonConfig folder_id: str gv_id: str - _storage_config: dict | None - _credentials: dict | None # These are needed in order to make further requests for credentials configured_resource: Node active_user: OSFUser + # retrieved from WB on-demand and cached + _credentials: dict = None + @property def short_name(self): return self.config.short_name def serialize_waterbutler_credentials(self): # sufficient for most OAuth services, including Box - # TODO: Define per-service translation (or common schemes) + # TODO: Define per-service translation (and/or common schemes) if self._credentials is None: self._fetch_wb_config() return self._credentials def serialize_waterbutler_settings(self): # sufficient for Box - # TODO: Define per-service translation (or common schemes) + # TODO: Define per-service translation (and/or common schemes) return { 'folder': self.folder_id, 'service': self.short_name, @@ -96,7 +98,6 @@ def _fetch_wb_config(self): requesting_user=self.active_user ) self._credentials = result.get_attribute('credentials') - self._storage_config = result.get_attribute('settings') @dataclasses.dataclass diff --git a/osf_tests/test_gv_utils.py b/osf_tests/test_gv_utils.py index 3f5d4af756d..7ac9899652a 100644 --- a/osf_tests/test_gv_utils.py +++ b/osf_tests/test_gv_utils.py @@ -409,7 +409,7 @@ def test_get_account(self, contributor, external_account, external_service, fake retrieved_account_name = result.get_attribute('display_name') assert retrieved_account_name == external_account.display_name retrieved_service_name = result.get_included_attribute( - include_path=['external_storage_service'], + include_path=('external_storage_service',), attribute_name='display_name' ) assert retrieved_service_name == external_service.name @@ -426,7 +426,7 @@ def test_get_addon(self, resource, contributor, configured_addon, external_accou retrieved_addon_name = result.get_attribute('display_name') assert retrieved_addon_name == configured_addon.display_name retrieved_service_name = result.get_included_attribute( - include_path=['base_account', 'external_storage_service'], + include_path=('base_account', 'external_storage_service'), attribute_name='display_name' ) assert retrieved_service_name == external_service.name @@ -450,7 +450,7 @@ def test_get_user_accounts(self, contributor, fake_gv, external_service): configured_account = expected_accounts.pop(retrieved_account.resource_id) assert retrieved_account.get_attribute('display_name') == configured_account.display_name assert retrieved_account.get_included_attribute( - include_path=['external_storage_service'], + include_path=('external_storage_service',), attribute_name='display_name' ) == external_service.name @@ -478,9 +478,9 @@ def test_get_resource_addons(self, resource, contributor, fake_gv): for retrieved_addon in addons_iterator: configured_addon = expected_addons.pop(retrieved_addon.resource_id) assert retrieved_addon.get_attribute('display_name') == configured_addon.display_name - assert retrieved_addon.get_nested_member('base_account').resource_id == configured_addon.base_account.pk + assert retrieved_addon.get_included_member('base_account').resource_id == configured_addon.base_account.pk assert retrieved_addon.get_included_attribute( - include_path=['base_account', 'external_storage_service'], + include_path=('base_account', 'external_storage_service'), attribute_name='display_name' ) == configured_addon.base_account.external_storage_service.name @@ -514,31 +514,31 @@ def fake_box_account(self, fake_gv, fake_box, contributor): def fake_box_addon(self, fake_gv, project, fake_box_account): return fake_gv.configure_fake_addon(project, fake_box_account) - def test_make_fake_user_settings(self, contributor, fake_box_account, fake_gv): + def test_make_ephemeral_user_settings(self, contributor, fake_box_account, fake_gv): with fake_gv.run_fake(): account_data = gv_requests.get_account( - gv_account_id=fake_box_account.pk, + gv_account_pk=fake_box_account.pk, requesting_user=contributor, ) - ephemeral_config = translations.make_epehemral_user_settings(account_data, requesting_user=contributor) + ephemeral_config = translations.make_ephemeral_user_settings(account_data, requesting_user=contributor) assert ephemeral_config.short_name == 'box' assert ephemeral_config.gv_id == fake_box_account.pk assert ephemeral_config.config.name == 'addons.box' - def testmake_fake_node_settings(self, contributor, project, fake_box_addon, fake_gv): + def test_make_ephemeral_node_settings(self, contributor, project, fake_box_addon, fake_gv): with fake_gv.run_fake(): addon_data = gv_requests.get_addon( - gv_addon_id=fake_box_addon.pk, + gv_addon_pk=fake_box_addon.pk, requesting_user=contributor, requested_resource=project, ) - ephemeral_config = translations.make_epehemral_node_settings( + ephemeral_config = translations.make_ephemeral_node_settings( addon_data, requesting_user=contributor, requested_resource=project ) assert ephemeral_config.short_name == 'box' assert ephemeral_config.gv_id == fake_box_addon.pk assert ephemeral_config.config.name == 'addons.box' - assert ephemeral_config.serialize_waterbutler_settings == { + assert ephemeral_config.serialize_waterbutler_settings() == { 'folder': fake_box_addon.root_folder, 'service': 'box' } From 2f16541d6e4e218297f30e8723ed2f791ef84be1 Mon Sep 17 00:00:00 2001 From: Jon Walz Date: Thu, 20 Jun 2024 17:43:09 -0400 Subject: [PATCH 8/8] CR feedback --- osf/external/gravy_valet/request_helpers.py | 42 ++++++++++++++----- osf/external/gravy_valet/translations.py | 2 +- .../external/gravy_valet/gv_fakes.py | 4 +- osf_tests/test_gv_utils.py | 2 +- 4 files changed, 35 insertions(+), 15 deletions(-) rename {osf => osf_tests}/external/gravy_valet/gv_fakes.py (99%) diff --git a/osf/external/gravy_valet/request_helpers.py b/osf/external/gravy_valet/request_helpers.py index 8bc4f21dc53..72a7911e6bb 100644 --- a/osf/external/gravy_valet/request_helpers.py +++ b/osf/external/gravy_valet/request_helpers.py @@ -89,12 +89,22 @@ def get_waterbutler_config(gv_addon_pk, requested_resource, requesting_user): # ) -def get_gv_result(**kwargs): # -> JSONAPIResultEntry - '''Processes the result of a request to a GravyValet detail endpoint into a single JSONAPIResultEntry. +def get_gv_result( + endpoint_url: str, + requesting_user, + requested_resource=None, + request_method='GET', + params: dict = None, +): # -> JSONAPIResultEntry + '''Processes the result of a request to a GravyValet detail endpoint into a single JSONAPIResultEntry.''' + response_json = _make_gv_request( + endpoint_url=endpoint_url, + requesting_user=requesting_user, + requested_resource=requested_resource, + request_method=request_method, + params=params, + ).json() - kwargs must match _make_gv_request - ''' - response_json = _make_gv_request(**kwargs).json() if not response_json['data']: return None data = response_json['data'] @@ -104,12 +114,22 @@ def get_gv_result(**kwargs): # -> JSONAPIResultEntry return JSONAPIResultEntry(data, included_entities_lookup) -def iterate_gv_results(**kwargs): # -> typing.Iterator[JSONAPIResultEntry] - '''Processes the result of a request to GravyValet list endpoint into a generator of JSONAPIResultEntires. +def iterate_gv_results( + endpoint_url: str, + requesting_user, + requested_resource=None, + request_method='GET', + params: dict = None, +): # -> typing.Iterator[JSONAPIResultEntry] + '''Processes the result of a request to GravyValet list endpoint into a generator of JSONAPIResultEntires.''' + response_json = _make_gv_request( + endpoint_url=endpoint_url, + requesting_user=requesting_user, + requested_resource=requested_resource, + request_method=request_method, + params=params + ).json() - kwargs must match _make_gv_request - ''' - response_json = _make_gv_request(**kwargs).json() if not response_json['data']: return # empty iterator included_entities_lookup = _format_included_entities(response_json.get('included', [])) @@ -122,7 +142,7 @@ def _make_gv_request( requesting_user, requested_resource=None, request_method='GET', - params: dict = None + params: dict = None, ): '''Generates HMAC-Signed auth headers and makes a request to GravyValet, returning the result.''' full_url = urlunparse(urlparse(endpoint_url)._replace(query=urlencode(params))) diff --git a/osf/external/gravy_valet/translations.py b/osf/external/gravy_valet/translations.py index 312a4d4996a..c5d9f98ed7e 100644 --- a/osf/external/gravy_valet/translations.py +++ b/osf/external/gravy_valet/translations.py @@ -8,7 +8,7 @@ class _LegacyConfigsForWBKey(enum.Enum): - """Mapping from a GravyValet StorageImp name to the Addon config.""" + """Mapping from a GV ExternalStorageService's waterbutler key to the legacy Addon config.""" box = BoxAddonAppConfig diff --git a/osf/external/gravy_valet/gv_fakes.py b/osf_tests/external/gravy_valet/gv_fakes.py similarity index 99% rename from osf/external/gravy_valet/gv_fakes.py rename to osf_tests/external/gravy_valet/gv_fakes.py index 7a0a60bd3f3..3144484fdd1 100644 --- a/osf/external/gravy_valet/gv_fakes.py +++ b/osf_tests/external/gravy_valet/gv_fakes.py @@ -10,7 +10,7 @@ import dataclasses # backport import responses -from . import auth_helpers +from osf.external.gravy_valet import auth_helpers from osf.models import OSFUser, AbstractNode from osf.utils import permissions as osf_permissions from website import settings @@ -491,7 +491,7 @@ def _format_response_body( list_view: bool = False, include_param='', ) -> str: - """Returns the expected (status, headers, json) tuple expected by callbacks for FakeRequest.""" + """Formates the stringified json body for responses.""" if list_view: if not isinstance(data, list): data = [data] diff --git a/osf_tests/test_gv_utils.py b/osf_tests/test_gv_utils.py index 7ac9899652a..a8525a0ac61 100644 --- a/osf_tests/test_gv_utils.py +++ b/osf_tests/test_gv_utils.py @@ -6,10 +6,10 @@ from osf.external.gravy_valet import ( auth_helpers as gv_auth, translations, - gv_fakes, request_helpers as gv_requests ) from osf_tests import factories +from osf_tests.external.gravy_valet import gv_fakes from website.settings import GRAVYVALET_URL logger = logging.getLogger(__name__)