From a1f8b00172d67a946e0e826651ba443c64b91108 Mon Sep 17 00:00:00 2001 From: abram axel booth Date: Thu, 9 Oct 2025 14:47:41 -0400 Subject: [PATCH] chore: remove 'old' institution dashboard stuff --- api/base/pagination.py | 17 ++ api/base/utils.py | 21 -- api/institutions/renderers.py | 39 --- api/institutions/serializers.py | 107 +------ api/institutions/urls.py | 4 +- api/institutions/views.py | 217 ++++----------- .../views/test_institution_department_list.py | 96 +++---- .../views/test_institution_summary_metrics.py | 97 +------ .../test_institution_user_metric_list.py | 262 +----------------- osf/features.yaml | 4 - .../populate_impact_institution_metrics.py | 113 -------- .../update_institution_project_counts.py | 52 ---- osf/metrics/__init__.py | 7 - osf/metrics/institution_metrics.py | 150 ---------- osf_tests/test_management_commands.py | 107 ------- tests/identifiers/test_datacite.py | 4 +- website/settings/defaults.py | 6 - 17 files changed, 113 insertions(+), 1190 deletions(-) delete mode 100644 api/institutions/renderers.py delete mode 100644 osf/management/commands/populate_impact_institution_metrics.py delete mode 100644 osf/management/commands/update_institution_project_counts.py delete mode 100644 osf/metrics/institution_metrics.py diff --git a/api/base/pagination.py b/api/base/pagination.py index a831673937c..f26e54bfbad 100644 --- a/api/base/pagination.py +++ b/api/base/pagination.py @@ -167,6 +167,23 @@ def paginate_queryset(self, queryset, request, view=None): return super().paginate_queryset(queryset, request, view=None) +class JSONAPINoPagination(pagination.BasePagination): + '''do not accept page params nor paginate the queryset, but (for consistency with + JSONAPIPagination) do format the data into a jsonapi doc + + possible future improvement: format jsonapi docs somewhere more reasonable + (like the renderer?) and delete this pagination class + ''' + def paginate_queryset(self, queryset, request, view=None): + return queryset # let it be + + def get_paginated_response(self, data): + return Response({ + 'data': data, + 'meta': {'total': len(data)}, + }) + + class MaxSizePagination(JSONAPIPagination): page_size = 1000 max_page_size = None diff --git a/api/base/utils.py b/api/base/utils.py index c08c8c73977..8c237a9aee3 100644 --- a/api/base/utils.py +++ b/api/base/utils.py @@ -266,27 +266,6 @@ def assert_resource_type(obj, resource_tuple): assert isinstance(obj, resource_tuple), f'obj must be {a_or_an} {error_message}; got {obj}' -class MockQueryset(list): - """ - This class is meant to convert a simple list into a filterable queryset look-a-like. - """ - - def __init__(self, items, search, default_attrs=None, **kwargs): - self.search = search - - for item in items: - if default_attrs: - item.update(default_attrs) - self.add_dict_as_item(item) - - def __len__(self): - return self.search.count() - - def add_dict_as_item(self, dict): - item = type('item', (object,), dict) - self.append(item) - - def toggle_view_by_flag(flag_name, old_view, new_view): '''toggle between view implementations based on a feature flag diff --git a/api/institutions/renderers.py b/api/institutions/renderers.py deleted file mode 100644 index 353fcdfcce0..00000000000 --- a/api/institutions/renderers.py +++ /dev/null @@ -1,39 +0,0 @@ -from rest_framework_csv.renderers import CSVRenderer - - -class MetricsCSVRenderer(CSVRenderer): - """ - CSVRenderer with updated render method to export `data` dictionary of API Response to CSV - """ - - def render(self, data, media_type=None, renderer_context={}, writer_opts=None): - """ - Overwrites CSVRenderer.render() to create a CSV with the data dictionary - instead of the entire API response. This is necessary for results to be - separated into different rows. - """ - data = data.get('data') - return super().render(data, media_type=media_type, renderer_context=renderer_context, writer_opts=writer_opts) - -class InstitutionUserMetricsCSVRenderer(MetricsCSVRenderer): - """ - MetricsCSVRenderer with headers and labels specific to the InstitutionUserMetrics Endpoint - """ - - header = ['id', 'attributes.user_name', 'attributes.public_projects', 'attributes.private_projects', 'type'] - labels = { - 'attributes.private_projects': 'private_projects', - 'attributes.public_projects': 'public_projects', - 'attributes.user_name': 'user_name', - } - -class InstitutionDepartmentMetricsCSVRenderer(MetricsCSVRenderer): - """ - MetricsCSVRenderer with headers and labels specific to the InstitutionDepartmentMetrics Endpoint - """ - - header = ['id', 'attributes.name', 'attributes.number_of_users', 'type'] - labels = { - 'attributes.name': 'name', - 'attributes.number_of_users': 'number_of_users', - } diff --git a/api/institutions/serializers.py b/api/institutions/serializers.py index 2af640f3b48..fac33923905 100644 --- a/api/institutions/serializers.py +++ b/api/institutions/serializers.py @@ -21,7 +21,6 @@ from api.base.serializers import YearmonthField from api.nodes.serializers import CompoundIDField from api.base.exceptions import RelationshipPostMakesNoChanges -from api.base.utils import absolute_reverse class InstitutionSerializer(JSONAPISerializer): @@ -205,30 +204,6 @@ def create(self, validated_data): } -class InstitutionSummaryMetricSerializer(JSONAPISerializer): - - class Meta: - type_ = 'institution-summary-metrics' - - id = IDField(source='institution_id', read_only=True) - public_project_count = ser.IntegerField(read_only=True) - private_project_count = ser.IntegerField(read_only=True) - user_count = ser.IntegerField(read_only=True) - - links = LinksField({ - 'self': 'get_absolute_url', - }) - - def get_absolute_url(self, obj): - return absolute_reverse( - 'institutions:institution-summary-metrics', - kwargs={ - 'institution_id': self.context['request'].parser_context['kwargs']['institution_id'], - 'version': 'v2', - }, - ) - - class UniqueDeptIDField(CompoundIDField): """Creates a unique department ID of the form "-".""" @@ -255,74 +230,9 @@ class Meta: name = ser.CharField(read_only=True) number_of_users = ser.IntegerField(read_only=True) - links = LinksField({ - 'self': 'get_absolute_url', - }) - - filterable_fields = frozenset([ - 'id', - 'name', - 'number_of_users', - ]) - def get_absolute_url(self, obj): - return absolute_reverse( - 'institutions:institution-department-metrics', - kwargs={ - 'institution_id': self.context['request'].parser_context['kwargs']['institution_id'], - 'version': 'v2', - }, - ) - - -class OldInstitutionUserMetricsSerializer(JSONAPISerializer): +class InstitutionUserMetricsSerializer(JSONAPISerializer): '''serializer for institution-users metrics - - used only when the INSTITUTIONAL_DASHBOARD_2024 feature flag is NOT active - (and should be removed when that flag is permanently active) - ''' - - class Meta: - type_ = 'institution-users' - - id = IDField(source='user_id', read_only=True) - user_name = ser.CharField(read_only=True) - public_projects = ser.IntegerField(source='public_project_count', read_only=True) - private_projects = ser.IntegerField(source='private_project_count', read_only=True) - department = ser.CharField(read_only=True) - - user = RelationshipField( - related_view='users:user-detail', - related_view_kwargs={'user_id': ''}, - ) - - links = LinksField({ - 'self': 'get_absolute_url', - }) - - filterable_fields = frozenset([ - 'id', - 'user_name', - 'public_projects', - 'private_projects', - 'department', - ]) - - def get_absolute_url(self, obj): - return absolute_reverse( - 'institutions:institution-user-metrics', - kwargs={ - 'institution_id': self.context['request'].parser_context['kwargs']['institution_id'], - 'version': 'v2', - }, - ) - - -class NewInstitutionUserMetricsSerializer(JSONAPISerializer): - '''serializer for institution-users metrics - - used only when the INSTITUTIONAL_DASHBOARD_2024 feature flag is active - (and should be renamed without "New" when that flag is permanently active) ''' class Meta: @@ -360,11 +270,6 @@ class Meta: ) contacts = ser.SerializerMethodField() - links = LinksField({}) - - def get_absolute_url(self): - return None # there is no detail view for institution-users - def get_contacts(self, obj): user = OSFUser.load(obj._d_['user_id']) if not user: @@ -381,11 +286,8 @@ def get_contacts(self, obj): return list(results) -class NewInstitutionSummaryMetricsSerializer(JSONAPISerializer): +class InstitutionSummaryMetricsSerializer(JSONAPISerializer): '''serializer for institution-summary metrics - - used only when the INSTITUTIONAL_DASHBOARD_2024 feature flag is active - (and should be renamed without "New" when that flag is permanently active) ''' class Meta: @@ -414,11 +316,6 @@ class Meta: related_view_kwargs={'institution_id': ''}, ) - links = LinksField({}) - - def get_absolute_url(self): - return None # there is no detail view for institution-users - class InstitutionRelated(JSONAPIRelationshipSerializer): id = ser.CharField(source='_id', required=False, allow_null=True) diff --git a/api/institutions/urls.py b/api/institutions/urls.py index 477fe8d9377..90b5bc3c7e7 100644 --- a/api/institutions/urls.py +++ b/api/institutions/urls.py @@ -13,7 +13,7 @@ re_path(r'^(?P\w+)/relationships/registrations/$', views.InstitutionRegistrationsRelationship.as_view(), name=views.InstitutionRegistrationsRelationship.view_name), re_path(r'^(?P\w+)/relationships/nodes/$', views.InstitutionNodesRelationship.as_view(), name=views.InstitutionNodesRelationship.view_name), re_path(r'^(?P\w+)/users/$', views.InstitutionUserList.as_view(), name=views.InstitutionUserList.view_name), - re_path(r'^(?P\w+)/metrics/summary/$', views.institution_summary_metrics_detail_view, name=views.institution_summary_metrics_detail_view.view_name), + re_path(r'^(?P\w+)/metrics/summary/$', views.InstitutionSummaryMetricsDetail.as_view(), name=views.InstitutionSummaryMetricsDetail.view_name), re_path(r'^(?P\w+)/metrics/departments/$', views.InstitutionDepartmentList.as_view(), name=views.InstitutionDepartmentList.view_name), - re_path(r'^(?P\w+)/metrics/users/$', views.institution_user_metrics_list_view, name=views.institution_user_metrics_list_view.view_name), + re_path(r'^(?P\w+)/metrics/users/$', views.InstitutionUserMetricsList.as_view(), name=views.InstitutionUserMetricsList.view_name), ] diff --git a/api/institutions/views.py b/api/institutions/views.py index b9709a15c80..c2d5cba8aa3 100644 --- a/api/institutions/views.py +++ b/api/institutions/views.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.db.models import F from rest_framework import generics from rest_framework import permissions as drf_permissions @@ -8,10 +9,7 @@ from framework.auth.oauth_scopes import CoreScopes -import osf.features -from osf.metrics import InstitutionProjectCounts from osf.models import OSFUser, Node, Institution, Registration -from osf.metrics import UserInstitutionProjectCounts from osf.metrics.reports import InstitutionalUserReport, InstitutionMonthlySummaryReport from osf.metrics.utils import YearMonth from osf.utils import permissions as osf_permissions @@ -22,18 +20,12 @@ from api.base.views import JSONAPIBaseView from api.base.serializers import JSONAPISerializer from api.base.utils import get_object_or_error, get_user_auth -from api.base.pagination import JSONAPIPagination, MaxSizePagination +from api.base.pagination import MaxSizePagination, JSONAPINoPagination from api.base.parsers import ( JSONAPIRelationshipParser, JSONAPIRelationshipParserForRegularJSON, ) -from api.base.settings import MAX_SIZE_OF_ES_QUERY from api.base.exceptions import RelationshipPostMakesNoChanges -from api.base.utils import ( - MockQueryset, - toggle_view_by_flag, -) -from api.base.settings import DEFAULT_ES_NULL_VALUE from api.metrics.permissions import IsInstitutionalMetricsUser from api.metrics.renderers import ( MetricsReportsCsvRenderer, @@ -50,14 +42,11 @@ InstitutionSerializer, InstitutionNodesRelationshipSerializer, InstitutionRegistrationsRelationshipSerializer, - InstitutionSummaryMetricSerializer, InstitutionDepartmentMetricsSerializer, - NewInstitutionUserMetricsSerializer, - OldInstitutionUserMetricsSerializer, - NewInstitutionSummaryMetricsSerializer, + InstitutionUserMetricsSerializer, + InstitutionSummaryMetricsSerializer, ) from api.institutions.permissions import UserIsAffiliated -from api.institutions.renderers import InstitutionDepartmentMetricsCSVRenderer, InstitutionUserMetricsCSVRenderer, MetricsCSVRenderer class InstitutionMixin: @@ -398,155 +387,64 @@ def create(self, *args, **kwargs): return ret -class _OldInstitutionSummaryMetrics(JSONAPIBaseView, generics.RetrieveAPIView, InstitutionMixin): - permission_classes = ( - drf_permissions.IsAuthenticatedOrReadOnly, - base_permissions.TokenHasScope, - IsInstitutionalMetricsUser, - ) - - required_read_scopes = [CoreScopes.INSTITUTION_METRICS_READ] - required_write_scopes = [CoreScopes.NULL] - +class InstitutionDepartmentList(InstitutionMixin, ElasticsearchListView): view_category = 'institutions' - view_name = 'institution-summary-metrics' - - serializer_class = InstitutionSummaryMetricSerializer - - # overrides RetrieveAPIView - def get_object(self): - institution = self.get_institution() - return InstitutionProjectCounts.get_latest_institution_project_document(institution) - + view_name = 'institution-department-metrics' -class InstitutionImpactList(JSONAPIBaseView, ListFilterMixin, generics.ListAPIView, InstitutionMixin): permission_classes = ( drf_permissions.IsAuthenticatedOrReadOnly, base_permissions.TokenHasScope, IsInstitutionalMetricsUser, ) - required_read_scopes = [CoreScopes.INSTITUTION_METRICS_READ] required_write_scopes = [CoreScopes.NULL] - view_category = 'institutions' - - @property - def is_csv_export(self): - if isinstance(self.request.accepted_renderer, MetricsCSVRenderer): - return True - return False - - @property - def pagination_class(self): - if self.is_csv_export: - return MaxSizePagination - return JSONAPIPagination - - def _format_search(self, search, default_kwargs=None): - raise NotImplementedError() - - def _paginate(self, search): - if self.pagination_class is MaxSizePagination: - return search.extra(size=MAX_SIZE_OF_ES_QUERY) - - page = self.request.query_params.get('page') - page_size = self.request.query_params.get('page[size]') - - if page_size: - page_size = int(page_size) - else: - page_size = api_settings.PAGE_SIZE - - if page: - search = search.extra(size=int(page) * page_size) - return search - - def _make_elasticsearch_results_filterable(self, search, **kwargs) -> MockQueryset: - """ - Since ES returns a list obj instead of a awesome filterable queryset we are faking the filter feature used by - querysets by create a mock queryset with limited filterbility. - - :param departments: Dict {'Department Name': 3} means "Department Name" has 3 users. - :return: mock_queryset - """ - items = self._format_search(search, default_kwargs=kwargs) - - search = self._paginate(search) - - queryset = MockQueryset(items, search) - return queryset - - # overrides RetrieveApiView - def get_queryset(self): - return self.get_queryset_from_request() - - -class InstitutionDepartmentList(InstitutionImpactList): - view_name = 'institution-department-metrics' - serializer_class = InstitutionDepartmentMetricsSerializer - renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (InstitutionDepartmentMetricsCSVRenderer,) - - ordering_fields = ('-number_of_users', 'name') - ordering = ('-number_of_users', 'name') + renderer_classes = ( + *api_settings.DEFAULT_RENDERER_CLASSES, + MetricsReportsCsvRenderer, + MetricsReportsTsvRenderer, + MetricsReportsJsonRenderer, + ) + pagination_class = JSONAPINoPagination - def _format_search(self, search, default_kwargs=None): - results = search.execute() + def get_default_search(self): + _base_search = ( + InstitutionalUserReport.search() + .filter('term', institution_id=self.get_institution()._id) + ) + _yearmonth = InstitutionalUserReport.most_recent_yearmonth(base_search=_base_search) + if _yearmonth is None: + return None + _search = ( + _base_search + .filter('term', report_yearmonth=str(_yearmonth)) + .exclude('term', user_name='Deleted user') + ) + # add aggregation on department name + _search.aggs.bucket( + 'agg_departments', + 'terms', + field='department_name', + missing=settings.DEFAULT_ES_NULL_VALUE, + size=settings.MAX_SIZE_OF_ES_QUERY, + ) + return _search - if results.aggregations: - buckets = results.aggregations['date_range']['departments'].buckets - department_data = [{'name': bucket['key'], 'number_of_users': bucket['doc_count']} for bucket in buckets] - return department_data - else: + def get_queryset(self): + # execute the search and return a list from the aggregation on department name + _search = super().get_queryset() + if not _search: return [] + _results = _search[0:0].execute() + return [ + {'name': _bucket['key'], 'number_of_users': _bucket['doc_count']} + for _bucket in _results.aggregations['agg_departments'].buckets + ] - def get_default_queryset(self): - institution = self.get_institution() - search = UserInstitutionProjectCounts.get_department_counts(institution) - return self._make_elasticsearch_results_filterable(search, id=institution._id) - - -class _OldInstitutionUserMetricsList(InstitutionImpactList): - '''list view for institution-users metrics - - used only when the INSTITUTIONAL_DASHBOARD_2024 feature flag is NOT active - (and should be removed when that flag is permanently active) - ''' - view_name = 'institution-user-metrics' - - serializer_class = OldInstitutionUserMetricsSerializer - renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (InstitutionUserMetricsCSVRenderer,) - - ordering_fields = ('user_name', 'department') - ordering = ('user_name',) - - def _format_search(self, search, default_kwargs=None): - results = search.execute() - - users = [] - for user_record in results: - record_dict = {} - record_dict.update(default_kwargs) - record_dict.update(user_record.to_dict()) - user_id = user_record.user_id - fullname = OSFUser.objects.get(guids___id=user_id).fullname - record_dict['user_name'] = fullname - users.append(record_dict) - - return users - - def get_default_queryset(self): - institution = self.get_institution() - search = UserInstitutionProjectCounts.get_current_user_metrics(institution) - return self._make_elasticsearch_results_filterable(search, id=institution._id, department=DEFAULT_ES_NULL_VALUE) - -class _NewInstitutionUserMetricsList(InstitutionMixin, ElasticsearchListView): +class InstitutionUserMetricsList(InstitutionMixin, ElasticsearchListView): '''list view for institution-users metrics - - used only when the INSTITUTIONAL_DASHBOARD_2024 feature flag is active - (and should be renamed without "New" when that flag is permanently active) ''' permission_classes = ( drf_permissions.IsAuthenticatedOrReadOnly, @@ -566,7 +464,7 @@ class _NewInstitutionUserMetricsList(InstitutionMixin, ElasticsearchListView): MetricsReportsJsonRenderer, ) - serializer_class = NewInstitutionUserMetricsSerializer + serializer_class = InstitutionUserMetricsSerializer default_ordering = '-storage_byte_count' ordering_fields = frozenset(( @@ -600,11 +498,8 @@ def get_default_search(self): ) -class _NewInstitutionSummaryMetricsDetail(JSONAPIBaseView, generics.RetrieveAPIView, InstitutionMixin): +class InstitutionSummaryMetricsDetail(JSONAPIBaseView, generics.RetrieveAPIView, InstitutionMixin): '''detail view for institution-summary metrics - - used only when the INSTITUTIONAL_DASHBOARD_2024 feature flag is active - (and should be renamed without "New" when that flag is permanently active) ''' permission_classes = ( drf_permissions.IsAuthenticatedOrReadOnly, @@ -618,7 +513,7 @@ class _NewInstitutionSummaryMetricsDetail(JSONAPIBaseView, generics.RetrieveAPIV view_category = 'institutions' view_name = 'institution-summary-metrics' - serializer_class = NewInstitutionSummaryMetricsSerializer + serializer_class = InstitutionSummaryMetricsSerializer def get_object(self): institution = self.get_institution() @@ -647,19 +542,3 @@ def get_default_search(self): 'term', report_yearmonth=str(yearmonth), ) - - -institution_summary_metrics_detail_view = toggle_view_by_flag( - flag_name=osf.features.INSTITUTIONAL_DASHBOARD_2024, - old_view=_OldInstitutionSummaryMetrics.as_view(), - new_view=_NewInstitutionSummaryMetricsDetail.as_view(), -) -institution_summary_metrics_detail_view.view_name = 'institution-summary-metrics' - - -institution_user_metrics_list_view = toggle_view_by_flag( - flag_name=osf.features.INSTITUTIONAL_DASHBOARD_2024, - old_view=_OldInstitutionUserMetricsList.as_view(), - new_view=_NewInstitutionUserMetricsList.as_view(), -) -institution_user_metrics_list_view.view_name = 'institution-user-metrics' diff --git a/api_tests/institutions/views/test_institution_department_list.py b/api_tests/institutions/views/test_institution_department_list.py index f2a335eed85..c2a5c0fcf99 100644 --- a/api_tests/institutions/views/test_institution_department_list.py +++ b/api_tests/institutions/views/test_institution_department_list.py @@ -1,4 +1,3 @@ -import time import pytest import datetime @@ -7,7 +6,8 @@ InstitutionFactory, AuthUserFactory, ) -from osf.metrics import UserInstitutionProjectCounts +from osf.metrics.reports import InstitutionalUserReport +from osf.metrics.utils import YearMonth @pytest.mark.es_metrics @@ -37,50 +37,55 @@ def user4(self): @pytest.fixture() def populate_counts(self, user, user2, user3, user4, admin, institution): # This represents a Department that had a user, but no longer has any users, so does not appear in results. - UserInstitutionProjectCounts.record( + InstitutionalUserReport( + report_yearmonth=YearMonth(2017, 2), user_id=user._id, institution_id=institution._id, - department='Old Department', + department_name='Old Department', public_project_count=1, private_project_count=1, - timestamp=datetime.date(2017, 2, 4) - ).save() + ).save(refresh=True) + + _this_month = YearMonth.from_date(datetime.date.today()) # The user has left the department - UserInstitutionProjectCounts.record( + InstitutionalUserReport( + report_yearmonth=_this_month, user_id=user._id, institution_id=institution._id, - department='New Department', + department_name='New Department', public_project_count=1, private_project_count=1, - ).save() + ).save(refresh=True) # A second user entered the department - UserInstitutionProjectCounts.record( + InstitutionalUserReport( + report_yearmonth=_this_month, user_id=user2._id, institution_id=institution._id, - department='New Department', + department_name='New Department', public_project_count=1, - private_project_count=1 - ).save() + private_project_count=1, + ).save(refresh=True) # A new department with a single user to test sorting - UserInstitutionProjectCounts.record( + InstitutionalUserReport( + report_yearmonth=_this_month, user_id=user3._id, institution_id=institution._id, - department='Smaller Department', + department_name='Smaller Department', public_project_count=1, - private_project_count=1 - ).save() + private_project_count=1, + ).save(refresh=True) # A user with no department - UserInstitutionProjectCounts.record( + InstitutionalUserReport( + report_yearmonth=_this_month, user_id=user4._id, institution_id=institution._id, public_project_count=1, - private_project_count=1 - ).save() - time.sleep(5) # ES is slow + private_project_count=1, + ).save(refresh=True) @pytest.fixture() def admin(self, institution): @@ -117,23 +122,23 @@ def test_get(self, app, url, admin, institution, populate_counts): 'name': 'New Department', 'number_of_users': 2 }, - 'links': {'self': f'http://localhost:8000/v2/institutions/{institution._id}/metrics/departments/'} + 'links': {}, }, { - 'id': f'{institution._id}-Smaller-Department', + 'id': f'{institution._id}-{DEFAULT_ES_NULL_VALUE}', 'type': 'institution-departments', 'attributes': { - 'name': 'Smaller Department', + 'name': DEFAULT_ES_NULL_VALUE, 'number_of_users': 1 }, - 'links': {'self': f'http://localhost:8000/v2/institutions/{institution._id}/metrics/departments/'} + 'links': {}, }, { - 'id': f'{institution._id}-{DEFAULT_ES_NULL_VALUE}', + 'id': f'{institution._id}-Smaller-Department', 'type': 'institution-departments', 'attributes': { - 'name': DEFAULT_ES_NULL_VALUE, + 'name': 'Smaller Department', 'number_of_users': 1 }, - 'links': {'self': f'http://localhost:8000/v2/institutions/{institution._id}/metrics/departments/'} + 'links': {}, }] # Tests CSV Export @@ -148,33 +153,10 @@ def test_get(self, app, url, admin, institution, populate_counts): rows = response_body.split('\r\n') header_row = rows[0].split(',') new_department_row = rows[1].split(',') - smaller_department_row = rows[2].split(',') - na_row = rows[3].split(',') - - assert header_row == ['id', 'name', 'number_of_users', 'type'] - assert new_department_row == [f'{institution._id}-New-Department', 'New Department', '2', 'institution-departments'] - assert smaller_department_row == [f'{institution._id}-Smaller-Department', 'Smaller Department', '1', 'institution-departments'] - assert na_row == [f'{institution._id}-N/A', 'N/A', '1', 'institution-departments'] - - def test_pagination(self, app, url, admin, institution, populate_counts): - resp = app.get(f'{url}?filter[name]=New Department', auth=admin.auth) - - assert resp.json['data'] == [{ - 'id': '{}-{}'.format(institution._id, 'New-Department'), - 'type': 'institution-departments', - 'attributes': { - 'name': 'New Department', - 'number_of_users': 2 - }, - 'links': {'self': f'http://localhost:8000/v2/institutions/{institution._id}/metrics/departments/'} - }] - - resp = app.get(f'{url}?page[size]=2', auth=admin.auth) - assert len(resp.json['data']) == 2 - assert resp.json['links']['meta']['per_page'] == 2 - assert resp.json['links']['meta']['total'] == 3 + na_row = rows[2].split(',') + smaller_department_row = rows[3].split(',') - resp = app.get(f'{url}?page[size]=2&page=2', auth=admin.auth) - assert len(resp.json['data']) == 1 - assert resp.json['links']['meta']['per_page'] == 2 - assert resp.json['links']['meta']['total'] == 3 + assert header_row == ['name', 'number_of_users'] + assert new_department_row == ['New Department', '2'] + assert smaller_department_row == ['Smaller Department', '1'] + assert na_row == ['N/A', '1'] diff --git a/api_tests/institutions/views/test_institution_summary_metrics.py b/api_tests/institutions/views/test_institution_summary_metrics.py index f1641ea923c..41983458d2e 100644 --- a/api_tests/institutions/views/test_institution_summary_metrics.py +++ b/api_tests/institutions/views/test_institution_summary_metrics.py @@ -1,8 +1,4 @@ import pytest -import datetime - -from waffle.testutils import override_flag -from osf.metrics import InstitutionProjectCounts from api.base.settings.defaults import API_BASE from osf_tests.factories import ( @@ -10,102 +6,11 @@ AuthUserFactory, ) from osf.metrics.reports import InstitutionMonthlySummaryReport -from osf import features @pytest.mark.es_metrics @pytest.mark.django_db -class TestInstitutionSummaryMetrics: - - @pytest.fixture() - def institution(self): - return InstitutionFactory() - - @pytest.fixture() - def user(self): - return AuthUserFactory() - - @pytest.fixture() - def admin(self, institution): - user = AuthUserFactory() - group = institution.get_group('institutional_admins') - group.user_set.add(user) - group.save() - return user - - @pytest.fixture() - def url(self, institution): - return f'/{API_BASE}institutions/{institution._id}/metrics/summary/' - - def test_get(self, app, url, institution, user, admin): - # Tests unauthenticated user - resp = app.get(url, expect_errors=True) - assert resp.status_code == 401 - - # Tests unauthorized user - resp = app.get(url, auth=user.auth, expect_errors=True) - assert resp.status_code == 403 - - # Record latest institutional metrics to ES - public_project_count_latest = 24 - private_project_count_latest = 26 - institution_user_count_latest = 9 - timestamp_latest = datetime.datetime.now() - - # Uses record to specify user_count - InstitutionProjectCounts.record( - institution_id=institution._id, - user_count=institution_user_count_latest, - public_project_count=public_project_count_latest, - private_project_count=private_project_count_latest, - timestamp=timestamp_latest - ).save() - - # Record earlier institutional metrics to ES - public_project_count_early = 20 - private_project_count_early = 18 - institution_user_count_early = 4 - timestamp_early = timestamp_latest - datetime.timedelta(days=1) - - # Uses record to specify user_count - InstitutionProjectCounts.record( - institution_id=institution._id, - user_count=institution_user_count_early, - public_project_count=public_project_count_early, - private_project_count=private_project_count_early, - timestamp=timestamp_early - ).save() - - import time - time.sleep(5) - - # Tests authorized user with institution with metrics - resp = app.get(url, auth=admin.auth) - assert resp.status_code == 200 - - # Validates the summary metrics returned by the API - assert resp.json['data'] == { - 'id': institution._id, - 'type': 'institution-summary-metrics', - 'attributes': { - 'public_project_count': public_project_count_latest, - 'private_project_count': private_project_count_latest, - 'user_count': institution_user_count_latest - }, - 'links': { - 'self': f'http://localhost:8000/v2/institutions/{institution._id}/metrics/summary/' - } - } - - -@pytest.mark.es_metrics -@pytest.mark.django_db -class TestNewInstitutionSummaryMetricsList: - @pytest.fixture(autouse=True) - def _waffled(self): - with override_flag(features.INSTITUTIONAL_DASHBOARD_2024, active=True): - yield - +class TestInstitutionSummaryMetricsList: @pytest.fixture() def institution(self): return InstitutionFactory() diff --git a/api_tests/institutions/views/test_institution_user_metric_list.py b/api_tests/institutions/views/test_institution_user_metric_list.py index 3c0e1bf055f..ef9ea28b0bd 100644 --- a/api_tests/institutions/views/test_institution_user_metric_list.py +++ b/api_tests/institutions/views/test_institution_user_metric_list.py @@ -2,281 +2,23 @@ import csv import json from io import StringIO -from random import random from urllib.parse import urlencode import pytest -from waffle.testutils import override_flag -from api.base.settings.defaults import API_BASE, DEFAULT_ES_NULL_VALUE, REPORT_FILENAME_FORMAT -import osf.features +from api.base.settings.defaults import API_BASE, REPORT_FILENAME_FORMAT from osf_tests.factories import ( InstitutionFactory, AuthUserFactory, ) -from osf.metrics import UserInstitutionProjectCounts from osf.metrics.reports import InstitutionalUserReport from osf.models import UserMessage @pytest.mark.es_metrics @pytest.mark.django_db -class TestOldInstitutionUserMetricList: - - @pytest.fixture(autouse=True) - def _waffled(self): - with override_flag(osf.features.INSTITUTIONAL_DASHBOARD_2024, active=False): - yield # these tests apply only before institution dashboard improvements - - @pytest.fixture() - def institution(self): - return InstitutionFactory() - - @pytest.fixture() - def user(self): - user = AuthUserFactory() - user.fullname = user.fullname + ',a' - user.save() - return user - - @pytest.fixture() - def user2(self): - return AuthUserFactory() - - @pytest.fixture() - def user3(self): - return AuthUserFactory(fullname='Zedd') - - @pytest.fixture() - def user4(self): - return AuthUserFactory() - - @pytest.fixture() - def admin(self, institution): - user = AuthUserFactory() - group = institution.get_group('institutional_admins') - group.user_set.add(user) - group.save() - return user - - @pytest.fixture() - def populate_counts(self, institution, user, user2): - # Old data that shouldn't appear in responses - UserInstitutionProjectCounts( - user_id=user._id, - institution_id=institution._id, - department='Biology dept', - public_project_count=4, - private_project_count=4, - timestamp=datetime.date(2019, 6, 4) - ).save(refresh=True) - - # New data - UserInstitutionProjectCounts( - user_id=user._id, - institution_id=institution._id, - department='Biology dept', - public_project_count=6, - private_project_count=5, - ).save(refresh=True) - - UserInstitutionProjectCounts( - user_id=user2._id, - institution_id=institution._id, - department='Psychology dept', - public_project_count=3, - private_project_count=2, - ).save(refresh=True) - - @pytest.fixture() - def populate_more_counts(self, institution, user, user2, user3, populate_counts): - # Creates 9 more user records to test pagination with - - users = [] - for i in range(0, 8): - users.append(AuthUserFactory()) - - for test_user in users: - UserInstitutionProjectCounts( - user_id=test_user._id, - institution_id=institution._id, - department='Psychology dept', - public_project_count=int(10 * random()), - private_project_count=int(10 * random()), - ).save(refresh=True) - - UserInstitutionProjectCounts( - user_id=user3._id, - institution_id=institution._id, - department='Psychology dept', - public_project_count=int(10 * random()), - private_project_count=int(10 * random()), - ).save(refresh=True) - - @pytest.fixture() - def populate_na_department(self, institution, user4): - UserInstitutionProjectCounts( - user_id=user4._id, - institution_id=institution._id, - public_project_count=1, - private_project_count=1, - ).save(refresh=True) - - @pytest.fixture() - def url(self, institution): - return f'/{API_BASE}institutions/{institution._id}/metrics/users/' - - def test_auth(self, app, url, user, admin): - - resp = app.get(url, expect_errors=True) - assert resp.status_code == 401 - - resp = app.get(url, auth=user.auth, expect_errors=True) - assert resp.status_code == 403 - - resp = app.get(url, auth=admin.auth) - assert resp.status_code == 200 - - assert resp.json['data'] == [] - - def test_get(self, app, url, user, user2, admin, institution, populate_counts): - resp = app.get(url, auth=admin.auth) - - assert resp.json['data'] == [ - { - 'id': user._id, - 'type': 'institution-users', - 'attributes': { - 'user_name': user.fullname, - 'public_projects': 6, - 'private_projects': 5, - 'department': 'Biology dept' - }, - 'relationships': { - 'user': { - 'links': { - 'related': { - 'href': f'http://localhost:8000/v2/users/{user._id}/', - 'meta': {} - } - }, - 'data': { - 'id': user._id, - 'type': 'users' - } - } - }, - 'links': { - 'self': f'http://localhost:8000/v2/institutions/{institution._id}/metrics/users/' - } - }, - { - 'id': user2._id, - 'type': 'institution-users', - 'attributes': { - 'user_name': user2.fullname, - 'public_projects': 3, - 'private_projects': 2, - 'department': 'Psychology dept' - }, - 'relationships': { - 'user': { - 'links': { - 'related': { - 'href': f'http://localhost:8000/v2/users/{user2._id}/', - 'meta': {} - } - }, - 'data': { - 'id': user2._id, - 'type': 'users' - } - } - }, - 'links': { - 'self': f'http://localhost:8000/v2/institutions/{institution._id}/metrics/users/' - } - } - ] - - # Tests CSV Export - headers = { - 'accept': 'text/csv' - } - resp = app.get(url, auth=admin.auth, headers=headers) - assert resp.status_code == 200 - assert resp.headers['Content-Type'] == 'text/csv; charset=utf-8' - - response_body = resp.text - - expected_response = [['id', 'user_name', 'public_projects', 'private_projects', 'type'], - [user._id, user.fullname, '6', '5', 'institution-users'], - [user2._id, user2.fullname, '3', '2', 'institution-users']] - - with StringIO(response_body) as csv_file: - csvreader = csv.reader(csv_file, delimiter=',') - for index, row in enumerate(csvreader): - assert row == expected_response[index] - - def test_filter(self, app, url, admin, populate_counts): - resp = app.get(f'{url}?filter[department]=Psychology dept', auth=admin.auth) - assert resp.json['data'][0]['attributes']['department'] == 'Psychology dept' - - def test_sort_and_pagination(self, app, url, user, user2, user3, admin, populate_counts, populate_more_counts, institution): - resp = app.get(f'{url}?sort=user_name&page[size]=1&page=2', auth=admin.auth) - assert resp.status_code == 200 - assert resp.json['links']['meta']['total'] == 11 - resp = app.get(f'{url}?sort=user_name&page[size]=1&page=11', auth=admin.auth) - assert resp.json['data'][0]['attributes']['user_name'] == 'Zedd' - resp = app.get(f'{url}?sort=user_name&page=2', auth=admin.auth) - assert resp.json['links']['meta']['total'] == 11 - assert resp.json['data'][-1]['attributes']['user_name'] == 'Zedd' - - def test_filter_and_pagination(self, app, user, user2, user3, url, admin, populate_counts, populate_more_counts, institution): - resp = app.get(f'{url}?page=2', auth=admin.auth) - assert resp.json['links']['meta']['total'] == 11 - assert resp.json['data'][0]['attributes']['user_name'] == 'Zedd' - resp = app.get(f'{url}?filter[user_name]=Zedd', auth=admin.auth) - assert resp.json['links']['meta']['total'] == 1 - assert resp.json['data'][0]['attributes']['user_name'] == 'Zedd' - - def test_filter_and_sort(self, app, url, user, user2, user3, admin, user4, populate_counts, populate_na_department, institution): - """ - Testing for bug where sorting and filtering would throw 502. - :param app: - :param url: - :param admin: - :param populate_more_counts: - :return: - """ - resp = app.get(f'{url}?page=1&page[size]=10&filter[department]={DEFAULT_ES_NULL_VALUE}&sort=user_name', auth=admin.auth) - assert resp.status_code == 200 - - data = resp.json['data'] - assert len(data) == 1 - assert resp.json['links']['meta']['total'] == 1 - assert data[0]['id'] == user4._id - - resp = app.get(f'{url}?page=1&page[size]=10&sort=department', auth=admin.auth) - assert resp.status_code == 200 - - data = resp.json['data'] - assert len(data) == 3 - assert resp.json['links']['meta']['total'] == 3 - assert data[0]['attributes']['department'] == 'Biology dept' - assert data[1]['attributes']['department'] == 'N/A' - assert data[2]['attributes']['department'] == 'Psychology dept' - - -@pytest.mark.es_metrics -@pytest.mark.django_db -class TestNewInstitutionUserMetricList: - @pytest.fixture(autouse=True) - def _waffled(self): - with override_flag(osf.features.INSTITUTIONAL_DASHBOARD_2024, active=True): - yield # these tests apply only after institution dashboard improvements - +class TestInstitutionUserMetricList: @pytest.fixture() def institution(self): return InstitutionFactory() diff --git a/osf/features.yaml b/osf/features.yaml index 98eff0aef9a..2391482220f 100644 --- a/osf/features.yaml +++ b/osf/features.yaml @@ -195,10 +195,6 @@ flags: note: This is not used everyone: true - - flag_name: INSTITUTIONAL_DASHBOARD_2024 - name: institutional_dashboard_2024 - note: whether to surface older or updated (in 2024) institutional metrics - - flag_name: DISABLE_COMMENTS name: disable_comments note: This flag controls wheter users can create or interact with comments via BE or FE. diff --git a/osf/management/commands/populate_impact_institution_metrics.py b/osf/management/commands/populate_impact_institution_metrics.py deleted file mode 100644 index 9c8ddd03a80..00000000000 --- a/osf/management/commands/populate_impact_institution_metrics.py +++ /dev/null @@ -1,113 +0,0 @@ -import datetime as dt -from random import random -from django.core.management.base import BaseCommand - -from osf.metrics import ( - InstitutionProjectCounts, - UserInstitutionProjectCounts, -) - -from osf.models import Institution, OSFUser - - -""" -This management command can be run to populate impact with fake -institutional metrics data. - -All flags are optional with the script defaulting to 3 users and 1 institution -from your local database with metrics for the past 7 days. - ---users: Specify user guids ---num_users: Specify the number of users to use from the database (if -user guids aren't specified) ---institutions: Specify institution guids ---num_institutions: Specify the number of institutions to use from the database -(if institution guids aren't specified) ---days: Specify the number of days to write metrics data for - -Example: docker-compose run --rm web python3 manage.py populate_impact_institution_metrics --days 3 --institutions cos --users hy84n bd53u -""" - - -def populate_institution_metrics(users, institutions, dates): - institution_public_project_count = 25 + (int(25 * random())) - institution_private_project_count = 25 + (int(25 * random())) - - for date in dates: - - for institution in institutions: - institution_public_project_count = institution_public_project_count - int(5 * random()) - institution_private_project_count = institution_private_project_count - int(5 * random()) - - InstitutionProjectCounts.record_institution_project_counts( - institution=institution, - public_project_count=institution_public_project_count, - private_project_count=institution_private_project_count, - timestamp=date - ) - - for user in users: - user_public_project_count = (int(10 * random())) - user_private_project_count = (int(10 * random())) - - UserInstitutionProjectCounts.record_user_institution_project_counts( - institution=institution, - user=user, - public_project_count=user_public_project_count, - private_project_count=user_private_project_count, - timestamp=date - ) - - -class Command(BaseCommand): - - def add_arguments(self, parser): - super().add_arguments(parser) - parser.add_argument( - '--users', - nargs='*', - help='Specify user guids' - ) - parser.add_argument( - '--num_users', - type=int, - default=3, - help='Specify number of users to use if not specifying users' - ) - parser.add_argument( - '--institutions', - nargs='*', - help='Specify insitutions guids' - ) - parser.add_argument( - '--num_institutions', - type=int, - default=1, - help='Specify number of institutions to use if not specifying institutions' - ) - parser.add_argument( - '--days', - type=int, - default=7, - help='Specify number of past days to write metrics data for' - ) - - def handle(self, *args, **options): - days = options.get('days') - num_users = options.get('num_users') - num_institutions = options.get('num_institutions') - - if options.get('users'): - users = OSFUser.objects.filter(guids___id__in=options.get('users')) - else: - users = OSFUser.objects.all()[:num_users] - - if options.get('institutions'): - institutions = Institution.objects.filter(_id__in=options.get('institutions')) - else: - institutions = Institution.objects.all()[:num_institutions] - - today = dt.datetime.today() - last_x_days = [(today - dt.timedelta(days=num_days)) for num_days in range(0, days)] - - populate_institution_metrics(users, institutions, last_x_days) diff --git a/osf/management/commands/update_institution_project_counts.py b/osf/management/commands/update_institution_project_counts.py deleted file mode 100644 index 53a0d735ddc..00000000000 --- a/osf/management/commands/update_institution_project_counts.py +++ /dev/null @@ -1,52 +0,0 @@ -import datetime as dt - -from django.core.management.base import BaseCommand - -from framework.celery_tasks import app as celery_app -from osf.metrics import InstitutionProjectCounts, UserInstitutionProjectCounts -from osf.models import Institution, Node - - -@celery_app.task(name='management.commands.update_institution_project_counts') -def update_institution_project_counts(): - now = dt.datetime.now() - - for institution in Institution.objects.all(): - - institution_public_projects_qs = institution.nodes.filter(type='osf.node', parent_nodes=None, is_public=True, is_deleted=False) - institution_private_projects_qs = institution.nodes.filter(type='osf.node', parent_nodes=None, is_public=False, is_deleted=False) - - institution_public_projects_count = institution_public_projects_qs.count() - institution_private_projects_count = institution_private_projects_qs.count() - - InstitutionProjectCounts.record_institution_project_counts( - institution=institution, - public_project_count=institution_public_projects_count, - private_project_count=institution_private_projects_count, - timestamp=now - ) - - for user in institution.get_institution_users(): - user_public_project_count = Node.objects.get_nodes_for_user( - user=user, - base_queryset=institution_public_projects_qs - ).count() - - user_private_project_count = Node.objects.get_nodes_for_user( - user=user, - base_queryset=institution_private_projects_qs - ).count() - - UserInstitutionProjectCounts.record_user_institution_project_counts( - user=user, - institution=institution, - public_project_count=user_public_project_count, - private_project_count=user_private_project_count, - timestamp=now - ) - - -class Command(BaseCommand): - - def handle(self, *args, **options): - update_institution_project_counts() diff --git a/osf/metrics/__init__.py b/osf/metrics/__init__.py index 9ac7e49a573..0e7b1a1cf32 100644 --- a/osf/metrics/__init__.py +++ b/osf/metrics/__init__.py @@ -5,11 +5,6 @@ PreprintDownload, ) -from .institution_metrics import ( - InstitutionProjectCounts, - UserInstitutionProjectCounts, -) - from .registry_metrics import RegistriesModerationMetrics from .reports import ( @@ -38,9 +33,7 @@ __all__ = ( 'CountedAuthUsage', 'DAILY_REPORTS', - 'InstitutionProjectCounts', 'PreprintView', 'PreprintDownload', 'RegistriesModerationMetrics', - 'UserInstitutionProjectCounts', ) diff --git a/osf/metrics/institution_metrics.py b/osf/metrics/institution_metrics.py deleted file mode 100644 index f04bb53ce49..00000000000 --- a/osf/metrics/institution_metrics.py +++ /dev/null @@ -1,150 +0,0 @@ -from datetime import datetime, timedelta - -from elasticsearch_metrics import metrics - -from api.base.settings import MAX_SIZE_OF_ES_QUERY, DEFAULT_ES_NULL_VALUE -from .metric_mixin import MetricMixin - - -class UserInstitutionProjectCounts(MetricMixin, metrics.Metric): - user_id = metrics.Keyword(index=True, doc_values=True, required=True) - institution_id = metrics.Keyword(index=True, doc_values=True, required=True) - department = metrics.Keyword(index=True, doc_values=True, required=False) - public_project_count = metrics.Integer(index=True, doc_values=True, required=True) - private_project_count = metrics.Integer(index=True, doc_values=True, required=True) - - class Index: - settings = { - 'number_of_shards': 1, - 'number_of_replicas': 1, - 'refresh_interval': '1s', - } - - class Meta: - source = metrics.MetaField(enabled=True) - - @classmethod - def filter_institution(cls, institution): - return cls.search().filter('match', institution_id=institution._id) - - @classmethod - def get_recent_datetime(cls, institution): - search = cls.filter_institution(institution).sort('-timestamp') - - # Rounding to the nearest minute - results = search.execute() - if results: - return search.execute()[0].timestamp.replace(microsecond=0, second=0) - # If there are no results, assume yesterday. - return datetime.now() - timedelta(days=1) - - @classmethod - def get_department_counts(cls, institution) -> list: - """ - Gets the most recent document for every unique user. - :param institution: Institution - :return: list - """ - search = cls.filter_institution(institution).sort('timestamp') - last_record_time = cls.get_recent_datetime(institution) - - return search.update_from_dict({ - 'aggs': { - 'date_range': { - 'filter': { - 'range': { - 'timestamp': { - 'gte': last_record_time, - } - } - }, - 'aggs': { - 'departments': { - 'terms': { - 'field': 'department', - 'missing': DEFAULT_ES_NULL_VALUE, - 'size': 250 - }, - 'aggs': { - 'users': { - 'terms': { - 'field': 'user_id' - } - } - } - } - } - } - } - }) - - @classmethod - def record_user_institution_project_counts(cls, user, institution, public_project_count, private_project_count, **kwargs): - affiliation = user.get_institution_affiliation(institution._id) - return cls.record( - user_id=user._id, - institution_id=institution._id, - department=getattr(affiliation, 'sso_department', DEFAULT_ES_NULL_VALUE), - public_project_count=public_project_count, - private_project_count=private_project_count, - **kwargs - ) - - @classmethod - def get_current_user_metrics(cls, institution) -> list: - """ - Gets the most recent document for every unique user. - :param institution: Institution - :return: list - """ - last_record_time = cls.get_recent_datetime(institution) - - search = cls.filter_institution( - institution - ).filter( - 'range', - timestamp={ - 'gte': last_record_time - } - ).sort( - 'user_id' - ) - search.update_from_dict({ - 'size': MAX_SIZE_OF_ES_QUERY - }) - - return search - - -class InstitutionProjectCounts(MetricMixin, metrics.Metric): - institution_id = metrics.Keyword(index=True, doc_values=True, required=True) - user_count = metrics.Integer(index=True, doc_values=True, required=True) - public_project_count = metrics.Integer(index=True, doc_values=True, required=True) - private_project_count = metrics.Integer(index=True, doc_values=True, required=True) - - class Index: - settings = { - 'number_of_shards': 1, - 'number_of_replicas': 1, - 'refresh_interval': '1s', - } - - class Meta: - source = metrics.MetaField(enabled=True) - - @classmethod - def record_institution_project_counts(cls, institution, public_project_count, private_project_count, **kwargs): - return cls.record( - institution_id=institution._id, - user_count=institution.get_institution_users().count(), - public_project_count=public_project_count, - private_project_count=private_project_count, - **kwargs - ) - - @classmethod - def get_latest_institution_project_document(cls, institution): - search = cls.search().filter('match', institution_id=institution._id).sort('-timestamp')[:1] - response = search.execute() - if response: - return response[0] diff --git a/osf_tests/test_management_commands.py b/osf_tests/test_management_commands.py index 26e34601648..80deee182d6 100644 --- a/osf_tests/test_management_commands.py +++ b/osf_tests/test_management_commands.py @@ -1,6 +1,5 @@ from unittest import mock import pytest -import time from collections import OrderedDict @@ -9,13 +8,10 @@ from addons.osfstorage import settings as osfstorage_settings from api_tests.utils import create_test_file from framework.auth import Auth -from osf.management.commands.update_institution_project_counts import update_institution_project_counts from osf.management.commands.project_to_draft_registration_contributor_sync import retrieve_draft_registrations_to_sync, project_to_draft_registration_contributor_sync from osf.models import RegistrationSchema -from osf.metrics import InstitutionProjectCounts, UserInstitutionProjectCounts from osf_tests.factories import ( AuthUserFactory, - InstitutionFactory, PreprintFactory, ProjectFactory, RegistrationFactory, @@ -265,109 +261,6 @@ def test_data_storage_usage_command(self): assert (key, expected_summary_data[key]) == (key, actual_summary_data[key]) -@pytest.mark.es_metrics -@pytest.mark.django_db -class TestInstitutionMetricsUpdate: - - @pytest.fixture() - def institution(self): - # Private: 14, Public: 4 - return InstitutionFactory() - - @pytest.fixture() - def user1(self, institution): - # Private: 4, Public: 4 (+1 from user2 fixture) - user = AuthUserFactory() - user.add_or_update_affiliated_institution(institution) - - for i in range(5): - project = ProjectFactory(creator=user, is_public=False) - project.affiliated_institutions.add(institution) - project.save() - - project.delete() - - for i in range(3): - project = ProjectFactory(creator=user, is_public=True) - project.affiliated_institutions.add(institution) - project.save() - - ProjectFactory(creator=user, is_public=True) - ProjectFactory(creator=user, is_public=False) - - return user - - @pytest.fixture() - def user2(self, institution, user1): - # Private: 10, Public: 1 - user = AuthUserFactory() - user.add_or_update_affiliated_institution(institution) - - for i in range(10): - project = ProjectFactory(creator=user, is_public=False) - project.affiliated_institutions.add(institution) - project.save() - for i in range(1): - project = ProjectFactory(creator=user, is_public=True) - project.add_contributor(user1) - project.affiliated_institutions.add(institution) - project.save() - - return user - - @pytest.fixture() - def user3(self, institution): - # Private: 0, Public: 0 - user = AuthUserFactory() - user.add_or_update_affiliated_institution(institution) - - return user - - @pytest.fixture() - def user4(self): - # Projects should not be included in results - user = AuthUserFactory() - - for i in range(3): - project = ProjectFactory(creator=user, is_public=False) - project.save() - for i in range(6): - project = ProjectFactory(creator=user, is_public=True) - project.save() - - return user - - def test_update_institution_counts(self, app, institution, user1, user2, user3, user4): - update_institution_project_counts() - - time.sleep(2) - - user_search = UserInstitutionProjectCounts.get_current_user_metrics(institution) - user_results = user_search.execute() - sorted_results = sorted(user_results, key=lambda x: x['private_project_count']) - - user3_record = sorted_results[0] - user1_record = sorted_results[1] - user2_record = sorted_results[2] - - assert user1_record['user_id'] == user1._id - assert user1_record['public_project_count'] == 4 - assert user1_record['private_project_count'] == 4 - - assert user2_record['user_id'] == user2._id - assert user2_record['public_project_count'] == 1 - assert user2_record['private_project_count'] == 10 - - assert user3_record['user_id'] == user3._id - assert user3_record['public_project_count'] == 0 - assert user3_record['private_project_count'] == 0 - - institution_results = InstitutionProjectCounts.get_latest_institution_project_document(institution) - - assert institution_results['public_project_count'] == 4 - assert institution_results['private_project_count'] == 14 - - @pytest.mark.django_db class TestProjectDraftRegContributorSync: @pytest.fixture() diff --git a/tests/identifiers/test_datacite.py b/tests/identifiers/test_datacite.py index 768a400fc59..e2e9560ede2 100644 --- a/tests/identifiers/test_datacite.py +++ b/tests/identifiers/test_datacite.py @@ -155,7 +155,7 @@ def test_datcite_format_contributors(self, datacite_client): assert f'{invisible_contrib.fullname}' not in metadata_xml def test_datacite_format_related_resources(self, datacite_client): - registration = RegistrationFactory(is_public=True, has_doi=True, article_doi='publication') + registration = RegistrationFactory(is_public=True, has_doi=True, article_doi='10.pub/lication') outcome = Outcome.objects.for_registration(registration, create=True) data_artifact = outcome.artifact_metadata.create( identifier=IdentifierFactory(category='doi'), artifact_type=ArtifactTypes.DATA, finalized=True @@ -179,7 +179,7 @@ def test_datacite_format_related_resources(self, datacite_client): 'relationType': 'References', }, { - 'relatedIdentifier': 'publication', + 'relatedIdentifier': '10.pub/lication', 'relatedIdentifierType': 'DOI', 'relationType': 'References', }, diff --git a/website/settings/defaults.py b/website/settings/defaults.py index 80cc6b18ed1..aaa8173acf9 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -445,7 +445,6 @@ class CeleryConfig: 'osf.management.commands.sync_doi_metadata', 'osf.management.commands.sync_collection_provider_indices', 'osf.management.commands.sync_datacite_doi_metadata', - 'osf.management.commands.update_institution_project_counts', 'osf.management.commands.populate_branched_from', 'osf.management.commands.spam_metrics', 'osf.management.commands.daily_reporters_go', @@ -575,7 +574,6 @@ class CeleryConfig: 'osf.management.commands.deactivate_requested_accounts', 'osf.management.commands.check_crossref_dois', 'osf.management.commands.find_spammy_files', - 'osf.management.commands.update_institution_project_counts', 'osf.management.commands.correct_registration_moderation_states', 'osf.management.commands.sync_collection_provider_indices', 'osf.management.commands.sync_datacite_doi_metadata', @@ -718,10 +716,6 @@ class CeleryConfig: 'task': 'management.commands.check_crossref_dois', 'schedule': crontab(minute=0, hour=4), # Daily 11:00 p.m. }, - 'update_institution_project_counts': { - 'task': 'management.commands.update_institution_project_counts', - 'schedule': crontab(minute=0, hour=9), # Daily 05:00 a.m. EDT - }, # 'archive_registrations_on_IA': { # 'task': 'osf.management.commands.archive_registrations_on_IA', # 'schedule': crontab(minute=0, hour=5), # Daily 4:00 a.m.