From eb46911fd9c4d732ca82b669ef0b9e9fb97dbc72 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Wed, 17 Feb 2021 16:59:13 -0500 Subject: [PATCH 01/14] update share.transform.chain to use share.schema --- share/transform/chain/parsers.py | 29 ++++++++++++++++++---------- share/transform/chain/transformer.py | 4 ++-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/share/transform/chain/parsers.py b/share/transform/chain/parsers.py index 24699abd5..48c66ef0b 100644 --- a/share/transform/chain/parsers.py +++ b/share/transform/chain/parsers.py @@ -3,9 +3,10 @@ import logging from functools import reduce -from django.apps import apps from django.core.exceptions import FieldDoesNotExist +from share.schema import ShareV2Schema +from share.schema.shapes import RelationShape from share.transform.chain.exceptions import ChainError from share.transform.chain.links import Context from share.transform.chain.links import AbstractLink @@ -63,7 +64,7 @@ def __init__(self, context, config=None): def validate(self, field, value): if field.is_relation: - if field.one_to_many or field.remote_field.many_to_many: + if field.relation_shape in (RelationShape.ONE_TO_MANY, RelationShape.MANY_TO_MANY): assert isinstance(value, (list, tuple)), 'Values for field {} must be lists. Found {}'.format(field, value) else: assert isinstance(value, dict) and '@id' in value and '@type' in value, 'Values for field {} must be a dictionary with keys @id and @type. Found {}'.format(field, value) @@ -86,16 +87,16 @@ def _do_parse(self): else: schema = self.schema - model = apps.get_model('share', schema) + schema_type = ShareV2Schema().get_type(schema) self.ref = {'@id': self.id, '@type': schema} inst = {**self.ref} # Shorthand for copying ref for key, chain in self.parsers.items(): try: - field = model._meta.get_field(key) + field = ShareV2Schema().get_field(schema_type.name, key) except FieldDoesNotExist: - raise Exception('Tried to parse value {} which does not exist on {}'.format(key, model)) + raise Exception('Tried to parse value {} which does not exist on {}'.format(key, schema_type)) try: value = chain.run(self.context) @@ -103,15 +104,23 @@ def _do_parse(self): e.push('{}.{}'.format(self.__class__.__name__, key)) raise e - if value and field.is_relation and (field.one_to_many or field.remote_field.many_to_many): - field_name = field.field.name if field.one_to_many else field.m2m_field_name() + if ( + value + and field.is_relation + and field.relation_shape in (RelationShape.ONE_TO_MANY, RelationShape.MANY_TO_MANY) + ): + if field.relation_shape == RelationShape.ONE_TO_MANY: + field_to_set = field.inverse_relation + else: + field_to_set = field.incoming_through_relation + for v in tuple(value): # Freeze list so we can modify it will iterating # Allow filling out either side of recursive relations - if model._meta.concrete_model == field.related_model and field_name in ctx.pool[v]: - ctx.pool[v][field.m2m_reverse_field_name()] = self.ref + if schema_type.concrete_type == field.related_concrete_type and field.name in ctx.pool[v]: + ctx.pool[v][field_to_set] = self.ref value.remove(v) # Prevent CyclicalDependency error. Only "subjects" should have related_works else: - ctx.pool[v][field_name] = self.ref + ctx.pool[v][field_to_set] = self.ref if value is not None: self.validate(field, value) diff --git a/share/transform/chain/transformer.py b/share/transform/chain/transformer.py index 0af49b04c..8acd66f1f 100644 --- a/share/transform/chain/transformer.py +++ b/share/transform/chain/transformer.py @@ -4,6 +4,7 @@ import json import xmltodict +from share.schema import ShareV2Schema from share.transform.base import BaseTransformer from share.transform.chain.links import Context @@ -30,8 +31,7 @@ class ChainTransformer(BaseTransformer): @property def allowed_roots(self): - from share.models import AbstractCreativeWork - return set(t.__name__ for t in AbstractCreativeWork.get_type_classes()) + return ShareV2Schema().get_type_names('abstractcreativework') def do_transform(self, data, **kwargs): # Parsed data will be loaded into ctx From c56ba5c38e67cbc30a9495ba3befd323c0d4eaf2 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Wed, 17 Feb 2021 12:31:08 -0500 Subject: [PATCH 02/14] delete ShareObject! and pull some threads --- api/schemas/urls.py | 8 +- api/schemas/views.py | 120 +--- api/shareobjects/generator.py | 63 -- api/shareobjects/serializers.py | 13 - api/shareobjects/urls.py | 8 - api/shareobjects/views.py | 24 - api/urls.py | 4 - project/settings.py | 20 +- setup.py | 3 +- share/admin/__init__.py | 72 +-- share/admin/share_objects.py | 194 ------ share/disambiguation/criteria.py | 91 --- share/disambiguation/matcher.py | 82 --- share/disambiguation/strategies/__init__.py | 4 - share/disambiguation/strategies/base.py | 45 -- share/disambiguation/strategies/database.py | 266 -------- share/disambiguation/strategies/graph.py | 84 --- share/graphql/__init__.py | 0 share/graphql/agent.py | 65 -- share/graphql/base.py | 79 --- share/graphql/elasticsearch.py | 47 -- share/graphql/fields.py | 9 - share/graphql/query.py | 53 -- share/graphql/relations.py | 59 -- share/graphql/schema.py | 7 - share/graphql/work.py | 96 --- share/management/commands/addsubjects.py | 35 -- share/management/commands/editsubjects.py | 77 --- .../management/commands/enforce_set_lists.py | 119 ---- share/management/commands/fix_datacite.py | 125 ---- share/management/commands/forceingest.py | 201 ------ .../commands/removeanonymousworks.py | 25 - share/models/__init__.py | 7 - share/models/agents.py | 50 -- share/models/agents.yaml | 27 - share/models/base.py | 265 -------- share/models/change.py | 249 -------- share/models/creative.py | 78 --- share/models/creative.yaml | 47 -- share/models/fields.py | 443 +------------- share/models/identifiers.py | 69 --- share/models/meta.py | 102 ---- share/models/relations/__init__.py | 3 - share/models/relations/agent.py | 23 - share/models/relations/agent.yaml | 12 - share/models/relations/agentwork.py | 80 --- share/models/relations/agentwork.yaml | 35 -- share/models/relations/creative.py | 23 - share/models/relations/creative.yaml | 139 ----- share/models/sql.py | 99 --- share/models/validators.py | 23 +- share/oaipmh/legacy/renderers.py | 212 ------- share/oaipmh/legacy/repository.py | 251 -------- share/oaipmh/views.py | 22 +- share/regulate/steps/deduplicate.py | 54 +- share/search/__init__.py | 63 -- share/search/fetchers.py | 573 ------------------ share/search/index_setup/__init__.py | 4 +- .../search/index_setup/postrend_backcompat.py | 5 +- share/search/index_setup/share_classic.py | 62 -- share/search/index_setup/trove_v0.py | 19 + share/shell_util.py | 40 +- share/tasks/jobs.py | 60 -- share/util/__init__.py | 46 -- tests/share/normalize/test_models.py | 31 +- .../share/regulate/steps/test_deduplicate.py | 6 +- 66 files changed, 141 insertions(+), 5179 deletions(-) delete mode 100644 api/shareobjects/generator.py delete mode 100644 api/shareobjects/serializers.py delete mode 100644 api/shareobjects/urls.py delete mode 100644 api/shareobjects/views.py delete mode 100644 share/admin/share_objects.py delete mode 100644 share/disambiguation/criteria.py delete mode 100644 share/disambiguation/matcher.py delete mode 100644 share/disambiguation/strategies/__init__.py delete mode 100644 share/disambiguation/strategies/base.py delete mode 100644 share/disambiguation/strategies/database.py delete mode 100644 share/disambiguation/strategies/graph.py delete mode 100644 share/graphql/__init__.py delete mode 100644 share/graphql/agent.py delete mode 100644 share/graphql/base.py delete mode 100644 share/graphql/elasticsearch.py delete mode 100644 share/graphql/fields.py delete mode 100644 share/graphql/query.py delete mode 100644 share/graphql/relations.py delete mode 100644 share/graphql/schema.py delete mode 100644 share/graphql/work.py delete mode 100644 share/management/commands/addsubjects.py delete mode 100644 share/management/commands/editsubjects.py delete mode 100644 share/management/commands/enforce_set_lists.py delete mode 100644 share/management/commands/fix_datacite.py delete mode 100644 share/management/commands/forceingest.py delete mode 100644 share/management/commands/removeanonymousworks.py delete mode 100644 share/models/agents.py delete mode 100644 share/models/agents.yaml delete mode 100644 share/models/base.py delete mode 100644 share/models/change.py delete mode 100644 share/models/creative.py delete mode 100644 share/models/creative.yaml delete mode 100644 share/models/identifiers.py delete mode 100644 share/models/meta.py delete mode 100644 share/models/relations/__init__.py delete mode 100644 share/models/relations/agent.py delete mode 100644 share/models/relations/agent.yaml delete mode 100644 share/models/relations/agentwork.py delete mode 100644 share/models/relations/agentwork.yaml delete mode 100644 share/models/relations/creative.py delete mode 100644 share/models/relations/creative.yaml delete mode 100644 share/oaipmh/legacy/renderers.py delete mode 100644 share/oaipmh/legacy/repository.py delete mode 100644 share/search/fetchers.py delete mode 100644 share/search/index_setup/share_classic.py create mode 100644 share/search/index_setup/trove_v0.py diff --git a/api/schemas/urls.py b/api/schemas/urls.py index 8e57a65cf..9fed8fda4 100644 --- a/api/schemas/urls.py +++ b/api/schemas/urls.py @@ -2,12 +2,6 @@ from api.schemas import views -schema_patterns = [ - url(r'^{}/?'.format(view.MODEL.__name__), view.as_view()) - for view in views.ModelSchemaView.model_views -] - urlpatterns = [ url(r'^$', views.SchemaView.as_view(), name='schema'), - url(r'^creativework/hierarchy/?$', views.ModelTypesView.as_view(), name='modeltypes'), -] + schema_patterns +] diff --git a/api/schemas/views.py b/api/schemas/views.py index 57051f55f..20db28d26 100644 --- a/api/schemas/views.py +++ b/api/schemas/views.py @@ -1,131 +1,13 @@ -import re -import os -import yaml - from rest_framework import views from rest_framework.response import Response -from share import models -from share.util import sort_dict_by_key from share.models.validators import JSONLDValidator -from api.deprecation import deprecate - -__all__ = ('SchemaView', 'ModelSchemaView', 'ModelTypesView') - -INDENT = 4 * ' ' -DEFAULT_DOC = r'{}\(.*\)' +__all__ = ('SchemaView',) class SchemaView(views.APIView): def get(self, request, *args, **kwargs): schema = JSONLDValidator.jsonld_schema.schema return Response(schema) - - -schema_models = set() - - -def format_link(model): - schema_models.add(model) - link = '- [{0}](/api/schema/{0})'.format(model.__name__) - if model.__doc__ and not re.fullmatch(DEFAULT_DOC.format(model.__name__), model.__doc__): - link += ': ' + next(line for line in model.__doc__.splitlines() if line) - return link - - -def subclass_links(base_model, include_base=True): - links = [format_link(base_model)] if include_base else [] - for model in sorted(base_model.__subclasses__(), key=lambda m: m.__name__): - subclasses = subclass_links(model) - if include_base: - subclasses = [INDENT + link for link in subclasses] - links.extend(subclasses) - return links - - -def section(*models): - return '\n'.join(format_link(m) for m in sorted(models, key=lambda m: m.__name__)) - - -def typed_model(base_model, include_base=True): - return '\n'.join(subclass_links(base_model, include_base)) - - -SchemaView.__doc__ = """ -Schema used to validate changes or additions to the SHARE dataset. - -To submit changes, see [`/api/normalizeddata`](/api/normalizeddata) - -Each node in the submitted `@graph` is validated by a model schema determined by its `@type`. - -## Object schemas - -### Work types -{works} - -### Agents -{agents} - -### Identifiers -{identifiers} - -### Other Objects -{others} - -## Relation schemas - -### Relations between Agents -{agent_relations} - -### Relations between Works -{work_relations} - -### Relations between Agents and Works -{agentwork_relations} - -### Other Relations -{other_relations} -""".format( - works=typed_model(models.CreativeWork), - agents=typed_model(models.Agent, include_base=False), - agent_relations=typed_model(models.AgentRelation, include_base=False), - work_relations=typed_model(models.WorkRelation, include_base=False), - agentwork_relations=typed_model(models.AgentWorkRelation, include_base=False), - identifiers=section(models.WorkIdentifier, models.AgentIdentifier), - others=section(models.Award, models.Subject, models.Tag), - other_relations=section(models.ThroughAwards, models.ThroughContributor, models.ThroughSubjects, models.ThroughTags), -) - - -@deprecate(pls_hide=False) -class ModelSchemaView(views.APIView): - """ - Schema used to validate submitted changes with `@type='{}'`. See [`/api/schema`](/api/schema) - - {} - """ - model_views = [] - - def get(self, request, *args, **kwargs): - schema = JSONLDValidator().validator_for(self.MODEL).schema - return Response(schema) - - -for model in schema_models: - name = '{}SchemaView'.format(model.__name__) - ModelSchemaView.model_views.append(type(name, (ModelSchemaView,), { - 'MODEL': model, - '__doc__': ModelSchemaView.__doc__.format(model.__name__, model.__doc__) - })) - - -class ModelTypesView(views.APIView): - - yaml_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../share/models/creative.yaml') - with open(yaml_file) as fobj: - model_specs = sort_dict_by_key(yaml.load(fobj)) - - def get(self, request, *args, **kwargs): - return Response(self.model_specs) diff --git a/api/shareobjects/generator.py b/api/shareobjects/generator.py deleted file mode 100644 index 441e654f1..000000000 --- a/api/shareobjects/generator.py +++ /dev/null @@ -1,63 +0,0 @@ -from typedmodels.models import TypedModel - -from rest_framework.utils.field_mapping import get_detail_view_name - -from share import models - -from api import fields -from api.shareobjects.serializers import ShareObjectSerializer -from api.shareobjects.views import ShareObjectViewSet - - -# TODO: Get rid of these generated endpoints, require using GraphQL endpoint for ShareObjects -class EndpointGenerator: - - def __init__(self, router): - self._router = router - subclasses = models.ShareObject.__subclasses__() - - generated_endpoints = [] - for subclass in subclasses: - if issubclass(subclass, TypedModel) and subclass._meta.concrete_model is subclass: - generated_endpoints.extend(subclass.get_type_classes()) - else: - generated_endpoints.append(subclass) - self.generate_endpoints(generated_endpoints) - - def generate_endpoints(self, subclasses): - for subclass in subclasses: - self.generate_serializer(subclass) - - def generate_serializer(self, subclass): - class_name = subclass.__name__ + 'Serializer' - meta_class = type('Meta', tuple(), {'model': subclass, 'fields': '__all__'}) - generated_serializer = type(class_name, (ShareObjectSerializer,), { - 'Meta': meta_class, - 'type': fields.TypeField(), - 'url': fields.ShareIdentityField(view_name='api:{}'.format(get_detail_view_name(subclass))) - }) - globals().update({class_name: generated_serializer}) - self.generate_viewset(subclass, generated_serializer) - - def generate_viewset(self, subclass, serializer): - class_name = subclass.__name__ + 'ViewSet' - # Pre-join all fields foreign keys - # Note: we can probably avoid this all together if we fix DRF - # We don't need to load the entire objects, just the PKs - queryset = serializer.Meta.model.objects.all().select_related(*( - field.name for field in serializer.Meta.model._meta.get_fields() - if field.is_relation and field.editable and not field.many_to_many - )) - if subclass.__name__ == 'AgentIdentifier': - queryset = queryset.exclude(scheme='mailto') - - generated_viewset = type(class_name, (ShareObjectViewSet,), { - 'queryset': queryset, - 'serializer_class': serializer, - }) - globals().update({class_name: generated_viewset}) - self.register_url(subclass, generated_viewset) - - def register_url(self, subclass, viewset): - route_name = subclass._meta.verbose_name_plural.replace(' ', '') - self._router.register(route_name, viewset, base_name=viewset.serializer_class.Meta.model._meta.model_name) diff --git a/api/shareobjects/serializers.py b/api/shareobjects/serializers.py deleted file mode 100644 index 7cd6d06cc..000000000 --- a/api/shareobjects/serializers.py +++ /dev/null @@ -1,13 +0,0 @@ -from api.base.serializers import ShareSerializer - - -class ShareObjectSerializer(ShareSerializer): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # Exclude `change` and `sources` fields, plus anything that points to a version table - excluded_fields = ['change', 'sources'] - for field_name in tuple(self.fields.keys()): - if 'version' in field_name or field_name in excluded_fields: - self.fields.pop(field_name) diff --git a/api/shareobjects/urls.py b/api/shareobjects/urls.py deleted file mode 100644 index 1c8527aa4..000000000 --- a/api/shareobjects/urls.py +++ /dev/null @@ -1,8 +0,0 @@ -from rest_framework.routers import SimpleRouter -from api.shareobjects.generator import EndpointGenerator - - -# generate share object routes -router = SimpleRouter() -EndpointGenerator(router) -urlpatterns = router.urls diff --git a/api/shareobjects/views.py b/api/shareobjects/views.py deleted file mode 100644 index b9202011c..000000000 --- a/api/shareobjects/views.py +++ /dev/null @@ -1,24 +0,0 @@ -from rest_framework import viewsets - -from api.base.views import ShareViewSet -from api.deprecation import deprecate -from api.pagination import CursorPagination -from api.permissions import IsDeletedPremissions - - -@deprecate(pls_hide=True) -class ShareObjectViewSet(ShareViewSet, viewsets.ReadOnlyModelViewSet): - pagination_class = CursorPagination - permission_classes = (IsDeletedPremissions, ) - - # Can't expose these until we have indexes added, both ascending and descending - # filter_backends = (filters.OrderingFilter,) - # ordering_fields = ('id', 'date_updated') - - # Override get_queryset to handle items marked as deleted. - def get_queryset(self, list=True): - queryset = super().get_queryset() - lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field - if lookup_url_kwarg not in self.kwargs and hasattr(queryset.model, 'is_deleted'): - return queryset.exclude(is_deleted=True) - return queryset diff --git a/api/urls.py b/api/urls.py index 5b668d285..52d266d9f 100644 --- a/api/urls.py +++ b/api/urls.py @@ -1,8 +1,6 @@ from django.conf.urls import include from django.conf.urls import url -from graphene_django.views import GraphQLView - from api import views from api.base.views import RootView @@ -16,7 +14,6 @@ url('^', include('api.ingestjobs.urls')), url('^', include('api.normalizeddata.urls')), url('^', include('api.rawdata.urls')), - url('^', include('api.shareobjects.urls')), url('^', include('api.sourceregistrations.urls')), url('^', include('api.sourceconfigs.urls')), url('^', include('api.sources.urls')), @@ -30,7 +27,6 @@ url(r'^status/?', views.ServerStatusView.as_view(), name='status'), url(r'^rss/?', views.LegacyCreativeWorksRSS(), name='rss'), url(r'^atom/?', views.LegacyCreativeWorksAtom(), name='atom'), - url(r'^graph/?', GraphQLView.as_view(graphiql=True)), url(r'^feeds/rss/?', views.MetadataRecordsRSS(), name='feeds.rss'), url(r'^feeds/atom/?', views.MetadataRecordsAtom(), name='feeds.atom'), diff --git a/project/settings.py b/project/settings.py index 2c2a2730f..e727342e1 100644 --- a/project/settings.py +++ b/project/settings.py @@ -311,16 +311,16 @@ def split(string, delim): 'ACTIVE_INDEXES': split(os.environ.get('ELASTICSEARCH_ACTIVE_INDEXES', 'share_postrend_backcompat'), ','), # NOTE: indexes here won't be created automatically -- run `sharectl search setup ` BEFORE the daemon starts 'INDEXES': { - 'share_v3': { - 'DEFAULT_QUEUE': 'es-triton-share', - 'URGENT_QUEUE': 'es-triton-share.urgent', - 'INDEX_SETUP': 'share_classic', - }, - 'share_customtax_1': { - 'DEFAULT_QUEUE': 'es-share', - 'URGENT_QUEUE': 'es-share.urgent', - 'INDEX_SETUP': 'share_classic', - }, + # 'share_v3': { + # 'DEFAULT_QUEUE': 'es-triton-share', + # 'URGENT_QUEUE': 'es-triton-share.urgent', + # 'INDEX_SETUP': 'share_classic', + # }, + # 'share_customtax_1': { + # 'DEFAULT_QUEUE': 'es-share', + # 'URGENT_QUEUE': 'es-share.urgent', + # 'INDEX_SETUP': 'share_classic', + # }, 'share_postrend_backcompat': { 'DEFAULT_QUEUE': 'es-share-postrend-backcompat', 'URGENT_QUEUE': 'es-share-postrend-backcompat.urgent', diff --git a/setup.py b/setup.py index 4242683ab..bbf5c14ac 100644 --- a/setup.py +++ b/setup.py @@ -110,9 +110,8 @@ 'oai_dc = share.metadata_formats.oai_dc:OaiDcFormatter', ], 'share.search.index_setup': [ - 'share_classic = share.search.index_setup:ShareClassicIndexSetup', 'postrend_backcompat = share.search.index_setup:PostRendBackcompatIndexSetup', - # 'trove_v0 = share.search.index_setup:TroveV0IndexSetup', + 'trove_v0 = share.search.index_setup:TroveV0IndexSetup', ], } ) diff --git a/share/admin/__init__.py b/share/admin/__init__.py index f61745bc0..abd2102b9 100644 --- a/share/admin/__init__.py +++ b/share/admin/__init__.py @@ -3,13 +3,12 @@ from django import forms from django.conf.urls import url from django.contrib import admin -from django.contrib.admin import SimpleListFilter from django.contrib.admin.widgets import AdminDateWidget from django.http import HttpResponseRedirect from django.template.response import TemplateResponse from django.urls import reverse from django.utils import timezone -from django.utils.html import format_html, format_html_join, mark_safe +from django.utils.html import format_html from oauth2_provider.models import AccessToken @@ -18,26 +17,20 @@ from share.admin.jobs import HarvestJobAdmin from share.admin.jobs import IngestJobAdmin from share.admin.readonly import ReadOnlyAdmin -from share.admin.share_objects import CreativeWorkAdmin, SubjectAdmin from share.admin.util import FuzzyPaginator, linked_fk, linked_many, SourceConfigFilter from share.harvest.scheduler import HarvestScheduler from share.ingest.scheduler import IngestScheduler from share.models.banner import SiteBanner from share.models.celery import CeleryTaskResult -from share.models.change import ChangeSet from share.models.core import NormalizedData, ShareUser -from share.models.creative import AbstractCreativeWork from share.models.fields import DateTimeAwareJSONField from share.models.ingest import RawDatum, Source, SourceConfig, Harvester, Transformer, SourceUniqueIdentifier from share.models.jobs import HarvestJob from share.models.jobs import IngestJob -from share.models.meta import Subject, SubjectTaxonomy from share.models.registration import ProviderRegistration from share.models.sources import SourceStat -admin.site.register(AbstractCreativeWork, CreativeWorkAdmin) -admin.site.register(Subject, SubjectAdmin) admin.site.register(CeleryTaskResult, CeleryTaskResultAdmin) @@ -63,38 +56,6 @@ class NormalizedDataAdmin(admin.ModelAdmin): } -class ChangeSetSubmittedByFilter(SimpleListFilter): - title = 'Source' - parameter_name = 'source_id' - - def lookups(self, request, model_admin): - return ShareUser.objects.filter(is_active=True).values_list('id', 'username') - - def queryset(self, request, queryset): - if self.value(): - return queryset.filter(normalized_data__source_id=self.value()) - return queryset - - -class ChangeSetAdmin(admin.ModelAdmin): - list_display = ('status_', 'count_changes', 'submitted_by', 'submitted_at') - actions = ['accept_changes'] - list_filter = ['status', ChangeSetSubmittedByFilter] - raw_id_fields = ('normalized_data',) - paginator = FuzzyPaginator - - def submitted_by(self, obj): - return obj.normalized_data.source - submitted_by.short_description = 'submitted by' - - def count_changes(self, obj): - return obj.changes.count() - count_changes.short_description = 'number of changes' - - def status_(self, obj): - return ChangeSet.STATUS[obj.status].title() - - @linked_fk('suid') class RawDatumAdmin(admin.ModelAdmin): show_full_result_count = False @@ -222,35 +183,6 @@ def access_token(self, obj): return None -class SubjectTaxonomyAdmin(admin.ModelAdmin): - readonly_fields = ('source', 'subject_links',) - fields = ('source', 'is_deleted', 'subject_links') - list_display = ('id', 'source_',) - list_select_related = ('source',) - - def source_(self, obj): - return obj.source.long_title - - def subject_links(self, obj): - def recursive_link_list(subjects): - if not subjects: - return '' - items = format_html_join( - '', '
  • {}{}
    • ', - ( - ( - reverse('admin:share_subject_change', args=(s.id,)), - s.name, - mark_safe(recursive_link_list(list(s.children.all()))) - ) for s in sorted(subjects, key=lambda s: s.name) - ) - ) - return format_html('
      {}
    ', mark_safe(items)) - roots = obj.subject_set.filter(parent__isnull=True).prefetch_related('children', 'children__children', 'children__children__children') - return recursive_link_list(list(roots)) - subject_links.short_description = 'Subjects' - - class SourceStatAdmin(admin.ModelAdmin): search_fields = ('config__label', 'config__source__long_title') list_display = ('label', 'date_created', 'base_urls_match', 'earliest_datestamps_match', 'response_elapsed_time', 'response_status_code', 'grade_') @@ -308,7 +240,6 @@ def get_search_results(self, request, queryset, search_term): admin.site.unregister(AccessToken) admin.site.register(AccessToken, AccessTokenAdmin) -admin.site.register(ChangeSet, ChangeSetAdmin) admin.site.register(HarvestJob, HarvestJobAdmin) admin.site.register(IngestJob, IngestJobAdmin) admin.site.register(NormalizedData, NormalizedDataAdmin) @@ -320,7 +251,6 @@ def get_search_results(self, request, queryset, search_term): admin.site.register(ShareUser) admin.site.register(Source, SourceAdmin) admin.site.register(SourceConfig, SourceConfigAdmin) -admin.site.register(SubjectTaxonomy, SubjectTaxonomyAdmin) admin.site.register(SourceStat, SourceStatAdmin) admin.site.register(SourceUniqueIdentifier, SourceUniqueIdentifierAdmin) admin.site.register(Transformer) diff --git a/share/admin/share_objects.py b/share/admin/share_objects.py deleted file mode 100644 index f08c62e7e..000000000 --- a/share/admin/share_objects.py +++ /dev/null @@ -1,194 +0,0 @@ -import operator -from functools import reduce - -from django.conf import settings -from django.db import models -from django.contrib import admin -from django.forms import ModelChoiceField -from django.urls import reverse -from django.utils.html import format_html, format_html_join, mark_safe - -from share.admin.util import FuzzyPaginator -from share.models import AbstractCreativeWork -from share.models import Source -from share.models import Subject -from share.util import IDObfuscator -from share.util import InvalidID - - -class SourcesInline(admin.TabularInline): - extra = 1 - verbose_name = 'Source' - verbose_name_plural = 'Sources' - model = AbstractCreativeWork.sources.through - - def __init__(self, parent_model, admin_site): - self.model = parent_model.sources.through - super().__init__(parent_model, admin_site) - - def formfield_for_dbfield(self, db_field, **kwargs): - ret = super().formfield_for_dbfield(db_field, **kwargs) - if db_field.name == 'shareuser': - ret.widget.can_add_related = False - ret.widget.can_change_related = False - ret.widget.can_delete_related = False - return ret - - -class TypedModelFilter(admin.SimpleListFilter): - title = 'Type' - parameter_name = 'type' - - def lookups(self, request, model_admin): - return sorted([ - (k, v._meta.verbose_name_plural.title()) - for k, v - in model_admin.model._typedmodels_registry.items() - ]) - - def queryset(self, request, queryset): - if self.value(): - return queryset.filter(type=self.value()) - return queryset - - -class SourcesFilter(admin.SimpleListFilter): - title = 'Source' - parameter_name = 'source' - - def lookups(self, request, model_admin): - return sorted([ - (source.user.username, source.long_title) - for source in Source.objects.select_related('user').all() - ], key=lambda x: x[1]) - - def queryset(self, request, queryset): - if self.value(): - # Note: If this ever can filter on multiple sources - # this will require a .distinct() - return queryset.filter(sources__username=self.value()) - return queryset - - -class ShareObjectAdmin(admin.ModelAdmin): - actions = None - exclude = ('extra', 'same_as', 'change', ) - inlines = (SourcesInline, ) - # Django forces order by pk desc which results in a seqscan if we add anything else - # ordering = ('-date_modified', ) - readonly_fields = ('encoded_id', 'date_created', 'date_modified', ) - show_full_result_count = False - - def get_search_results(self, request, queryset, search_term): - ret = super().get_search_results(request, queryset, search_term) - return ret - - def get_type(self, obj): - return obj._meta.verbose_name.title() - get_type.short_description = 'Type' - - def encoded_id(self, obj): - return IDObfuscator.encode(obj) - - def has_add_permission(self, *args, **kwargs): - return False - - def has_delete_permission(self, *args, **kwargs): - return False - - def save_model(self, request, obj, form, change): - if form.changed_data: - obj.administrative_change(**{ - key: form.cleaned_data[key] - for key in form.changed_data - }) - - def save_related(self, request, form, formsets, change): - super().save_related(request, form, formsets, change) - - if form.changed_data: - # If we've already been changed there's nothing to worry about - return - - # If a m 2 m has changed, force date_modified to update so this - # object get re-indexed - for fs in formsets: - if fs.has_changed(): - form.instance.administrative_change(allow_empty=True) - break - - -class CreativeWorkAdmin(ShareObjectAdmin): - list_filter = (TypedModelFilter, SourcesFilter, ) - list_display = ('encoded_id', 'id', 'get_type', 'title', 'date_modified', 'date_created', ) - search_fields = ('identifiers__uri', ) - show_change_link = False - paginator = FuzzyPaginator - - def get_search_results(self, request, queryset, search_term): - try: - return queryset.filter(id=IDObfuscator.decode_id(search_term)), False - except InvalidID: - pass - - # Overriden because there is no way to opt out of a case insensitive search - search_fields = self.get_search_fields(request) - use_distinct = bool(search_term) - if search_fields and search_term: - orm_lookups = ['{}__startswith'.format(search_field) for search_field in search_fields] - for bit in search_term.split(): - or_queries = [models.Q(**{orm_lookup: bit}) for orm_lookup in orm_lookups] - queryset = queryset.filter(reduce(operator.or_, or_queries)) - - return queryset, use_distinct - - -class SubjectChoiceField(ModelChoiceField): - def label_from_instance(self, obj): - return '{}: {}'.format(obj.taxonomy.source.long_title, obj.name) - - -class SubjectAdmin(ShareObjectAdmin): - search_fields = ('name',) - readonly_fields = ('taxonomy_link', 'children_links', 'lineage') - fields = ('lineage', 'children_links', 'name', 'parent', 'taxonomy_link', 'central_synonym', 'is_deleted', 'uri',) - list_display = ('name', 'taxonomy_link', 'central_synonym', 'is_deleted') - list_filter = ('taxonomy',) - list_select_related = ('taxonomy', 'taxonomy__source',) - - def lineage(self, obj): - return format_html_join( - ' > ', '{}', - ((reverse('admin:share_subject_change', args=(s.id,)), s.name) for s in obj.lineage()) - ) - - def taxonomy_link(self, obj): - taxonomy_url = reverse('admin:share_subjecttaxonomy_change', args=(obj.taxonomy_id,)) - return format_html('{}', taxonomy_url, obj.taxonomy.source.long_title) - taxonomy_link.short_description = 'Taxonomy' - - def children_links(self, obj): - items = format_html_join( - '', '
  • {}
    • ', - ((reverse('admin:share_subject_change', args=(child.id,)), child.name) for child in obj.children.order_by('name')) - ) - return format_html('
      {}
    ', mark_safe(items)) - children_links.short_description = 'Children' - - def get_queryset(self, request): - return super().get_queryset(request).select_related('taxonomy', 'taxonomy__source', 'parent', 'parent__parent',).prefetch_related('children') - - def formfield_for_foreignkey(self, db_field, request, **kwargs): - subject_queryset = None - subject_id = request.resolver_match.args[0] - if db_field.name == 'parent': - # Limit to subjects from the same taxonomy - subject_queryset = Subject.objects.filter(taxonomy__subject__id=subject_id) - elif db_field.name == 'central_synonym': - # Limit to subjects from the central taxonomy, or none if this subject is in the central taxonomy - subject_queryset = Subject.objects.filter(taxonomy__source__user__username=settings.APPLICATION_USERNAME).exclude(taxonomy__subject__id=subject_id) - - if subject_queryset is not None: - kwargs['queryset'] = subject_queryset.order_by('name').select_related('taxonomy', 'taxonomy__source') - return SubjectChoiceField(required=not db_field.blank, **kwargs) - return super().formfield_for_foreignkey(db_field, request, **kwargs) diff --git a/share/disambiguation/criteria.py b/share/disambiguation/criteria.py deleted file mode 100644 index 6225029ae..000000000 --- a/share/disambiguation/criteria.py +++ /dev/null @@ -1,91 +0,0 @@ -import abc - - -class MatchingCriterion(abc.ABC): - @abc.abstractmethod - def match(self, strategy, nodes, model): - raise NotImplementedError - - @abc.abstractmethod - def model_dependencies(self, model): - raise NotImplementedError - - -class MatchByAttrs(MatchingCriterion): - def __init__(self, *attr_names, allowed_models=None): - self.attr_names = attr_names - self.allowed_models = allowed_models - - def match(self, strategy, nodes, model): - if any(model._meta.get_field(f).is_relation for f in self.attr_names): - raise ValueError('No relations allowed in MatchByAttrs: {}'.format(self.attr_names)) - - return strategy.match_by_attrs(nodes, model, self.attr_names, self.allowed_models) - - def model_dependencies(self, model): - return [] - - -class MatchByManyToOne(MatchingCriterion): - def __init__(self, *relation_names, constrain_types=False): - self.relation_names = relation_names - self.constrain_types = constrain_types - - def match(self, strategy, nodes, model): - if not all(model._meta.get_field(f).many_to_one for f in self.relation_names): - raise ValueError('Only many-to-one relations allowed in MatchByManyToOne: {}'.format(self.relation_names)) - - if not self.constrain_types: - allowed_models = None - elif model is model._meta.concrete_model: - allowed_models = [model] - else: - subclasses = model.get_type_classes() - superclasses = [ - m for m in model.__mro__ - if issubclass(m, model._meta.concrete_model) and m._meta.proxy - ] - allowed_models = set(subclasses + superclasses) - - return strategy.match_by_many_to_one(nodes, model, self.relation_names, allowed_models) - - def model_dependencies(self, model): - return [ - model._meta.get_field(f).related_model - for f in self.relation_names - ] - - -class MatchByOneToMany(MatchingCriterion): - def __init__(self, relation_name): - self.relation_name = relation_name - - def match(self, strategy, nodes, model): - if not model._meta.get_field(self.relation_name).one_to_many: - raise ValueError('MatchByOneToMany requires a one-to-many relation: {}'.format(self.relation_name)) - - return strategy.match_by_one_to_many(nodes, model, self.relation_name) - - def model_dependencies(self, model): - return [ - model._meta.get_field(self.relation_name).related_model - ] - - -class MatchSubjects(MatchingCriterion): - def match(self, strategy, nodes, model): - return strategy.match_subjects(nodes) - - def model_dependencies(self, model): - return [] - - -class MatchAgentWorkRelations(MatchingCriterion): - def match(self, strategy, nodes, model): - return strategy.match_agent_work_relations(nodes) - - def model_dependencies(self, model): - return [ - model._meta.get_field(f).related_model - for f in ('agent', 'creative_work') - ] diff --git a/share/disambiguation/matcher.py b/share/disambiguation/matcher.py deleted file mode 100644 index 47dc088b6..000000000 --- a/share/disambiguation/matcher.py +++ /dev/null @@ -1,82 +0,0 @@ -from django.apps import apps - -from share.exceptions import MergeRequired -from share.util import TopologicalSorter, ensure_iterable - -from share.disambiguation.criteria import MatchingCriterion - - -class Matcher: - def __init__(self, strategy): - self.strategy = strategy - - def find_all_matches(self, graph): - self.strategy.initial_pass(graph) - - for model, nodes in self._group_nodes_by_model(graph): - for criterion in self._get_model_criteria(model): - criterion.match(self.strategy, nodes, model) - - # for now, enforce only one match - # TODO: remove this restriction - single_matches = {} - for node, matches in self.strategy._matches.items(): - if not matches: - continue - if len(matches) > 1: - raise MergeRequired('Multiple matches for node {}'.format(node.id), matches) - single_matches[node] = list(matches)[0] - - return single_matches - - def chunk_matches(self, graph): - self.strategy.initial_pass(graph) - yield self.strategy._matches - self.strategy.clear_matches() - - for model, nodes in self._group_nodes_by_model(graph): - for criteria in self._get_model_criteria(model): - criteria.match(self.strategy, nodes, model) - yield self.strategy._matches - self.strategy.clear_matches() - - def _group_nodes_by_model(self, graph): - nodes_by_model = {} - for node in graph: - model = apps.get_model('share', node.type) - nodes_by_model.setdefault(model, []).append(node) - - models = list(nodes_by_model.keys()) - - sorted_models = TopologicalSorter( - models, - lambda n: self._get_model_dependencies(n, models), - ).sorted() - - return ((model, nodes_by_model[model]) for model in sorted_models) - - def _get_model_criteria(self, model): - # TODO: cache? - criteria = getattr(model, 'matching_criteria', None) - if not criteria: - return None - criteria = ensure_iterable(criteria) - if not all(isinstance(c, MatchingCriterion) for c in criteria): - raise ValueError('{}.matching_criteria must be a MatchingCriterion instance'.format(model._meta.model_name)) - return criteria - - def _get_model_dependencies(self, model, all_models): - criteria = self._get_model_criteria(model) - if not criteria: - return [] - concrete_dependencies = set( - dep - for c in criteria - for dep in c.model_dependencies(model) - ) - - dependencies = [ - m for m in all_models - if m._meta.concrete_model in concrete_dependencies - ] - return dependencies diff --git a/share/disambiguation/strategies/__init__.py b/share/disambiguation/strategies/__init__.py deleted file mode 100644 index 3fb394670..000000000 --- a/share/disambiguation/strategies/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .database import DatabaseStrategy -from .graph import GraphStrategy - -__all__ = ('DatabaseStrategy', 'GraphStrategy') diff --git a/share/disambiguation/strategies/base.py b/share/disambiguation/strategies/base.py deleted file mode 100644 index fdcef0db3..000000000 --- a/share/disambiguation/strategies/base.py +++ /dev/null @@ -1,45 +0,0 @@ -import abc - - -class MatchingStrategy(abc.ABC): - def __init__(self): - self._matches = {} - - def get_matches(self, node): - return self._matches.get(node, frozenset()) - - def add_match(self, node, match): - self._matches.setdefault(node, set()).add(match) - - def add_matches(self, node, matches): - self._matches.setdefault(node, set()).update(matches) - - def has_matches(self, node): - return bool(self._matches.get(node)) - - def clear_matches(self): - self._matches = {} - - @abc.abstractmethod - def initial_pass(self, nodes): - raise NotImplementedError - - @abc.abstractmethod - def match_by_attrs(self, nodes, model, attr_names, allowed_models): - raise NotImplementedError - - @abc.abstractmethod - def match_by_many_to_one(self, nodes, model, relation_names, allowed_models): - raise NotImplementedError - - @abc.abstractmethod - def match_by_one_to_many(self, nodes, model, relation_name): - raise NotImplementedError - - @abc.abstractmethod - def match_subjects(self, nodes): - raise NotImplementedError - - @abc.abstractmethod - def match_agent_work_relations(self, nodes): - raise NotImplementedError diff --git a/share/disambiguation/strategies/database.py b/share/disambiguation/strategies/database.py deleted file mode 100644 index a3499b8d2..000000000 --- a/share/disambiguation/strategies/database.py +++ /dev/null @@ -1,266 +0,0 @@ -import logging -from collections import namedtuple -from operator import attrgetter - -from django.conf import settings - -from share import exceptions -from share import models -from share.util import IDObfuscator, InvalidID -from share.util.nameparser import HumanName - - -from .base import MatchingStrategy - -logger = logging.getLogger(__name__) - - -class DatabaseStrategy(MatchingStrategy): - MAX_NAME_LENGTH = 200 - - def __init__(self, source=None, **kwargs): - super().__init__(**kwargs) - self.source = source - - def initial_pass(self, nodes): - for node in nodes: - if str(node.id).startswith('_:'): - continue - try: - match = IDObfuscator.resolve(node.id) - self.add_match(node, match) - except InvalidID: - pass - - def match_by_attrs(self, nodes, model, attr_names, allowed_models): - self._match_query( - nodes, - model, - column_names=[model._meta.get_field(a).column for a in attr_names], - get_values=lambda node: [node[a] for a in attr_names], - allowed_models=allowed_models, - ) - - def match_by_many_to_one(self, nodes, model, relation_names, allowed_models): - node_values = {} - for node in nodes: - related_matches = {r: list(self.get_matches(node[r])) for r in relation_names} - - if all(len(matches) == 1 for matches in related_matches.values()): - node_values[node] = [related_matches[r][0].id for r in relation_names] - else: - for relation_name, matches in related_matches.items(): - if len(matches) > 1: - raise exceptions.MergeRequired( - 'Multiple matches for node {}'.format(node[relation_name].id), - matches, - ) - - self._match_query( - node_values.keys(), - model, - column_names=[model._meta.get_field(r).column for r in relation_names], - get_values=lambda node: node_values[node], - allowed_models=allowed_models, - ) - - def match_by_one_to_many(self, nodes, model, relation_name): - remote_fk_attr = model._meta.get_field(relation_name).remote_field.attname - - for node in nodes: - match_ids = set( - getattr(instance, remote_fk_attr) - for related_node in node[relation_name] - for instance in self.get_matches(related_node) - ) - if match_ids: - self.add_matches(node, model._meta.concrete_model.objects.filter(id__in=match_ids)) - - def match_subjects(self, nodes): - # Look for (taxonomy AND uri) first, then (taxonomy AND name) - for node in nodes: - if node['central_synonym'] is None: - # Central taxonomy - qs = models.Subject.objects.filter(central_synonym__isnull=True) - elif self.source: - # Custom taxonomy - qs = models.Subject.objects.filter(taxonomy__source=self.source) - else: - continue - - for field_name in ('uri', 'name'): - value = node[field_name] - if value: - try: - match = qs.get(**{field_name: value}) - except models.Subject.DoesNotExist: - continue - self.add_match(node, match) - break - - def match_agent_work_relations(self, nodes): - work_nodes = set(n['creative_work'] for n in nodes) - for work_node in work_nodes: - for work in self.get_matches(work_node): - agent_relations = models.AbstractAgentWorkRelation.objects.filter( - creative_work=work, - ) - - # Skip parsing all the names on Frankenwork's monster - # TODO: work on defrankenization - if agent_relations.count() > settings.SHARE_LIMITS['MAX_AGENT_RELATIONS']: - continue - - relation_nodes = [ - n for n in work_node['agent_relations'] - if not self.has_matches(n) - ] - if not relation_nodes: - continue - - relation_names = [ - ParsedRelationNames(r, HumanName(r.cited_as), HumanName(r.agent.name)) - for r in agent_relations.select_related('agent') - if len(r.cited_as) <= self.MAX_NAME_LENGTH and len(r.agent.name) <= self.MAX_NAME_LENGTH - ] - - for node in relation_nodes: - if len(node['cited_as']) > self.MAX_NAME_LENGTH or len(node['agent']['name']) > self.MAX_NAME_LENGTH: - continue - - node_names = ParsedRelationNames(node, HumanName(node['cited_as']), HumanName(node['agent']['name'])) - - top_matches = sorted( - filter( - attrgetter('valid_match'), - (ComparableAgentWorkRelation(node_names, r) for r in relation_names), - ), - key=attrgetter('sort_key'), - reverse=True, - ) - if top_matches: - match = top_matches[0] - self.add_match(node['agent'], match.relation.agent) - self.add_match(node, match.relation) - - def _match_query(self, nodes, model, column_names, get_values, allowed_models): - if not nodes: - return - - if allowed_models: - query_builder = ConstrainedTypeQueryBuilder(model._meta.db_table, column_names, get_values, allowed_models) - else: - query_builder = QueryBuilder(model._meta.db_table, column_names, get_values) - - node_map = {n.id: n for n in nodes} - sql, values = query_builder.build(nodes) - matches = model.objects.raw(sql, values) - for match in matches: - self.add_match(node_map[match.node_id], match) - - -ParsedRelationNames = namedtuple('ParsedRelationNames', ['obj', 'cited_as', 'agent_name']) - - -class ComparableAgentWorkRelation: - def __init__(self, node_names, instance_names): - self.relation = instance_names.obj - self.node = node_names.obj - - # bit vector used to sort names by how close they are to the target name - self._name_key = self._get_name_key( - instance_names.cited_as, - node_names.cited_as, - ) - - self.sort_key = ( - *self._name_key, - *self._get_name_key( - instance_names.agent_name, - node_names.agent_name, - ), - self.relation.order_cited == self.node['order_cited'], - self.relation._meta.model_name == self.node.type, - ) - - @property - def valid_match(self): - return any(c for c in self._name_key) - - def _get_name_key(self, parsed_name, parsed_target): - # initial or None - def i(name_part): - return name_part[0] if name_part else None - - return ( - parsed_name.full_name == parsed_target.full_name, - (parsed_name.first, parsed_name.last) == (parsed_target.first, parsed_target.last), - (i(parsed_name.first), parsed_name.last) == (i(parsed_target.first), parsed_target.last), - ) - - -class QueryBuilder: - QUERY_TEMPLATE = ''' - WITH nodes(node_id, {column_names}) AS ( - VALUES {value_placeholders} - ) - SELECT nodes.node_id, {table_name}.* - FROM nodes - INNER JOIN {table_name} ON ({join_conditions}) - ''' - - def __init__(self, table_name, column_names, get_values): - self.table_name = table_name - self.column_names = column_names - self.get_values = get_values - - def build(self, nodes): - sql = self.QUERY_TEMPLATE.format( - table_name=self.table_name, - column_names=', '.join(self.column_names), - join_conditions=' AND '.join(self.join_conditions()), - value_placeholders=', '.join('%s' for _ in nodes), - ) - return (sql, self.params(nodes)) - - def params(self, nodes): - return [ - (n.id, *self.get_values(n)) - for n in nodes - ] - - def join_conditions(self): - return [ - 'nodes.{column} = {table}.{column}'.format( - column=column_name, - table=self.table_name, - ) - for column_name in self.column_names - ] - - -class ConstrainedTypeQueryBuilder(QueryBuilder): - TYPE_COLUMN = 'type' - - def __init__(self, table_name, column_names, get_values, allowed_models): - super().__init__(table_name, column_names, get_values) - self.allowed_models = allowed_models - - def params(self, nodes): - return [ - *super().params(nodes), - tuple( - '{}.{}'.format(m._meta.app_label, m._meta.model_name) - for m in self.allowed_models - ), - ] - - def join_conditions(self): - return [ - *super().join_conditions(), - - '{table}.{column} IN %s'.format( - column=self.TYPE_COLUMN, - table=self.table_name, - ) - ] diff --git a/share/disambiguation/strategies/graph.py b/share/disambiguation/strategies/graph.py deleted file mode 100644 index 8485a99ad..000000000 --- a/share/disambiguation/strategies/graph.py +++ /dev/null @@ -1,84 +0,0 @@ -from .base import MatchingStrategy - -from share import models - - -def equal_not_none(lhs, rhs): - return None not in (lhs, rhs) and lhs == rhs - - -class IndexByAttrs: - def __init__(self, attr_names): - self.attr_names = attr_names - self._index = {} - - def node_key(self, node): - return tuple(node[a] for a in self.attr_names) - - def add_nodes(self, nodes): - for node in nodes: - key = self.node_key(node) - if None not in key: - self._index.setdefault(key, set()).add(node) - - def get_matches(self, node): - key = self.node_key(node) - matches = self._index.get(key, set()) - return matches.difference([node]) - - -class GraphStrategy(MatchingStrategy): - def __init__(self, graph, **kwargs): - super().__init__(**kwargs) - self.graph = graph - - def initial_pass(self, nodes): - pass - - def match_by_attrs(self, nodes, model, attr_names, allowed_models): - graph_nodes = self._graph_nodes(model, allowed_models) - - attr_index = IndexByAttrs(attr_names) - attr_index.add_nodes(graph_nodes) - - for node in nodes: - matches = attr_index.get_matches(node) - if matches: - self.add_matches(node, matches) - - def match_by_many_to_one(self, nodes, model, relation_names, allowed_models): - self.match_by_attrs(nodes, model, relation_names, allowed_models) - - def match_by_one_to_many(self, nodes, model, relation_name): - # a one-to-many can't be two-to-many - pass - - def match_subjects(self, nodes): - graph_nodes = self._graph_nodes(models.Subject) - - for node in nodes: - matches = [ - n for n in graph_nodes - if n != node - and n['parent'] == node['parent'] - and n['central_synonym'] == node['central_synonym'] - and (equal_not_none(n['uri'], node['uri']) or equal_not_none(n['name'], node['name'])) - ] - self.add_matches(node, matches) - - def match_agent_work_relations(self, nodes): - # no special case when looking within a graph - pass - - def _graph_nodes(self, model, allowed_models=None): - if allowed_models is None: - return self.graph.filter_by_concrete_type(model._meta.concrete_model._meta.model_name) - else: - allowed_model_names = { - allowed_model._meta.model_name.lower() - for allowed_model in allowed_models - } - return filter( - lambda n: n.type in allowed_model_names, - self.graph, - ) diff --git a/share/graphql/__init__.py b/share/graphql/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/share/graphql/agent.py b/share/graphql/agent.py deleted file mode 100644 index 7d18d13b7..000000000 --- a/share/graphql/agent.py +++ /dev/null @@ -1,65 +0,0 @@ -import graphene - -from graphene_django import DjangoObjectType - -from share import models -from share.graphql.base import Identifier -from share.graphql.base import AbstractShareObject -from share.graphql.relations import AbstractAgentRelation -from share.graphql.relations import AbstractAgentWorkRelation -from share.util import IDObfuscator - - -class AbstractAgent(AbstractShareObject): - name = graphene.String() - identifiers = graphene.List(Identifier) - - total_related_works = graphene.Int() - related_works = graphene.List(AbstractAgentWorkRelation, limit=graphene.Int(), offset=graphene.Int()) - - total_incoming_agent_relations = graphene.Int() - incoming_agent_relations = graphene.List(AbstractAgentRelation, limit=graphene.Int(), offset=graphene.Int()) - - total_outgoing_agent_relations = graphene.Int() - outgoing_agent_relations = graphene.List(AbstractAgentRelation, limit=graphene.Int(), offset=graphene.Int()) - - @graphene.resolve_only_args - def resolve_identifiers(self): - return self.identifiers.exclude(scheme='mailto') - - @graphene.resolve_only_args - def resolve_total_related_works(self): - return self.work_relations.count() - - @graphene.resolve_only_args - def resolve_related_works(self, offset=None, limit=10): - limit = (offset or 0) + limit - return self.work_relations.all()[offset:limit] - - @graphene.resolve_only_args - def resolve_total_incoming_agent_relations(self): - return self.incoming_agent_relations.count() - - @graphene.resolve_only_args - def resolve_incoming_agent_relations(self, limit=None, offset=None): - if limit: - offset = (offset or 0) + limit - return self.incoming_agent_relations.all()[offset:limit] - - @graphene.resolve_only_args - def resolve_total_outgoing_agent_relations(self): - return self.outgoing_agent_relations.count() - - @graphene.resolve_only_args - def resolve_outgoing_agent_relations(self, limit=None, offset=None): - if limit: - offset = (offset or 0) + limit - return self.outgoing_agent_relations.all()[offset:limit] - - -for klass in models.Agent.get_type_classes(): - locals()[klass.__name__] = type(klass.__name__, (DjangoObjectType, ), { - 'id': graphene.String(), - 'resolve_id': graphene.resolve_only_args(lambda self: IDObfuscator.encode(self)), - 'Meta': type('Meta', (), {'model': klass, 'interfaces': (AbstractShareObject, AbstractAgent, )}) - }) diff --git a/share/graphql/base.py b/share/graphql/base.py deleted file mode 100644 index 187c15290..000000000 --- a/share/graphql/base.py +++ /dev/null @@ -1,79 +0,0 @@ -import graphene - -from graphene_django import DjangoObjectType - -from share import models -from share.graphql.fields import JSONField - - -class Source(DjangoObjectType): - title = graphene.String() - icon = graphene.String() - - class Meta: - model = models.Source - only_fields = ('id', 'home_page', ) - - @classmethod - def resolve_id(cls, instance, context, request, info): - return instance.name - - @classmethod - def resolve_title(cls, instance, context, request, info): - return instance.long_title - - @classmethod - def resolve_icon(cls, instance, context, request, info): - return instance.icon.url if instance.icon else None - - -class AbstractShareObject(graphene.Interface): - _implementers = set() - - id = graphene.String() - types = graphene.List(graphene.String) - extra = JSONField() - sources = graphene.List(Source, limit=graphene.Int(), offset=graphene.Int()) - - class Meta: - name = 'ShareObject' - - @classmethod - def implements(cls, type): - cls._implementers.add(type) - - @classmethod - def resolve_type(cls, instance, context, info): - return info.schema.get_type(type(instance).__name__) - - @graphene.resolve_only_args - def resolve_types(self): - types = [] - for parent in self.__class__.mro(): - if not parent._meta.proxy: - break - types.append(parent._meta.verbose_name.title()) - return types - - @graphene.resolve_only_args - def resolve_extra(self): - if self.extra_id: - return self.extra.data - return {} - - @graphene.resolve_only_args - def resolve_sources(self, limit=None, offset=None): - if limit: - offset = (offset or 0) + limit - return [user.source for user in self.sources.select_related('source').exclude(source__icon='').exclude(source__is_deleted=True)[offset:limit]] - - -class User(DjangoObjectType): - class Meta: - model = models.ShareUser - - -class Identifier(graphene.ObjectType): - uri = graphene.String() - host = graphene.String() - scheme = graphene.String() diff --git a/share/graphql/elasticsearch.py b/share/graphql/elasticsearch.py deleted file mode 100644 index 02671bbb0..000000000 --- a/share/graphql/elasticsearch.py +++ /dev/null @@ -1,47 +0,0 @@ -import graphene - -from share.graphql.fields import JSONField - - -class ElasticSearchQueryString(graphene.InputObjectType): - query = graphene.String(required=True) - - -class ElasticSearchTermsAggregation(graphene.InputObjectType): - field = graphene.String(required=True) - size = graphene.Int(required=True) - - -class ElasticSearchAggregation(graphene.InputObjectType): - name = graphene.String(required=True) - terms = ElasticSearchTermsAggregation() - - -class ElasticSearchQuery(graphene.InputObjectType): - query_string = ElasticSearchQueryString() - - -class ElasticSearchHit(graphene.ObjectType): - _id = graphene.String(name='_id') - _type = graphene.String(name='_type') - _score = graphene.String(name='_score') - _index = graphene.String(name='_index') - _source = JSONField(name='_source') - - -class ElasticSearchHits(graphene.ObjectType): - total = graphene.Int(description='The total number of documents matching the search criteria.') - max_score = graphene.Int(name='max_score', description='') - hits = graphene.List(ElasticSearchHit, description='An array of the actual results.') - - def resolve_hits(self, args, context, info): - return (ElasticSearchHit(**hit) for hit in self.hits) - - -class ElasticSearchResult(graphene.ObjectType): - hits = graphene.Field(ElasticSearchHits, description='The search results.') - took = graphene.Int(description='The time, in milliseconds, for Elasticsearch to execute the search.') - timed_out = graphene.Boolean(name='timed_out', description='Indicated whether this search timed out.') - - def resolve_hits(self, args, context, info): - return ElasticSearchHits(**self.hits) diff --git a/share/graphql/fields.py b/share/graphql/fields.py deleted file mode 100644 index 940450935..000000000 --- a/share/graphql/fields.py +++ /dev/null @@ -1,9 +0,0 @@ -from graphene.types.scalars import Scalar - - -# Note should define a couple parse methods but this class is only used for serializing -class JSONField(Scalar): - - @staticmethod - def serialize(val): - return val diff --git a/share/graphql/query.py b/share/graphql/query.py deleted file mode 100644 index 3ea156b58..000000000 --- a/share/graphql/query.py +++ /dev/null @@ -1,53 +0,0 @@ -import graphene - -from django.conf import settings - -from elasticsearch import Elasticsearch - -from share import models -from share.graphql.agent import AbstractAgent -from share.graphql.base import Source -from share.graphql.base import User -from share.graphql.base import AbstractShareObject -from share.graphql.elasticsearch import ElasticSearchAggregation -from share.graphql.elasticsearch import ElasticSearchQuery -from share.graphql.elasticsearch import ElasticSearchResult -from share.graphql.work import AbstractCreativeWork -from share.util import IDObfuscator - - -class Query(graphene.ObjectType): - - search = graphene.Field(ElasticSearchResult, args={ - 'aggregations': graphene.List(ElasticSearchAggregation), - 'from': graphene.Int(description='Retrieve hits from a certain offset. Defaults to 0.'), - 'query': ElasticSearchQuery(), - 'size': graphene.Int(description='The number of hits to return. Defaults to 10.'), - 'type': graphene.String(required=True), # TODO Make into an enum - }, description='Query the SHARE data set via Elasticsearch') - - me = graphene.Field(User) - sources = graphene.List(Source, limit=graphene.Int(), offset=graphene.Int()) - - client = Elasticsearch(settings.ELASTICSEARCH['URL'], retry_on_timeout=True, timeout=30) - - agent = graphene.Field(AbstractAgent, id=graphene.String(), resolver=IDObfuscator.resolver) - creative_work = graphene.Field(AbstractCreativeWork, id=graphene.String(), resolver=IDObfuscator.resolver) - share_object = graphene.Field(AbstractShareObject, id=graphene.String(), resolver=IDObfuscator.resolver) - - def resolve_me(self, args, context, info): - return context.user - - @graphene.resolve_only_args - def resolve_sources(self, limit=25, offset=0): - return models.Source.objects.exclude(icon='').order_by('long_title')[offset:offset + limit] - - def resolve_search(self, args, context, info): - args.setdefault('from', 0) - args.setdefault('size', 10) - - resp = Query.client.search(index=settings.ELASTICSEARCH['PRIMARY_INDEX'], doc_type=args.pop('type'), body=args) - - del resp['_shards'] # No need to expose server information - - return ElasticSearchResult(**resp) diff --git a/share/graphql/relations.py b/share/graphql/relations.py deleted file mode 100644 index 6653168ec..000000000 --- a/share/graphql/relations.py +++ /dev/null @@ -1,59 +0,0 @@ -import graphene - -from graphene_django import DjangoObjectType - -from share import models -from share.graphql.base import AbstractShareObject - - -def Agent(): - from share.graphql.agent import AbstractAgent - return AbstractAgent - - -def CreativeWork(): - from share.graphql.work import AbstractCreativeWork - return AbstractCreativeWork - - -class AbstractWorkRelation(AbstractShareObject): - subject = graphene.Field(CreativeWork) - related = graphene.Field(CreativeWork) - - @graphene.resolve_only_args - def resolve_related(self): - return self.related - - @graphene.resolve_only_args - def resolve_subject(self): - return self.subject - - -class AbstractAgentRelation(AbstractShareObject): - subject = graphene.Field(Agent) - related = graphene.Field(Agent) - - @graphene.resolve_only_args - def resolve_related(self): - return self.related - - @graphene.resolve_only_args - def resolve_subject(self): - return self.subject - - -class AbstractAgentWorkRelation(AbstractShareObject): - cited_as = graphene.String() - agent = graphene.Field(Agent) - creative_work = graphene.Field(CreativeWork) - - @graphene.resolve_only_args - def resolve_agent(self): - return self.agent - - -for base, interface in ((models.AgentRelation, AbstractAgentRelation), (models.WorkRelation, AbstractWorkRelation), (models.AgentWorkRelation, AbstractAgentWorkRelation)): - for klass in base.get_type_classes(): - locals()[klass.__name__] = type(klass.__name__, (DjangoObjectType, ), { - 'Meta': type('Meta', (), {'model': klass, 'interfaces': (interface, )}) - }) diff --git a/share/graphql/schema.py b/share/graphql/schema.py deleted file mode 100644 index 9e09288f8..000000000 --- a/share/graphql/schema.py +++ /dev/null @@ -1,7 +0,0 @@ -import graphene - -from share.graphql.query import Query -from share.graphql.base import AbstractShareObject - - -schema = graphene.Schema(query=Query, types=list(AbstractShareObject._implementers)) diff --git a/share/graphql/work.py b/share/graphql/work.py deleted file mode 100644 index 498985c8e..000000000 --- a/share/graphql/work.py +++ /dev/null @@ -1,96 +0,0 @@ -import graphene -import bleach - -from graphene_django import DjangoObjectType - -from project.settings import ALLOWED_TAGS - -from share import models -from share.graphql.base import Identifier -from share.graphql.base import AbstractShareObject -from share.graphql.relations import AbstractAgentWorkRelation, AbstractWorkRelation -from share.util import IDObfuscator - - -class Tag(DjangoObjectType): - tagged_works = graphene.List(lambda: AbstractCreativeWork, limit=graphene.Int(), offset=graphene.Int()) - - class Meta: - model = models.Tag - - @graphene.resolve_only_args - def resolve_tagged_works(self, limit=50, offset=0): - return models.AbstractCreativeWork.objects.filter(tags__name=self.name)[offset:offset + limit] - - -class AbstractCreativeWork(AbstractShareObject): - title = graphene.String() - description = graphene.String() - - # raw_data = graphene.List(RawData) - identifiers = graphene.List(Identifier) - tags = graphene.List(Tag, limit=graphene.Int(), offset=graphene.Int()) - - total_related_agents = graphene.Int() - related_agents = graphene.List(AbstractAgentWorkRelation, limit=graphene.Int(), offset=graphene.Int()) - - total_incoming_work_relations = graphene.Int() - incoming_work_relations = graphene.List(AbstractWorkRelation, limit=graphene.Int(), offset=graphene.Int()) - - total_outgoing_work_relations = graphene.Int() - outgoing_work_relations = graphene.List(AbstractWorkRelation, limit=graphene.Int(), offset=graphene.Int()) - - @graphene.resolve_only_args - def resolve_title(self): - return bleach.clean(self.title, strip=True, tags=ALLOWED_TAGS) - - @graphene.resolve_only_args - def resolve_description(self): - return bleach.clean(self.description, strip=True, tags=ALLOWED_TAGS) - - @graphene.resolve_only_args - def resolve_identifiers(self): - return self.identifiers.all() - - @graphene.resolve_only_args - def resolve_tags(self, limit=None, offset=None): - if limit: - offset = (offset or 0) + limit - return self.tags.all()[offset:limit] - - @graphene.resolve_only_args - def resolve_total_related_agents(self): - return self.agent_relations.count() - - @graphene.resolve_only_args - def resolve_related_agents(self, limit=None, offset=None): - if limit: - offset = (offset or 0) + limit - return self.agent_relations.order_by('order_cited')[offset:limit] - - @graphene.resolve_only_args - def resolve_total_incoming_work_relations(self): - return self.incoming_creative_work_relations.count() - - @graphene.resolve_only_args - def resolve_incoming_work_relations(self, limit=None, offset=None): - if limit: - offset = (offset or 0) + limit - return self.incoming_creative_work_relations.all()[offset:limit] - - @graphene.resolve_only_args - def resolve_total_outgoing_work_relations(self): - return self.outgoing_creative_work_relations.count() - - @graphene.resolve_only_args - def resolve_outgoing_work_relations(self, limit=None, offset=None): - if limit: - offset = (offset or 0) + limit - return self.outgoing_creative_work_relations.all()[offset:limit] - - -for klass in models.CreativeWork.get_type_classes(): - locals()[klass.__name__] = type(klass.__name__, (DjangoObjectType, ), { - 'resolve_id': graphene.resolve_only_args(lambda self: IDObfuscator.encode(self)), - 'Meta': type('Meta', (), {'model': klass, 'interfaces': (AbstractShareObject, AbstractCreativeWork, )}) - }) diff --git a/share/management/commands/addsubjects.py b/share/management/commands/addsubjects.py deleted file mode 100644 index a44085f40..000000000 --- a/share/management/commands/addsubjects.py +++ /dev/null @@ -1,35 +0,0 @@ -import yaml - -from django.core.management.base import BaseCommand -from django.conf import settings -from django.db import transaction - -from share.models import ShareUser, SubjectTaxonomy -from share.ingest.ingester import Ingester - - -class Command(BaseCommand): - - def handle(self, *args, **options): - with open(settings.SUBJECTS_YAML) as fobj: - subjects = yaml.load(fobj) - - self.save_subjects(subjects) - - @transaction.atomic - def save_subjects(self, subjects): - # Ensure central taxonomy exists - user = ShareUser.objects.get(username=settings.APPLICATION_USERNAME) - SubjectTaxonomy.objects.get_or_create(source=user.source) - - subjects = [ - { - '@id': '_:{}'.format(s['uri']), - '@type': 'subject', - 'name': s['name'], - 'uri': s['uri'], - 'parent': None if s['parent'] is None else {'@id': '_:{}'.format(s['parent']), '@type': 'subject'} - } for s in subjects - ] - - Ingester(subjects, 'share/management/commands/addsubjects.py').as_user(user).ingest() diff --git a/share/management/commands/editsubjects.py b/share/management/commands/editsubjects.py deleted file mode 100644 index 25fd4427b..000000000 --- a/share/management/commands/editsubjects.py +++ /dev/null @@ -1,77 +0,0 @@ -import argparse -import uuid -import yaml - -from django.core.management.base import BaseCommand -from django.conf import settings - - -# input file syntax: (TODO: if this command gets more complex, use JSON or YAML) -# - new subjects: each on their own line with full lineage separated by | -# e.g. Subject One|Subject Two|Subject Three -# - rename subjects: -# e.g. Subject One => Subject 1 - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument('changes', type=argparse.FileType('r'), help='changes to make') - - def handle(self, *args, **options): - with open(settings.SUBJECTS_YAML) as f: - subjects = self.load_subjects(f) - - for change in options['changes'].readlines(): - if not change.strip() or change.startswith('#'): - continue - subject_names = change.split('=>') - if len(subject_names) == 1: - self.add_subject(subjects, subject_names[0]) - elif len(subject_names) == 2: - self.rename_subject(subjects, *subject_names) - else: - raise ValueError('What is this: {}'.format(change)) - - subjects_list = sorted(subjects.values(), key=lambda s: s['name']) - with open(settings.SUBJECTS_YAML, 'w') as f: - yaml.dump(subjects_list, f, default_flow_style=False) - - def load_subjects(self, fobj): - subject_list = yaml.load(fobj) - subject_map = {} - for s in subject_list: - if s['name'] in subject_map: - raise ValueError('Duplicate subject: {}'.format(s['name'])) - if not s.get('uri'): - s['uri'] = uuid.uuid4().urn - s['name'] = s['name'].strip() - subject_map[s['name']] = s - return subject_map - - def add_subject(self, subjects, lineage): - if isinstance(lineage, str): - lineage = [s.strip() for s in lineage.split('|')] - new_subject = lineage[-1] - parent = None - if len(lineage) > 1: - parent = lineage[-2] - if parent not in subjects: - self.add_subject(subjects, lineage[:-1]) - if new_subject in subjects: - raise ValueError('Duplicate subject: {}'.format(new_subject)) - subjects[new_subject] = { - 'name': new_subject, - 'parent': subjects[parent]['uri'], - 'uri': uuid.uuid4().urn, - } - self.stdout.write('Added subject: {}'.format(new_subject)) - - def rename_subject(self, subjects, old_name, new_name): - old_name = old_name.strip() - new_name = new_name.strip() - if old_name not in subjects: - raise ValueError('Unknown subject: {}'.format(old_name)) - if new_name in subjects: - raise ValueError('Duplicate subject: {}'.format(new_name)) - subjects[old_name]['name'] = new_name - self.stdout.write('Renamed subject: {} => {}'.format(old_name, new_name)) diff --git a/share/management/commands/enforce_set_lists.py b/share/management/commands/enforce_set_lists.py deleted file mode 100644 index 9477b3f18..000000000 --- a/share/management/commands/enforce_set_lists.py +++ /dev/null @@ -1,119 +0,0 @@ -from django.db.models import Q -from share.management.commands import BaseShareCommand -from share.models import SourceConfig, AbstractCreativeWork, ThroughTags -from share.util import IDObfuscator - - -class Command(BaseShareCommand): - """Enforce set whitelist and blacklist for the given source config(s) - - Look at each source config's `transformer_kwargs` for `approved_sets` and `blocked_sets`. - Find and delete works in the database from that source that would not have been allowed, - according to those lists. If the work was ingested from multiple sources, remove the given - source from it instead of deleting. - - By default enforce blacklists only, because enforcing whitelists is slooow. - Pass --whitelist to enforce whitelists too. - """ - - def add_arguments(self, parser): - parser.add_argument('source_configs', nargs='+', type=str, help='Labels of the source configs to enforce') - parser.add_argument('--dry', action='store_true', help='Print changes that would be made, but make no changes') - parser.add_argument('--commit', action='store_true', help='If omitted, roll back the transaction after making changes') - parser.add_argument('--whitelist', action='store_true', help='Enforce whitelist too -- will be slow') - parser.add_argument('--delete-related', action='store_true', help='When deleting a work, also delete related works') - parser.add_argument('--superfluous', action='store_true', help='Reprocess already deleted works') - - def handle(self, *args, **options): - source_configs = options['source_configs'] - dry_run = options['dry'] - commit = options['commit'] - delete_related = options['delete_related'] - superfluous = options['superfluous'] - - with self.rollback_unless_commit(commit): - source_configs = SourceConfig.objects.filter(label__in=source_configs).select_related('source') - for source_config in source_configs: - self.stdout.write('\nEnforcing blacklist for {}'.format(source_config.label), style_func=self.style.SUCCESS) - to_delete = self.enforce_blacklist_qs(source_config) - if to_delete is not None: - self.delete_works(to_delete, source_config, dry_run, superfluous, delete_related) - - if options['whitelist']: - self.stdout.write('\nEnforcing whitelist for {}'.format(source_config.label), style_func=self.style.SUCCESS) - to_delete = self.enforce_whitelist_qs(source_config) - if to_delete is not None: - self.delete_works(to_delete, source_config, dry_run, superfluous, delete_related) - - def enforce_blacklist_qs(self, source_config): - blacklist = source_config.transformer_kwargs.get('blocked_sets') if source_config.transformer_kwargs else None - if not blacklist: - self.stdout.write('{} has no blocked sets, skipping...'.format(source_config.label), style_func=self.style.WARNING) - return - - bad_through_tags = ThroughTags.objects.filter( - sources__id=source_config.source.user_id, - tag__name__in=blacklist - ) - - return AbstractCreativeWork.objects.filter( - id__in=bad_through_tags.values_list('creative_work_id'), - sources__id=source_config.source.user_id, - ) - - def enforce_whitelist_qs(self, source_config): - whitelist = source_config.transformer_kwargs.get('approved_sets') if source_config.transformer_kwargs else None - if not whitelist: - self.stdout.write('{} has no approved sets, skipping...'.format(source_config.label), style_func=self.style.WARNING) - return - - good_through_tags = ThroughTags.objects.filter( - sources__id=source_config.source.user_id, - tag__name__in=whitelist - ) - - # This will be slooow - return AbstractCreativeWork.objects.filter( - sources__id=source_config.source.user_id, - ).exclude( - id__in=good_through_tags.values_list('creative_work_id') - ) - - def delete_works(self, works_qs, source_config, dry_run, superfluous, delete_related): - works_deleted = [] - - if not superfluous: - works_qs = works_qs.filter(is_deleted=False) - for work in works_qs.prefetch_related('sources'): - works_deleted.append(work.id) - # If we've heard about the work from another source, just remove this source from it instead of deleting - if len(work.sources.all()) > 1: - self.stdout.write('{}: {}'.format( - self.style.WARNING('Removing {} from {}'.format(source_config.source.name, IDObfuscator.encode(work))), - work.title - )) - if not dry_run: - work.sources.remove(source_config.source.user) - # poke it to reindex - work.administrative_change(allow_empty=True) - else: - self.stdout.write('{}: {}'.format( - self.style.NOTICE('Deleting work {}'.format(IDObfuscator.encode(work))), - work.title - )) - if not dry_run: - work.administrative_change(is_deleted=True) - self.stdout.write('\nProcessed {} works!'.format(len(works_deleted)), style_func=self.style.SUCCESS) - if not delete_related: - return - - self.stdout.write('\nNow deleting related works...\n') - - related_works = AbstractCreativeWork.objects.filter( - Q(incoming_creative_work_relations__subject_id__in=works_deleted) | Q(outgoing_creative_work_relations__related_id__in=works_deleted), - is_deleted=False, - sources__id=source_config.source.user_id - ).prefetch_related('sources') - - # Traverse related works only one level deep, please - self.delete_works(related_works, source_config, dry_run, superfluous, False) diff --git a/share/management/commands/fix_datacite.py b/share/management/commands/fix_datacite.py deleted file mode 100644 index ea52fae79..000000000 --- a/share/management/commands/fix_datacite.py +++ /dev/null @@ -1,125 +0,0 @@ -import random -import string -import pendulum - -from django.db.models import Q - -from share.models import WorkIdentifier, Preprint, ShareUser - -from share.management.commands import BaseShareCommand -from share.util import IDObfuscator - - -class Command(BaseShareCommand): - - LIMIT = 250 - OTHER_PROVIDERS = ['arXiv'] - AGGREGATORS = ['DataCite MDS', 'CrossRef'] - - def add_arguments(self, parser): - parser.add_argument('--commit', action='store_true', help='Should the script actually commit?') - parser.add_argument('--dry', action='store_true', help='Should the script actually make changes? (In a transaction)') - parser.add_argument('--osf-only', action='store_true', help='Should the script limit to works from OSF sources?') - parser.add_argument('--limit', type=int, default=self.LIMIT, help='Maximum number of works to fix') - parser.add_argument('--from', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider works modified on or after this date') - parser.add_argument('--until', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider works modified on or before this date') - - def handle(self, *args, **options): - with self.rollback_unless_commit(options.get('commit')): - self.stdout.write(self.style.SUCCESS('Entered Transaction')) - - graveyard = WorkIdentifier.objects.get(uri='http://osf.io/8bg7d/').creative_work - self.stdout.write(self.style.SUCCESS('Found the Graveyard @ "{}"'.format(graveyard.id))) - - user_query = Q(source__canonical=True) - if options.get('osf_only'): - user_query &= ~Q(source__long_title__in=self.OTHER_PROVIDERS) - else: - user_query |= Q(source__long_title__in=self.OTHER_PROVIDERS) - - original_user_ids = set(ShareUser.objects.filter(user_query).values_list('id', flat=True)) - - aggregator_user_ids = ShareUser.objects.filter(source__long_title__in=self.AGGREGATORS).values_list('id', flat=True) - - pps = Preprint.objects.filter( - sources__in=aggregator_user_ids, - id__in=Preprint.objects.filter(sources__in=original_user_ids) - ) - if options.get('from'): - pps = pps.filter(date_modified__gte=options.get('from')) - else: - pps = pps.filter(date_modified__gte='2017-07-30') - - if options.get('until'): - pps = pps.filter(date_modified__lte=options.get('until')) - - limit = options.get('limit') - i = 0 - for work in pps.iterator(): - if i >= limit: - self.stdout.write(self.style.SUCCESS('Fixed {} works, but there might be more. Stopping...'.format(i))) - break - - dupes = {} - - for agent in work.related_agents.filter(type='share.person').include('identifiers', 'sources', 'sources__source', 'work_relations'): - if not (agent.given_name and agent.family_name): - continue - dupes.setdefault((agent.given_name, agent.family_name), []).append(agent) - - # Filter down to just duplicated agents - for k in list(dupes.keys()): - if len(dupes[k]) < 2: - del dupes[k] - - # If there are no dupes, we have nothing todo - if not dupes: - continue - - i += 1 - self.stdout.write('=== Processing Work "{}" from {} Modified On {} ==='.format( - IDObfuscator.encode(work), - [u.source.long_title for u in work.sources.all()], - work.date_modified - )) - - for agents in dupes.values(): - # Order by # of identifiers and a preference towards original sources - core_agent = list(sorted(agents, key=lambda x: ( - len(original_user_ids.intersection([s.id for s in x.sources.all()])), - len(x.identifiers.all()), - len(x.work_relations.all()), - ), reverse=True))[0] - - self.stdout.write('\tSmashing {} into {} from {} identified by {}'.format( - [a for a in agents if a != core_agent], - core_agent, - # core_agent.sources.values_list('source__long_title', flat=True), - # core_agent.identifiers.values_list('uri', flat=True), - [user.source.long_title for user in core_agent.sources.all()], - [identifier.uri for identifier in core_agent.identifiers.all()] - )) - - for agent in agents: - if agent == core_agent: - continue - for identifier in agent.identifiers.all(): - self.stdout.write('\t\tRepointing {}: {} -> {}'.format(identifier.uri, agent, core_agent)) - if not options.get('dry'): - identifier.administrative_change(agent=core_agent) - - for rel in agent.work_relations.all(): - if rel.creative_work_id != work.id: - continue - self.stdout.write('\t\tReassigning {}: {} -> {}'.format(rel, IDObfuscator.encode(rel.creative_work), IDObfuscator.encode(graveyard))) - if not options.get('dry'): - rel.administrative_change(creative_work=graveyard, type=''.join(random.sample(string.ascii_letters, 5))) - - self.stdout.write(self.style.SUCCESS('\tSuccessfully Processed Agent "{}"'.format(IDObfuscator.encode(agent)))) - self.stdout.write('Bumping last_modified on work') - if not options.get('dry'): - work.administrative_change(allow_empty=True) - self.stdout.write(self.style.SUCCESS('Successfully Processed Work "{}"\n'.format(IDObfuscator.encode(work)))) - else: - # Did not break - self.stdout.write(self.style.SUCCESS('Fixed {} works, and that\'s all!'.format(i))) diff --git a/share/management/commands/forceingest.py b/share/management/commands/forceingest.py deleted file mode 100644 index ba31f5f85..000000000 --- a/share/management/commands/forceingest.py +++ /dev/null @@ -1,201 +0,0 @@ -import random -import string -import pendulum - -from django.db.utils import IntegrityError - -from share.exceptions import MergeRequired, ShareException -from share.models import AbstractCreativeWork, AbstractAgent -from share.management.commands import BaseShareCommand -from share.models import IngestJob -from share.tasks.jobs import IngestJobConsumer -from share.util.osf import osf_sources - - -MAX_RETRIES = 128 -GRAVEYARD_IDENTIFIERS = [ - 'http://osf.io/8bg7d/', # prod - 'http://staging.osf.io/8hym9/', # staging -] - - -class Command(BaseShareCommand): - - def add_arguments(self, parser): - parser.add_argument('--dry', action='store_true', help='Should the script actually make changes? (In a transaction)') - parser.add_argument('--commit', action='store_true', help='Should the script actually commit?') - parser.add_argument('--noninteractive', action='store_true', help='Should the script merge objects without asking?') - parser.add_argument('--from', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider jobs on or after this date') - parser.add_argument('--until', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider jobs on or before this date') - - def handle(self, *args, **options): - dry_run = options.get('dry') - interactive = not options.get('noninteractive') - - qs = IngestJob.objects.filter( - error_type='MergeRequired', - status=IngestJob.STATUS.failed, - source_config__source__in=osf_sources() - ).order_by('date_created') - if options.get('from'): - qs = qs.filter(date_modified__gte=options.get('from')) - if options.get('until'): - qs = qs.filter(date_modified__lte=options.get('until')) - - hacky_merger = HackyMerger(dry_run, interactive, self) - - for job in qs.select_related('suid', 'source_config'): - with self.rollback_unless_commit(options.get('commit')): - self.stdout.write('\n\nTrying job {!r}'.format(job), style_func=self.style.HTTP_INFO) - self._try_job(job, hacky_merger) - - def _try_job(self, job, hacky_merger): - for i in range(MAX_RETRIES): - self.stdout.write('Attempt {} of {}:'.format(i + 1, MAX_RETRIES)) - try: - IngestJobConsumer().consume(job_id=job.id, exhaust=False) - except MergeRequired as e: - (_, *dupe_sets) = e.args - - try: - for dupes in dupe_sets: - hacky_merger.merge(dupes) - except RejectMerge: - self.stdout.write('Skipping job...', style_func=self.style.WARNING) - return - except CannotMerge as e: - self.stdout.write('Failed to merge:', style_func=self.style.ERROR) - self.stdout.write('\t{!r}'.format(e)) - return - - if hacky_merger.dry_run: - return - continue - except Exception as e: - self.stdout.write('Failed in a way we cannot fix:', style_func=self.style.ERROR) - self.stdout.write('\t{!r}'.format(e)) - return - self.stdout.write('Success!', style_func=self.style.SUCCESS) - return - self.stdout.write('Failed to fix after {} tries'.format(MAX_RETRIES), style_func=self.style.ERROR) - - -class CannotMerge(ShareException): - pass - - -class RejectMerge(ShareException): - pass - - -class HackyMerger: - def __init__(self, dry_run, interactive, command): - self.dry_run = dry_run - self.interactive = interactive - self.command = command - - self.graveyard = AbstractCreativeWork.objects.get(identifiers__uri__in=GRAVEYARD_IDENTIFIERS) - - def merge(self, dupes): - if len(dupes) < 2: - return - - model = list(dupes)[0]._meta.concrete_model - if any(d._meta.concrete_model is not model for d in dupes): - raise CannotMerge('Things in different tables are not dupes: {}'.format(dupes)) - - if model is AbstractAgent: - return self.merge_agents(dupes) - if model is AbstractCreativeWork: - return self.merge_works(dupes) - raise CannotMerge('Cannot merge dupes of type {}'.format(model)) - - def merge_agents(self, dupes): - agents = AbstractAgent.objects.filter( - id__in=[d.id for d in dupes], - ).include( - 'identifiers', - 'sources', - 'sources__source', - 'work_relations', - ) - - # Order by # of canonical sources with a preference for more identifiers and relations - winner, *losers = list(sorted(agents, key=lambda agent: ( - sum(1 for s in agent.sources.all() if s.source.canonical), - len(agent.identifiers.all()), - len(agent.work_relations.all()), - ), reverse=True)) - - self.describe_smash(winner, losers) - if self.interactive: - safe_to_assume_yes = all( - loser.name == winner.name - and len(loser.identifiers.all()) == 0 - for loser in losers - ) - if not (safe_to_assume_yes or self.command.input_confirm('OK? (y/n) ')): - raise RejectMerge - - if self.dry_run: - return - - for loser in losers: - for identifier in list(loser.identifiers.all()): - identifier.administrative_change(agent=winner) - - for rel in list(loser.work_relations.all()): - try: - rel.administrative_change(agent=winner) - except IntegrityError: - # OK, to the graveyard instead - rel.administrative_change( - agent=loser, - creative_work=self.graveyard, - type=''.join(random.sample(string.ascii_letters, 5)) - ) - - def merge_works(self, dupes): - works = AbstractCreativeWork.objects.filter( - id__in=[d.id for d in dupes], - ).include( - 'identifiers', - 'sources', - 'sources__source', - 'agent_relations', - ) - - # Order by # of canonical sources with a preference for more identifiers and relations - winner, *losers = list(sorted(works, key=lambda work: ( - sum(1 for user in work.sources.all() if user.source.canonical), - len(work.identifiers.all()), - len(work.agent_relations.all()), - ), reverse=True)) - - self.describe_smash(winner, losers) - if self.interactive and not self.command.input_confirm('OK? (y/n) '): - return - - for loser in losers: - for identifier in loser.identifiers.all(): - if not self.dry_run: - identifier.administrative_change(creative_work=winner) - - if not self.dry_run: - loser.administrative_change(is_deleted=True) - - def describe_smash(self, winner, losers): - self.command.stdout.write('\tSmashing the following:', style_func=self.command.style.WARNING) - for loser in losers: - self.command.stdout.write(self.format_agent_or_work(loser)) - self.command.stdout.write('\tinto:', style_func=self.command.style.WARNING) - self.command.stdout.write(self.format_agent_or_work(winner)) - - def format_agent_or_work(self, obj): - return '\t\t{} ({})'.format( - obj, - ', '.join(sorted( - [i.uri for i in obj.identifiers.all()], - key=len, - )), - ) diff --git a/share/management/commands/removeanonymousworks.py b/share/management/commands/removeanonymousworks.py deleted file mode 100644 index 6f197b128..000000000 --- a/share/management/commands/removeanonymousworks.py +++ /dev/null @@ -1,25 +0,0 @@ -from django.core.management.base import BaseCommand -from django.db.models import Exists -from django.db.models import OuterRef - -from share.search import SearchIndexer - -from project import celery_app - -from share.models import WorkIdentifier -from share.models import AbstractCreativeWork - - -class Command(BaseCommand): - - def handle(self, *args, **options): - qs = AbstractCreativeWork.objects.annotate( - has_identifiers=Exists( - WorkIdentifier.objects.filter(creative_work=OuterRef('pk')) - ) - ).exclude(has_identifiers=True) - - indexer = SearchIndexer(celery_app) - - for id in qs.values_list('id', flat=True).iterator(): - indexer.index('creativework', id) diff --git a/share/models/__init__.py b/share/models/__init__.py index 5a1de7654..06d42493a 100644 --- a/share/models/__init__.py +++ b/share/models/__init__.py @@ -1,14 +1,7 @@ # NOTE: The order of these imports actually matter from share.models.core import * # noqa from share.models.ingest import * # noqa -from share.models.meta import * # noqa -from share.models.change import * # noqa -from share.models.agents import * # noqa -from share.models.creative import * # noqa -from share.models.base import ExtraData # noqa from share.models.registration import * # noqa -from share.models.identifiers import * # noqa -from share.models.relations import * # noqa from share.models.banner import * # noqa from share.models.ingest import * # noqa from share.models.jobs import * # noqa diff --git a/share/models/agents.py b/share/models/agents.py deleted file mode 100644 index 9784c15fd..000000000 --- a/share/models/agents.py +++ /dev/null @@ -1,50 +0,0 @@ -from django.db import models - -from share.disambiguation.criteria import MatchByAttrs, MatchByOneToMany -from share.models.base import ShareObject -from share.models.base import TypedShareObjectMeta -from share.models.fields import ShareManyToManyField -from share.util import ModelGenerator - - -class AbstractAgent(ShareObject, metaclass=TypedShareObjectMeta): - """ - An Agent is an entity that has the power to act, e.g. an individual person or a group of people. - - Agents make decisions and produce or contribute to the production of creative works. - """ - - name = models.TextField(blank=True, db_index=True) - location = models.TextField(blank=True) - related_agents = ShareManyToManyField('AbstractAgent', through='AbstractAgentRelation', through_fields=('subject', 'related'), symmetrical=False) - related_works = ShareManyToManyField('AbstractCreativeWork', through='AbstractAgentWorkRelation') - - matching_criteria = MatchByOneToMany('identifiers') - - class Meta(ShareObject.Meta): - db_table = 'share_agent' - index_together = ( - ('type', 'name',) - ) - - def __str__(self): - return self.name - - -generator = ModelGenerator(field_types={ - 'text': models.TextField -}) -globals().update(generator.subclasses_from_yaml(__file__, AbstractAgent)) - - -def add_unique_name_criteria(*model_classes): - match_by_identifiers_or_name = [ - AbstractAgent.matching_criteria, - MatchByAttrs('name', allowed_models=model_classes), - ] - for model_class in model_classes: - model_class.matching_criteria = match_by_identifiers_or_name - -add_unique_name_criteria(Institution, Organization, Consortium) # noqa - -Department.matching_criteria = AbstractAgent.matching_criteria # noqa diff --git a/share/models/agents.yaml b/share/models/agents.yaml deleted file mode 100644 index d64050bef..000000000 --- a/share/models/agents.yaml +++ /dev/null @@ -1,27 +0,0 @@ -Agent: - description: > - An Agent is a thing that has the power to act, to make decisions, - to produce or contribute to the production of creative works. - Either an individual person or a group of people. - children: - Organization: - children: - Institution: {} - Consortium: {} - Department: {} - Person: - verbose_name_plural: people - fields: - family_name: - type: text - kwargs: { blank: true } - given_name: - type: text - kwargs: { blank: true } - additional_name: - type: text - kwargs: { blank: true } - suffix: - type: text - kwargs: { blank: true } - children: {} diff --git a/share/models/base.py b/share/models/base.py deleted file mode 100644 index ad6ab3c69..000000000 --- a/share/models/base.py +++ /dev/null @@ -1,265 +0,0 @@ -import re -import sys -import copy - -from django.conf import settings -from django.contrib.contenttypes.models import ContentType -from django.db import DatabaseError -from django.db import models -from django.db import transaction -from django.db.models.base import ModelBase -from django.db.models.fields import AutoField -from django.utils.translation import ugettext_lazy as _ - -from typedmodels import models as typedmodels - -from db.deletion import DATABASE_CASCADE - -from share import util -from share.models import fields -from share.models.change import Change -from share.models.fuzzycount import FuzzyCountManager -from share.models.indexes import ConcurrentIndex -from share.models.sql import ShareObjectManager - - -class ShareObjectVersion(models.Model): - action = models.TextField(max_length=10) - objects = FuzzyCountManager() - - class Meta: - abstract = True - ordering = ('-date_modified', ) - base_manager_name = 'objects' - - def __repr__(self): - return '<{type}({id}, of {persistent_id} at {date_modified})>'.format( - type=type(self).__name__, - id=self.id, - persistent_id=self.persistent_id_id, - date_modified=self.date_modified, - ) - - -class ShareObjectJSONAPIMeta(util.BaseJSONAPIMeta): - skip_null_values = True - - -# Generates 2 class from the original definition of the model -# A concrete class, -# And a version class, Version -class ShareObjectMeta(ModelBase): - concrete_bases = () - version_bases = (ShareObjectVersion, ) - - # This if effectively the "ShareBaseClass" - # Due to limitations in Django and TypedModels we cannot have an actual inheritance chain - share_attrs = { - 'change': lambda: models.OneToOneField(Change, related_name='affected_%(class)s', editable=False, on_delete=DATABASE_CASCADE), - 'date_modified': lambda: models.DateTimeField(auto_now=True, editable=False, db_index=True, help_text=_('The date this record was modified by SHARE.')), - 'date_created': lambda: models.DateTimeField(auto_now_add=True, editable=False, help_text=_('The date of ingress to SHARE.')), - 'JSONAPIMeta': ShareObjectJSONAPIMeta, - } - - def __new__(cls, name, bases, attrs): - if (models.Model in bases and attrs['Meta'].abstract) or len(bases) > 1: - attrs = { - **{k: v() for k, v in cls.share_attrs.items()}, - **attrs - } - return super(ShareObjectMeta, cls).__new__(cls, name, bases, attrs) - - version_attrs = {} - for key, val in attrs.items(): - if isinstance(val, models.Field) and (val.unique or val.db_index): - val = copy.deepcopy(val) - val._unique = False - val.db_index = False - if isinstance(val, models.Field) and val.is_relation: - val = copy.deepcopy(val) - if isinstance(val, models.ForeignKey) and not isinstance(val, fields.ShareForeignKey): - val.remote_field.related_name = '+' - if isinstance(val, (fields.ShareForeignKey, fields.ShareManyToManyField, fields.ShareOneToOneField)): - val._kwargs = {**val._kwargs, 'related_name': '+', 'db_index': False} - if key == 'Meta': - val = type('VersionMeta', (val, ShareObjectVersion.Meta), {'unique_together': None, 'db_table': val.db_table + 'version' if hasattr(val, 'db_table') else None}) - version_attrs[key] = val - - # TODO Fix this in some non-horrid fashion - if name != 'ExtraData': - version_attrs['extra'] = fields.ShareForeignKey('ExtraData', null=True) - - version = super(ShareObjectMeta, cls).__new__(cls, name + 'Version', cls.version_bases, { - **version_attrs, - **cls.share_attrs, - **{k: v() for k, v in cls.share_attrs.items()}, # Excluded sources from versions. They never get filled out - 'persistent_id': models.ForeignKey(name, db_column='persistent_id', related_name='versions', on_delete=DATABASE_CASCADE), - '__qualname__': attrs['__qualname__'] + 'Version', - 'same_as': fields.ShareForeignKey(name, null=True, related_name='+'), - }) - - if name != 'ExtraData': - attrs['extra'] = fields.ShareOneToOneField('ExtraData', null=True) - - concrete = super(ShareObjectMeta, cls).__new__(cls, name, (bases[0], ) + cls.concrete_bases, { - **attrs, - **{k: v() for k, v in cls.share_attrs.items()}, - 'VersionModel': version, - 'same_as': fields.ShareForeignKey(name, null=True, related_name='+'), - 'version': models.OneToOneField(version, editable=False, related_name='%(app_label)s_%(class)s_version', on_delete=DATABASE_CASCADE), - # TypedManyToManyField works just like a normal field but has some special code to handle proxy models (if the exist) - # and makes the database use ON DELETE CASCADE as opposed to Djangos software cascade - 'sources': fields.TypedManyToManyField(settings.AUTH_USER_MODEL, related_name='source_%(class)s', editable=False), - }) - - # Inject Version into the module of the original class definition - __import__(concrete.__module__) - setattr(sys.modules[concrete.__module__], concrete.VersionModel.__name__, concrete.VersionModel) - - return concrete - - -class TypedShareObjectMeta(ShareObjectMeta, typedmodels.TypedModelMetaclass): - concrete_bases = (typedmodels.TypedModel,) - version_bases = (ShareObjectVersion, typedmodels.TypedModel) - - def __new__(cls, name, bases, attrs): - # Any subclasses of a class that already uses this metaclass will be - # turned into a proxy to the original table via TypedModelMetaclass - if ShareObject not in bases: - version = typedmodels.TypedModelMetaclass.__new__(cls, name + 'Version', (bases[0].VersionModel, ), { - **attrs, - '__qualname__': attrs['__qualname__'] + 'Version' - }) - - # Our triggers don't update django typed's type field. - # Makes the concrete type option resolve properly when loading versions from the db - # And forces queries to use the concrete models key - version._typedmodels_type = 'share.' + name.lower() - version._typedmodels_registry['share.' + name.lower()] = version - - return typedmodels.TypedModelMetaclass.__new__(cls, name, bases, {**attrs, 'VersionModel': version}) - return super(TypedShareObjectMeta, cls).__new__(cls, name, bases, attrs) - - -class ExtraData(models.Model, metaclass=ShareObjectMeta): - data = fields.DateTimeAwareJSONField(default=dict) - - objects = FuzzyCountManager() - - class Meta: - abstract = False - - -class ShareObject(models.Model, metaclass=ShareObjectMeta): - id = models.AutoField(primary_key=True) - objects = ShareObjectManager() - changes = fields.GenericRelationNoCascade('Change', related_query_name='share_objects', content_type_field='target_type', object_id_field='target_id', for_concrete_model=True) - - class Meta: - abstract = True - base_manager_name = 'objects' - indexes = [ - ConcurrentIndex(fields=['date_created']) - ] - - def __repr__(self): - return '<{}({}) {}>'.format(self.__class__.__name__, self.id, self) - - def get_absolute_url(self): - return '{}{}/{}'.format(settings.SHARE_WEB_URL, self._meta.model_name, util.IDObfuscator.encode(self)) - - def administrative_change(self, allow_empty=False, **kwargs): - from share.models import Change - from share.models import ChangeSet - from share.models import NormalizedData - from share.models import ShareUser - - with transaction.atomic(): - if not kwargs and not allow_empty: - # Empty changes can be made to force modified_date to update - raise ValueError('Pass allow_empty=True to allow empty changes') - - serialized = {} - for key, value in tuple(kwargs.items()): - if isinstance(value, ShareObject): - serialized[key] = {'@id': util.IDObfuscator.encode(value), '@type': value._meta.model_name} - else: - serialized[key] = value - - nd = NormalizedData.objects.create( - source=ShareUser.objects.get(username='system'), - data={ - '@graph': [{'@id': self.pk, '@type': self._meta.model_name, **serialized}] - } - ) - - cs = ChangeSet.objects.create(normalized_data=nd, status=ChangeSet.STATUS.accepted) - change = Change.objects.create(change={}, node_id=str(self.pk), type=Change.TYPE.update, target=self, target_version=self.version, change_set=cs, model_type=ContentType.objects.get_for_model(type(self))) - - acceptable_fields = set(f.name for f in self._meta.get_fields()) - for key, value in kwargs.items(): - if key not in acceptable_fields: - raise AttributeError(key) - setattr(self, key, value) - self.change = change - - self.save() - - # NOTE/TODO Version will be popluated when a share object is first created - # Updating a share object WILL NOT update the version - def _save_table(self, raw=False, cls=None, force_insert=False, force_update=False, using=None, update_fields=None): - """ - Does the heavy-lifting involved in saving. Updates or inserts the data - for a single table. - """ - meta = cls._meta - non_pks = [f for f in meta.local_concrete_fields if not f.primary_key] - - if update_fields: - non_pks = [f for f in non_pks - if f.name in update_fields or f.attname in update_fields] - - pk_val = self._get_pk_val(meta) - if pk_val is None: - pk_val = meta.pk.get_pk_value_on_save(self) - setattr(self, meta.pk.attname, pk_val) - pk_set = pk_val is not None - if not pk_set and (force_update or update_fields): - raise ValueError("Cannot force an update in save() with no primary key.") - updated = False - # If possible, try an UPDATE. If that doesn't update anything, do an INSERT. - if pk_set and not force_insert: - base_qs = cls._base_manager.using(using) - values = [(f, None, (getattr(self, f.attname) if raw else f.pre_save(self, False))) - for f in non_pks] - forced_update = update_fields or force_update - updated = self._do_update(base_qs, using, pk_val, values, update_fields, - forced_update) - if force_update and not updated: - raise DatabaseError("Forced update did not affect any rows.") - if update_fields and not updated: - raise DatabaseError("Save with update_fields did not affect any rows.") - if not updated: - if meta.order_with_respect_to: - # If this is a model with an order_with_respect_to - # autopopulate the _order field - field = meta.order_with_respect_to - filter_args = field.get_filter_kwargs_for_object(self) - order_value = cls._base_manager.using(using).filter(**filter_args).count() - self._order = order_value - - fields = meta.local_concrete_fields - if not pk_set: - fields = [f for f in fields if not isinstance(f, AutoField)] - - update_pk = bool(meta.auto_field is not None and not pk_set) - result = self._do_insert(cls._base_manager, using, fields, update_pk, raw) - if update_pk: - ### ACTUAL CHANGE HERE ### - # Use regex as it will, hopefully, fail if something get messed up - pk, version_id = re.match(r'\((\d+),(\d+)\)', result).groups() - setattr(self, meta.pk.attname, int(pk)) - setattr(self, meta.get_field('version').attname, int(version_id)) - ### /ACTUAL CHANGE HERE ### - return updated diff --git a/share/models/change.py b/share/models/change.py deleted file mode 100644 index d7311e35e..000000000 --- a/share/models/change.py +++ /dev/null @@ -1,249 +0,0 @@ -import logging - -from model_utils import Choices - -from django.apps import apps -from django.conf import settings -from django.contrib.contenttypes.fields import GenericForeignKey -from django.contrib.contenttypes.models import ContentType -from django.contrib.postgres.fields import JSONField -from django.db import connection -from django.db import models -from django.db import transaction -from django.db.utils import IntegrityError -from django.utils import timezone -from django.utils.translation import ugettext as _ - -from share.exceptions import IngestConflict -from share.models import NormalizedData -from share.models.fuzzycount import FuzzyCountManager -from share.models.indexes import ConcurrentIndex -from share.util import IDObfuscator, BaseJSONAPIMeta - - -__all__ = ('Change', 'ChangeSet', ) -logger = logging.getLogger(__name__) - - -class ChangeSet(models.Model): - STATUS = Choices((0, 'pending', _('pending')), (1, 'accepted', _('accepted')), (2, 'rejected', _('rejected'))) - - objects = FuzzyCountManager() - - status = models.IntegerField(choices=STATUS, default=STATUS.pending) - submitted_at = models.DateTimeField(auto_now_add=True) - normalized_data = models.ForeignKey(NormalizedData, on_delete=models.CASCADE) - - _changes_cache = [] - - class JSONAPIMeta(BaseJSONAPIMeta): - pass - - def accept(self, save=True): - ret = [] - with transaction.atomic(): - self._changes_cache = list(self.changes.all()) - for c in self._changes_cache: - ret.append(c.accept(save=save)) - self.status = ChangeSet.STATUS.accepted - if save: - self.save() - return ret - - def _resolve_ref(self, ref): - model = apps.get_model('share', model_name=ref['@type']) - ct = ContentType.objects.get_for_model(model, for_concrete_model=True) - try: - if ref['@id'].startswith('_:'): - return next( - change.target - for change in self._changes_cache - if change.target_type == ct - and change.node_id == ref['@id'] - and change.target - ) - return model._meta.concrete_model.objects.get(pk=IDObfuscator.decode_id(ref['@id'])) - except (StopIteration, model.DoesNotExist) as ex: - raise Exception('Could not resolve reference {}'.format(ref)) from ex - - def __repr__(self): - return '<{}({}, {}, {} changes)>'.format(self.__class__.__name__, self.STATUS[self.status].upper(), self.normalized_data.source, self.changes.count()) - - -class Change(models.Model): - TYPE = Choices((0, 'create', _('create')), (1, 'merge', _('merge')), (2, 'update', _('update'))) - - objects = FuzzyCountManager() - - change = JSONField() - node_id = models.TextField() - - type = models.IntegerField(choices=TYPE, editable=False) - # The non-concrete type that this change has made - model_type = models.ForeignKey(ContentType, related_name='+', db_index=False, on_delete=models.CASCADE) - - target_id = models.PositiveIntegerField(null=True) - target = GenericForeignKey('target_type', 'target_id') - target_type = models.ForeignKey(ContentType, related_name='target_%(class)s', on_delete=models.CASCADE) - - target_version_type = models.ForeignKey(ContentType, related_name='target_version_%(class)s', db_index=False, on_delete=models.CASCADE) - target_version_id = models.PositiveIntegerField(null=True, db_index=False) - target_version = GenericForeignKey('target_version_type', 'target_version_id') - - change_set = models.ForeignKey(ChangeSet, related_name='changes', on_delete=models.CASCADE) - - class JSONAPIMeta(BaseJSONAPIMeta): - pass - - class Meta: - ordering = ('pk', ) - indexes = ( - ConcurrentIndex(fields=['target_id', 'target_type']), - ) - - def accept(self, save=True): - # Little bit of blind faith here that all requirements have been accepted - assert self.change_set.status == ChangeSet.STATUS.pending, 'Cannot accept a change with status {}'.format(self.change_set.status) - logger.debug('Accepting change node ({}, {})'.format(ContentType.objects.get_for_id(self.model_type_id), self.node_id)) - try: - ret = self._accept(save) - except IntegrityError as e: - if e.args[0].startswith('duplicate key value violates unique constraint'): - raise IngestConflict - raise - - if save: - # Psuedo hack, sources.add(...) tries to do some safety checks. - # Don't do that. We have a database. That is its job. Let it do its job. - through_meta = ret._meta.get_field('sources').remote_field.through._meta - - with connection.cursor() as cursor: - cursor.execute(''' - INSERT INTO "{0}" - ("{1}", "{2}") - VALUES - (%s, %s) - ON CONFLICT DO NOTHING; - '''.format( - through_meta.db_table, - through_meta.get_field(ret._meta.concrete_model._meta.model_name).column, - through_meta.get_field('shareuser').column, - ), (ret.pk, self.change_set.normalized_data.source_id)) - - self.save() - else: - logger.warning('Calling accept with save=False will not update the sources field') - - return ret - - def _accept(self, save): - if self.type == Change.TYPE.create: - return self._create(save=save) - if self.type == Change.TYPE.update: - return self._update(save=save) - return self._merge(save=save) - - def _create(self, save=True): - resolved_change = self._resolve_change() - inst = ContentType.objects.get_for_id(self.model_type_id).model_class()(change=self, **resolved_change) - if save: - inst.save() - self.target = inst - return inst - - def _update(self, save=True): - self.target.change = self - - new_type = self.change.pop('@type', None) - if new_type: - self.target.recast(new_type) - - for k, v in self._resolve_change().items(): - setattr(self.target, k, v) - - if save: - self.target.save() - return self.target - - def _merge(self, save=True): - from share.models.base import ShareObject - assert save is True, 'Cannot perform merge without saving' - - change = self._resolve_change() - # Find all fields that reference this model - fields = [ - field.field for field in - self.target_type.model_class()._meta.get_fields() - if field.is_relation - and not field.many_to_many - and field.remote_field - and issubclass(field.remote_field.model, ShareObject) - and hasattr(field, 'field') - ] - - # NOTE: Date is pinned up here to ensure its the same for all changed rows - date_modified = timezone.now() - - for field in fields: - # Update all rows in "from" - # Updates the change, the field in question, the version pin of the field in question - # and date_modified must be manually updated - field.model.objects.select_for_update().filter(**{ - field.name + '__in': change['from'] - }).update(**{ - 'change': self, - field.name: change['into'], - field.name + '_version': change['into'].version, - 'date_modified': date_modified, - }) - - # Finally point all from rows' same_as and - # same_as_version to the canonical model. - type(change['into']).objects.select_for_update().filter( - pk__in=[i.pk for i in change['from']] - ).update( - change=self, - same_as=change['into'], - same_as_version=change['into'].version, - date_modified=date_modified, - ) - - return change['into'] - - def _resolve_change(self): - change = {} - for k, v in self.change.items(): - if k == 'extra': - if not v: - continue - if self.target and self.target.extra: - change[k] = self.target.extra - else: - from share.models.base import ExtraData - change[k] = ExtraData() - change[k].change = self - change[k].data.update({self.change_set.normalized_data.source.username: v}) - change[k].save() - change[k + '_version_id'] = change[k].version_id - elif isinstance(v, dict): - inst = self.change_set._resolve_ref(v) - change[k] = inst - try: - change[k + '_version_id'] = inst.version_id - except AttributeError: - # this isn't a ShareObject, no worries - pass - elif isinstance(v, list): - change[k] = [self.change_set._resolve_ref(r) for r in v] - else: - change[k] = v - - if self.target_type.model == 'subject': - SubjectTaxonomy = apps.get_model('share', 'subjecttaxonomy') - user = self.change_set.normalized_data.source - central_synonym = change.get('central_synonym', self.target.central_synonym if self.target else None) - if central_synonym is None and user.username != settings.APPLICATION_USERNAME: - raise PermissionError('Only the system user can modify the central subject taxonomy, not {}'.format(user)) - change['taxonomy'], _ = SubjectTaxonomy.objects.get_or_create(source=user.source) - - return change diff --git a/share/models/creative.py b/share/models/creative.py deleted file mode 100644 index 52220fb43..000000000 --- a/share/models/creative.py +++ /dev/null @@ -1,78 +0,0 @@ -import logging - -from django.db import models -from django.db import transaction -from django.utils.translation import ugettext_lazy as _ - -from share.disambiguation.criteria import MatchByOneToMany -from share.models.base import ShareObject -from share.models.base import TypedShareObjectMeta -from share.models.meta import Tag, Subject -from share.models.change import Change -from share.models.fields import ShareManyToManyField, ShareURLField - -from share.util import ModelGenerator - - -logger = logging.getLogger(__name__) - - -# Base Creative Work class -class AbstractCreativeWork(ShareObject, metaclass=TypedShareObjectMeta): - title = models.TextField(blank=True, help_text='') - description = models.TextField(blank=True, help_text='') - is_deleted = models.BooleanField(default=False, help_text=_('Determines whether or not this record will be discoverable via search.')) - date_published = models.DateTimeField(blank=True, null=True) - date_updated = models.DateTimeField(blank=True, null=True) - free_to_read_type = ShareURLField(blank=True) - free_to_read_date = models.DateTimeField(blank=True, null=True) - rights = models.TextField(blank=True, null=True) - language = models.TextField(blank=True, null=True, help_text=_('The ISO 3166-1 alpha-2 country code indicating the language of this record.')) - - subjects = ShareManyToManyField(Subject, related_name='subjected_works', through='ThroughSubjects') - tags = ShareManyToManyField(Tag, related_name='tagged_works', through='ThroughTags') - - related_agents = ShareManyToManyField('AbstractAgent', through='AbstractAgentWorkRelation') - related_works = ShareManyToManyField('AbstractCreativeWork', through='AbstractWorkRelation', through_fields=('subject', 'related'), symmetrical=False) - - matching_criteria = MatchByOneToMany('identifiers') - - class Meta(ShareObject.Meta): - db_table = 'share_creativework' - verbose_name_plural = 'Creative Works' - - def defrankenize(self, *_, im_really_sure_about_this=False): - if not im_really_sure_about_this: - raise ValueError('You have to be really sure about this') - - logger.info('Defrankenizing %r', self) - - with transaction.atomic(): - logger.info('Removing relations') - for field in AbstractCreativeWork._meta.get_fields(): - if not field.one_to_many or field.name in ('changes', 'versions'): - continue - - logger.warning('Removing all %s', field.related_name) - relation = getattr(self, field.get_accessor_name()) - num_deleted, stats = Change.objects.filter(id__in=relation.values_list('change_id', flat=True)).delete() - logger.warning('Deleted %d changes to remove %s', num_deleted, field.related_name) - - assert num_deleted == stats.pop('share.Change', 0) - - if stats: - logger.error('Unexpectedly removed other rows, %r', stats) - raise ValueError('Unexpectedly removed other rows, {!r}'.format(stats)) - - logger.info('Relations removed') - self.administrative_change(is_deleted=True, title='Defrankenized work') - - def __str__(self): - return self.title - - -generator = ModelGenerator(field_types={ - 'text': models.TextField, - 'boolean': models.NullBooleanField, # Has to be nullable for types models :( -}) -globals().update(generator.subclasses_from_yaml(__file__, AbstractCreativeWork)) diff --git a/share/models/creative.yaml b/share/models/creative.yaml deleted file mode 100644 index 6acd6689a..000000000 --- a/share/models/creative.yaml +++ /dev/null @@ -1,47 +0,0 @@ -CreativeWork: - description: Anything created by an Agent. - children: - DataSet: - description: A collection of information gathered in the course of research. - # Image: - # children: - # Diagram: {} - # Drawing: {} - # Figure: {} - # Photo: {} - # Plot: {} - # Lesson: {} - Patent: - description: A description of an invention, and a government-issued license to exclude others from making or selling that invention. - Poster: {} - Publication: - children: - Article: {} - Book: {} - ConferencePaper: {} - # Deliverable: {} - Dissertation: {} - Preprint: {} - Project: {} - Registration: - fields: - registration_type: - type: text - kwargs: { blank: true } - withdrawn: - type: boolean - justification: - type: text - kwargs: { blank: true } - Report: {} - # Section: {} - Thesis: - verbose_name_plural: theses - WorkingPaper: {} - Presentation: {} - Repository: - verbose_name_plural: repositories - Retraction: {} - Software: - verbose_name_plural: software - # Video: {} diff --git a/share/models/fields.py b/share/models/fields.py index 47b24f9ed..b0f2c39c8 100644 --- a/share/models/fields.py +++ b/share/models/fields.py @@ -6,26 +6,17 @@ from dateutil import parser import jwe from psycopg2.extras import Json -import six from django import forms from django.conf import settings -from django.contrib.contenttypes.fields import GenericRelation from django.contrib.postgres import lookups from django.contrib.postgres.fields.jsonb import JSONField -from django.core import exceptions, validators, checks +from django.core import validators from django.core.exceptions import ValidationError from django.core.serializers.json import DjangoJSONEncoder from django.db import models -from django.db.models.fields.related import lazy_related_operation -from django.db.models.fields.related import resolve_relation -from django.db.models.fields.related_descriptors import ManyToManyDescriptor -from django.db.models.utils import make_model_tuple -from django.utils.functional import curry from django.utils.translation import ugettext_lazy as _ -from db.deletion import DATABASE_CASCADE - class DateTimeAwareJSONEncoder(DjangoJSONEncoder): def default(self, o): @@ -92,9 +83,6 @@ def validate(self, value, model_instance): ) -DatetimeAwareJSONField = DateTimeAwareJSONField - - JSONField.register_lookup(lookups.DataContains) JSONField.register_lookup(lookups.ContainedBy) JSONField.register_lookup(lookups.HasKey) @@ -102,426 +90,9 @@ def validate(self, value, model_instance): JSONField.register_lookup(lookups.HasAnyKeys) -# typedmodels modifies _meta.get_fields() on typed subclasses to filter out -# fields that don't belong, based on the fields given in the model definition. -# Share related fields don't directly contribute to classes, they make -# instances of their base class that contribute instead. Keep track of these -# fields that do a Share field's job, so they'll show up in _meta.get_fields(). -class ShareRelatedField: - - PRETENDING_TO_BE = None - - def __init__(self, *args, **kwargs): - # Correct M2M fields - if 'to' in kwargs and not args: - args = args + (kwargs.pop('to'), ) - - self._kwargs = kwargs - self.__equivalent_fields = set() - super().__init__(*args, **kwargs) - - def add_equivalent_fields(self, *fields): - self.__equivalent_fields.update(fields) - - def contribute_to_class(self, cls, name, **kwargs): - actual = self.PRETENDING_TO_BE(self.remote_field.model, **self._get_kwargs(cls)) - actual.contribute_to_class(cls, name, **kwargs) - - if isinstance(self.remote_field.model, str): - version_model = self.remote_field.model + 'Version' - elif hasattr(self.remote_field.model, 'VersionModel'): - version_model = self.remote_field.model.VersionModel - else: - return # Not pointing at a ShareObject subclass - - version = self.PRETENDING_TO_BE(version_model, **self._get_kwargs(cls, version=True)) - - suffix = '_version' - if self.many_to_many: - suffix += 's' - name = name.rstrip('s') - - version.contribute_to_class(cls, name + suffix, **kwargs) - - actual._share_version_field = version - self.add_equivalent_fields(actual, version) - - def _get_kwargs(self, cls, version=False): - kwargs = {**self._kwargs} - through_fields = kwargs.get('through_fields', None) - - if version: - kwargs['db_index'] = False - kwargs['editable'] = False - kwargs['related_name'] = '+' - - if through_fields: - kwargs['through_fields'] = ( - '{}_version'.format(through_fields[0]) if 'version' in cls._meta.model_name else through_fields[0], - '{}_version'.format(through_fields[1]) if version else through_fields[1] - ) - return kwargs - - def __eq__(self, other): - if other in self.__equivalent_fields: - return True - return super().__eq__(other) - - def __hash__(self): - return super().__hash__() - - -class ShareOneToOneField(ShareRelatedField, models.OneToOneField): - PRETENDING_TO_BE = models.OneToOneField - - def __init__(self, *args, **kwargs): - # Default to delete cascade - kwargs.setdefault('on_delete', DATABASE_CASCADE) - super().__init__(*args, **kwargs) - - -class ShareForeignKey(ShareRelatedField, models.ForeignKey): - PRETENDING_TO_BE = models.ForeignKey - - def __init__(self, *args, **kwargs): - # Default to delete cascade - kwargs.setdefault('on_delete', DATABASE_CASCADE) - super().__init__(*args, **kwargs) - - -def create_many_to_many_intermediary_model(field, klass): - from django.db import models - - def set_managed(model, related, through): - through._meta.managed = model._meta.managed or related._meta.managed - - to_model = resolve_relation(klass, field.remote_field.model) - name = '%s_%s' % (klass._meta.object_name, field.name) - lazy_related_operation(set_managed, klass, to_model, name) - - to = make_model_tuple(to_model)[1] - from_ = klass._meta.model_name - if to == from_: - to = 'to_%s' % to - from_ = 'from_%s' % from_ - - meta = type(str('Meta'), (object,), { - 'db_table': field._get_m2m_db_table(klass._meta), - 'auto_created': klass, - 'app_label': klass._meta.app_label, - 'db_tablespace': klass._meta.db_tablespace, - 'unique_together': (from_, to), - 'verbose_name': _('%(from)s-%(to)s relationship') % {'from': from_, 'to': to}, - 'verbose_name_plural': _('%(from)s-%(to)s relationships') % {'from': from_, 'to': to}, - 'apps': field.model._meta.apps, - }) - # Construct and return the new class. - return type(str(name), (models.Model,), { - 'Meta': meta, - '__module__': klass.__module__, - from_: models.ForeignKey( - klass, - related_name='%s+' % name, - db_tablespace=field.db_tablespace, - db_constraint=field.remote_field.db_constraint, - on_delete=DATABASE_CASCADE, - ), - to: models.ForeignKey( - to_model, - related_name='%s+' % name, - db_tablespace=field.db_tablespace, - db_constraint=field.remote_field.db_constraint, - on_delete=DATABASE_CASCADE, - ) - }) - - +# stub left just for migrations class TypedManyToManyField(models.ManyToManyField): - - def _check_relationship_model(self, from_model=None, **kwargs): - if hasattr(self.remote_field.through, '_meta'): - qualified_model_name = "%s.%s" % ( - self.remote_field.through._meta.app_label, self.remote_field.through.__name__) - else: - qualified_model_name = self.remote_field.through - - errors = [] - - if self.remote_field.through not in self.opts.apps.get_models(include_auto_created=True): - # The relationship model is not installed. - errors.append( - checks.Error( - ("Field specifies a many-to-many relation through model " - "'%s', which has not been installed.") % - qualified_model_name, - hint=None, - obj=self, - id='fields.E331', - ) - ) - - else: - - assert from_model is not None, ( - "ManyToManyField with intermediate " - "tables cannot be checked if you don't pass the model " - "where the field is attached to." - ) - - # Set some useful local variables - to_model = resolve_relation(from_model, self.remote_field.model) - from_model_name = from_model._meta.object_name - if isinstance(to_model, six.string_types): - to_model_name = to_model - else: - to_model_name = to_model._meta.object_name - relationship_model_name = self.remote_field.through._meta.object_name - self_referential = from_model == to_model - - # Check symmetrical attribute. - if (self_referential and self.remote_field.symmetrical - and not self.remote_field.through._meta.auto_created): - errors.append( - checks.Error( - 'Many-to-many fields with intermediate tables must not be symmetrical.', - hint=None, - obj=self, - id='fields.E332', - ) - ) - - # Count foreign keys in intermediate model - if self_referential: - seen_self = sum(from_model == getattr(field.remote_field, 'model', None) - for field in self.remote_field.through._meta.fields) - - if seen_self > 2 and not self.remote_field.through_fields: - errors.append( - checks.Error( - ("The model is used as an intermediate model by " - "'%s', but it has more than two foreign keys " - "to '%s', which is ambiguous. You must specify " - "which two foreign keys Django should use via the " - "through_fields keyword argument.") % (self, from_model_name), - hint=("Use through_fields to specify which two " - "foreign keys Django should use."), - obj=self.remote_field.through, - id='fields.E333', - ) - ) - - else: - # Count foreign keys in relationship model - # HERE IS THE ACTUAL CHANGE - # Look at models _meta.concrete_model to make typed models work - seen_from = len([ - field for field in self.remote_field.through._meta.fields - if hasattr(field.remote_field, 'model') - and from_model._meta.concrete_model == field.remote_field.model._meta.concrete_model - ]) - seen_to = len([ - field for field in self.remote_field.through._meta.fields - if hasattr(field.remote_field, 'model') - and to_model._meta.concrete_model == field.remote_field.model._meta.concrete_model - ]) - - if seen_from > 1 and not self.remote_field.through_fields: - errors.append( - checks.Error( - ("The model is used as an intermediate model by " - "'%s', but it has more than one foreign key " - "from '%s', which is ambiguous. You must specify " - "which foreign key Django should use via the " - "through_fields keyword argument.") % (self, from_model_name), - hint=('If you want to create a recursive relationship, ' - 'use ForeignKey("self", symmetrical=False, ' - 'through="%s").') % relationship_model_name, - obj=self, - id='fields.E334', - ) - ) - - if seen_to > 1 and not self.remote_field.through_fields: - errors.append( - checks.Error( - ("The model is used as an intermediate model by " - "'%s', but it has more than one foreign key " - "to '%s', which is ambiguous. You must specify " - "which foreign key Django should use via the " - "through_fields keyword argument.") % (self, to_model_name), - hint=('If you want to create a recursive ' - 'relationship, use ForeignKey("self", ' - 'symmetrical=False, through="%s").') % relationship_model_name, - obj=self, - id='fields.E335', - ) - ) - - if seen_from == 0 or seen_to == 0: - errors.append( - checks.Error( - ("The model is used as an intermediate model by " - "'%s', but it does not have a foreign key to '%s' or '%s'.") % ( - self, from_model_name, to_model_name - ), - hint=None, - obj=self.remote_field.through, - id='fields.E336', - ) - ) - - # Validate `through_fields`. - if self.remote_field.through_fields is not None: - # Validate that we're given an iterable of at least two items - # and that none of them is "falsy". - if not (len(self.remote_field.through_fields) >= 2 - and self.remote_field.through_fields[0] - and self.remote_field.through_fields[1]): - errors.append( - checks.Error( - ("Field specifies 'through_fields' but does not " - "provide the names of the two link fields that should be " - "used for the relation through model " - "'%s'.") % qualified_model_name, - hint=("Make sure you specify 'through_fields' as " - "through_fields=('field1', 'field2')"), - obj=self, - id='fields.E337', - ) - ) - - # Validate the given through fields -- they should be actual - # fields on the through model, and also be foreign keys to the - # expected models. - else: - assert from_model is not None, ( - "ManyToManyField with intermediate " - "tables cannot be checked if you don't pass the model " - "where the field is attached to." - ) - - source, through, target = from_model, self.remote_field.through, self.remote_field.model - source_field_name, target_field_name = self.remote_field.through_fields[:2] - - for field_name, related_model in ((source_field_name, source), - (target_field_name, target)): - - possible_field_names = [] - for f in through._meta.fields: - if hasattr(f, 'remote_field') and getattr(f.remote_field, 'model', None) == related_model: - possible_field_names.append(f.name) - if possible_field_names: - hint = ("Did you mean one of the following foreign " - "keys to '%s': %s?") % (related_model._meta.object_name, - ', '.join(possible_field_names)) - else: - hint = None - - try: - field = through._meta.get_field(field_name) - except exceptions.FieldDoesNotExist: - errors.append( - checks.Error( - ("The intermediary model '%s' has no field '%s'.") % ( - qualified_model_name, field_name), - hint=hint, - obj=self, - id='fields.E338', - ) - ) - else: - if not (hasattr(field, 'remote_field') - and getattr(field.remote_field, 'model', None) == related_model): - errors.append( - checks.Error( - "'%s.%s' is not a foreign key to '%s'." % ( - through._meta.object_name, field_name, - related_model._meta.object_name), - hint=hint, - obj=self, - id='fields.E339', - ) - ) - - return errors - - def _get_m2m_reverse_attr(self, related, attr): - """ - Function that can be curried to provide the related accessor or DB - column name for the m2m table. - """ - cache_attr = '_m2m_reverse_%s_cache' % attr - if hasattr(self, cache_attr): - return getattr(self, cache_attr) - found = False - if self.remote_field.through_fields is not None: - link_field_name = self.remote_field.through_fields[1] - else: - link_field_name = None - for f in self.remote_field.through._meta.fields: - # HERE IS THE ACTUAL CHANGE - # Look at models _meta.concrete_model to make typed models work - if f.is_relation and f.remote_field.model._meta.concrete_model == related.model._meta.concrete_model: - if link_field_name is None and related.related_model._meta.concrete_model == related.model._meta.concrete_model: - # If this is an m2m-intermediate to self, - # the first foreign key you find will be - # the source column. Keep searching for - # the second foreign key. - if found: - setattr(self, cache_attr, getattr(f, attr)) - break - else: - found = True - elif link_field_name is None or link_field_name == f.name: - setattr(self, cache_attr, getattr(f, attr)) - break - return getattr(self, cache_attr) - - def contribute_to_class(self, cls, name, **kwargs): - # To support multiple relations to self, it's useful to have a non-None - # related name on symmetrical relations for internal reasons. The - # concept doesn't make a lot of sense externally ("you want me to - # specify *what* on my non-reversible relation?!"), so we set it up - # automatically. The funky name reduces the chance of an accidental - # clash. - if self.remote_field.symmetrical and ( - self.remote_field.model == "self" or self.remote_field.model == cls._meta.object_name): - self.remote_field.related_name = "%s_rel_+" % name - elif self.remote_field.is_hidden(): - # If the backwards relation is disabled, replace the original - # related_name with one generated from the m2m field name. Django - # still uses backwards relations internally and we need to avoid - # clashes between multiple m2m fields with related_name == '+'. - self.remote_field.related_name = "_%s_%s_+" % (cls.__name__.lower(), name) - - super(models.ManyToManyField, self).contribute_to_class(cls, name, **kwargs) - - # The intermediate m2m model is not auto created if: - # 1) There is a manually specified intermediate, or - # 2) The class owning the m2m field is abstract. - # 3) The class owning the m2m field has been swapped out. - if not cls._meta.abstract: - if self.remote_field.through: - def resolve_through_model(_, model, field): - field.remote_field.through = model - lazy_related_operation(resolve_through_model, cls, self.remote_field.through, field=self) - elif not cls._meta.swapped: - self.remote_field.through = create_many_to_many_intermediary_model(self, cls) - - # Add the descriptor for the m2m relation. - setattr(cls, self.name, ManyToManyDescriptor(self.remote_field, reverse=False)) - - # Set up the accessor for the m2m table name for the relation. - self.m2m_db_table = curry(self._get_m2m_db_table, cls._meta) - - -class ShareManyToManyField(ShareRelatedField, TypedManyToManyField): - PRETENDING_TO_BE = TypedManyToManyField - - -class URIField(models.TextField): - def __init__(self, *args, **kwargs): - super(URIField, self).__init__(*args, **kwargs) + pass class ShareURLField(models.TextField): @@ -548,14 +119,6 @@ def formfield(self, **kwargs): return super(ShareURLField, self).formfield(**defaults) -class GenericRelationNoCascade(GenericRelation): - @property - def bulk_related_objects(self): - # https://github.com/django/django/blob/master/django/db/models/deletion.py#L151 - # Disable django cascading deletes for this field - raise AttributeError('This is a dirty hack') - - class EncryptedJSONField(models.BinaryField): """ This field transparently encrypts data in the database. It should probably only be used with PG unless diff --git a/share/models/identifiers.py b/share/models/identifiers.py deleted file mode 100644 index 8fa587309..000000000 --- a/share/models/identifiers.py +++ /dev/null @@ -1,69 +0,0 @@ -from django.db import models -from django.utils.translation import ugettext_lazy as _ - -from share.disambiguation.criteria import MatchByAttrs -from share.models.base import ShareObject -from share.models.fields import ShareForeignKey, ShareURLField - -__all__ = ('WorkIdentifier', 'AgentIdentifier', 'WorkIdentifierVersion', 'AgentIdentifierVersion') # noqa - - -# TODO Common interface, so we're not duplicating code. Doesn't work because -# ShareObjectMeta doesn't look at bases when building Version classes. -# -# class Identifier(ShareObject): -# # http://twitter.com/berniethoughts/, mailto://contact@cos.io -# uri = ShareURLField(unique=True) -# -# # twitter.com, cos.io -# host = models.TextField(editable=False) -# -# # http, mailto -# scheme = models.TextField(editable=False) -# -# def save(self, *args, **kwargs): -# f = furl(self.uri) -# self.host = f.host -# self.scheme = f.scheme -# super(Identifier, self).save(*args, **kwargs) -# -# class Meta: -# abstract = True - -class FilteredEmailsManager(models.Manager): - def get_queryset(self): - return super().get_queryset().exclude(scheme='mailto') - - -class WorkIdentifier(ShareObject): - """ - Unique identifier (in IRI form) for a creative work. - """ - uri = ShareURLField(unique=True) - host = models.TextField(blank=True) - scheme = models.TextField(blank=True, help_text=_('A prefix to URI indicating how the following data should be interpreted.')) - creative_work = ShareForeignKey('AbstractCreativeWork', related_name='identifiers') - - # objects = FilteredEmailsManager() - # objects_unfiltered = models.Manager() - - def __repr__(self): - return '<{}({}, {})>'.format(self.__class__.__name__, self.uri, self.creative_work_id) - - matching_criteria = MatchByAttrs('uri') - - -class AgentIdentifier(ShareObject): - """Unique identifier (in IRI form) for an agent.""" - uri = ShareURLField(unique=True) - host = models.TextField(blank=True) - scheme = models.TextField(blank=True) - agent = ShareForeignKey('AbstractAgent', related_name='identifiers') - - # objects = FilteredEmailsManager() - # objects_unfiltered = models.Manager() - - def __repr__(self): - return '<{}({}, {})>'.format(self.__class__.__name__, self.uri, self.agent_id) - - matching_criteria = MatchByAttrs('uri') diff --git a/share/models/meta.py b/share/models/meta.py deleted file mode 100644 index a2189e44a..000000000 --- a/share/models/meta.py +++ /dev/null @@ -1,102 +0,0 @@ -from django.db import models - -from share.disambiguation.criteria import MatchByAttrs, MatchByManyToOne, MatchSubjects -from share.models.base import ShareObject -from share.models.fields import ShareForeignKey, ShareURLField - - -__all__ = ('Tag', 'Subject', 'ThroughTags', 'ThroughSubjects', 'SubjectTaxonomy') - -# TODO Rename this file - - -class CyclicalTaxonomyError(Exception): - pass - - -class Tag(ShareObject): - name = models.TextField(unique=True) - - matching_criteria = MatchByAttrs('name') - - def __str__(self): - return self.name - - -class SubjectTaxonomy(models.Model): - source = models.OneToOneField('Source') - - is_deleted = models.BooleanField(default=False) - date_created = models.DateTimeField(auto_now_add=True) - date_modified = models.DateTimeField(auto_now=True) - - def __repr__(self): - return '<{} {}: {}>'.format(self.__class__.__name__, self.id, self.source.long_title) - - class Meta: - verbose_name_plural = 'Subject Taxonomies' - - -class Subject(ShareObject): - name = models.TextField() - is_deleted = models.BooleanField(default=False) - uri = ShareURLField(null=True, blank=True) - taxonomy = models.ForeignKey(SubjectTaxonomy, editable=False, on_delete=models.CASCADE) - parent = ShareForeignKey('Subject', blank=True, null=True, related_name='children') - central_synonym = ShareForeignKey('Subject', blank=True, null=True, related_name='custom_synonyms') - - matching_criteria = MatchSubjects() - - def save(self, *args, **kwargs): - if self.id is not None and self.parent is not None: - new_lineage = self.parent.lineage() - if self in new_lineage: - raise CyclicalTaxonomyError('Making {} a child of {} would cause a cycle!'.format(self, self.parent)) - return super().save(*args, **kwargs) - - def lineage(self): - query = ''' - WITH RECURSIVE lineage_chain(id, parent, depth, path, cycle) AS ( - SELECT id, parent_id, 1, ARRAY[id], false FROM {table} WHERE id = %(id)s - UNION - SELECT {table}.id, {table}.parent_id, lineage_chain.depth + 1, path || {table}.id, {table}.id = ANY(path) - FROM lineage_chain JOIN {table} ON lineage_chain.parent = {table}.id - WHERE NOT cycle - ) - SELECT {table}.* FROM {table} INNER JOIN lineage_chain ON {table}.id = lineage_chain.id ORDER BY lineage_chain.depth DESC - '''.format(table=self._meta.db_table) - lineage = list(self._meta.model.objects.raw(query, params={'id': self.id})) - if lineage[0].parent is not None: - raise CyclicalTaxonomyError('Subject taxonomy cycle! {}'.format(lineage)) - return lineage - - def __str__(self): - return self.name - - class Meta: - unique_together = (('name', 'taxonomy'), ('uri', 'taxonomy')) - - -# Through Tables for all the things - -class ThroughTags(ShareObject): - tag = ShareForeignKey(Tag, related_name='work_relations') - creative_work = ShareForeignKey('AbstractCreativeWork', related_name='tag_relations') - - class Meta(ShareObject.Meta): - unique_together = ('tag', 'creative_work') - verbose_name_plural = 'through tags' - - matching_criteria = MatchByManyToOne('tag', 'creative_work') - - -class ThroughSubjects(ShareObject): - subject = ShareForeignKey('Subject', related_name='work_relations') - creative_work = ShareForeignKey('AbstractCreativeWork', related_name='subject_relations') - is_deleted = models.BooleanField(default=False) - - class Meta(ShareObject.Meta): - unique_together = ('subject', 'creative_work') - verbose_name_plural = 'through subjects' - - matching_criteria = MatchByManyToOne('subject', 'creative_work') diff --git a/share/models/relations/__init__.py b/share/models/relations/__init__.py deleted file mode 100644 index 6c6cb5604..000000000 --- a/share/models/relations/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from share.models.relations.agentwork import * # noqa -from share.models.relations.creative import * # noqa -from share.models.relations.agent import * # noqa diff --git a/share/models/relations/agent.py b/share/models/relations/agent.py deleted file mode 100644 index 61778d7ab..000000000 --- a/share/models/relations/agent.py +++ /dev/null @@ -1,23 +0,0 @@ -from share.disambiguation.criteria import MatchByManyToOne -from share.models.base import ShareObject, TypedShareObjectMeta -from share.models.fields import ShareForeignKey - -from share.util import ModelGenerator - - -class AbstractAgentRelation(ShareObject, metaclass=TypedShareObjectMeta): - subject = ShareForeignKey('AbstractAgent', related_name='outgoing_agent_relations') - related = ShareForeignKey('AbstractAgent', related_name='incoming_agent_relations') - - matching_criteria = MatchByManyToOne('subject', 'related', constrain_types=True) - - class Meta(ShareObject.Meta): - db_table = 'share_agentrelation' - unique_together = ('subject', 'related', 'type') - - -generator = ModelGenerator() -globals().update(generator.subclasses_from_yaml(__file__, AbstractAgentRelation)) - - -__all__ = tuple(key for key, value in globals().items() if isinstance(value, type) and issubclass(value, ShareObject)) diff --git a/share/models/relations/agent.yaml b/share/models/relations/agent.yaml deleted file mode 100644 index 2e49e478b..000000000 --- a/share/models/relations/agent.yaml +++ /dev/null @@ -1,12 +0,0 @@ -AgentRelation: - children: - IsAffiliatedWith: - verbose_name_plural: is affiliated with - description: The subject agent is officially attached or connected to the related agent. - children: - IsEmployedBy: - verbose_name_plural: is employed by - description: The subject agent is employed by the related agent. - IsMemberOf: - verbose_name_plural: is member of - description: The subject agent is a member of the related group/organization. diff --git a/share/models/relations/agentwork.py b/share/models/relations/agentwork.py deleted file mode 100644 index aba5af61a..000000000 --- a/share/models/relations/agentwork.py +++ /dev/null @@ -1,80 +0,0 @@ -from django.core.exceptions import ValidationError -from django.db import models -from django.utils.translation import ugettext_lazy as _ - -from share.disambiguation.criteria import MatchAgentWorkRelations, MatchByAttrs, MatchByManyToOne -from share.models.base import ShareObject, ShareObjectVersion, TypedShareObjectMeta -from share.models.fields import ShareForeignKey, ShareURLField, ShareManyToManyField - -from share.util import ModelGenerator - - -class AbstractAgentWorkRelation(ShareObject, metaclass=TypedShareObjectMeta): - creative_work = ShareForeignKey('AbstractCreativeWork', related_name='agent_relations') - agent = ShareForeignKey('AbstractAgent', related_name='work_relations') - - cited_as = models.TextField(blank=True) - - matching_criteria = [ - MatchByManyToOne('creative_work', 'agent', constrain_types=True), - MatchAgentWorkRelations(), - ] - - class Meta(ShareObject.Meta): - db_table = 'share_agentworkrelation' - unique_together = ('agent', 'creative_work', 'type') - - -class ThroughContributor(ShareObject): - subject = ShareForeignKey(AbstractAgentWorkRelation, related_name='+') - related = ShareForeignKey(AbstractAgentWorkRelation, related_name='+') - - def clean(self): - if self.subject.creative_work != self.related.creative_work: - raise ValidationError(_('ThroughContributors must contribute to the same AbstractCreativeWork')) - if self.subject.agent == self.related.agent: - raise ValidationError(_('A contributor may not contribute through itself')) - - def save(self, *args, **kwargs): - self.clean() - return super().save(*args, **kwargs) - - matching_criteria = MatchByManyToOne('subject', 'related') - - class Meta(ShareObject.Meta): - unique_together = ('subject', 'related') - - -class Award(ShareObject): - # ScholarlyArticle has an award object - # it's just a text field, I assume our 'description' covers it. - name = models.TextField(blank=True) - description = models.TextField(blank=True) - date = models.DateTimeField(blank=True, null=True) - award_amount = models.PositiveIntegerField(blank=True, null=True) - uri = ShareURLField(unique=True, blank=True, null=True) - - def __str__(self): - return self.description - - matching_criteria = MatchByAttrs('uri') - - -class ThroughAwards(ShareObject): - funder = ShareForeignKey(AbstractAgentWorkRelation) - award = ShareForeignKey(Award) - - class Meta(ShareObject.Meta): - unique_together = ('funder', 'award') - verbose_name_plural = 'through awards' - - matching_criteria = MatchByManyToOne('funder', 'award') - - -generator = ModelGenerator(field_types={ - 'm2m': ShareManyToManyField, - 'positive_int': models.PositiveIntegerField -}) -globals().update(generator.subclasses_from_yaml(__file__, AbstractAgentWorkRelation)) - -__all__ = tuple(key for key, value in globals().items() if isinstance(value, type) and issubclass(value, (ShareObject, ShareObjectVersion))) diff --git a/share/models/relations/agentwork.yaml b/share/models/relations/agentwork.yaml deleted file mode 100644 index 47bb86dd3..000000000 --- a/share/models/relations/agentwork.yaml +++ /dev/null @@ -1,35 +0,0 @@ -AgentWorkRelation: - children: - Contributor: - description: The agent directly contributed to the creation of the work. - fields: - contributed_through: - type: m2m - args: [AbstractAgentWorkRelation] - kwargs: - symmetrical: false - through: ThroughContributor - through_fields: ['subject', 'related'] - children: - Creator: - description: The agent authored or was a major contributor to the creation of the work. - fields: - order_cited: - type: positive_int - PrincipalInvestigator: - description: The agent oversaw the creation of the work, as head of the laboratory or research group in which the work was created. - children: - PrincipalInvestigatorContact: - description: The agent oversaw the creation of the work, as head of the laboratory or research group in which the work was created and is the primary contact. - Funder: - description: The agent gave money that enabled or supported the creation of the work. - fields: - awards: - type: m2m - args: [Award] - kwargs: - through: ThroughAwards - Publisher: - description: The agent published the work. - Host: - description: The agent stores the completed work for public retrieval. diff --git a/share/models/relations/creative.py b/share/models/relations/creative.py deleted file mode 100644 index 0f5e62428..000000000 --- a/share/models/relations/creative.py +++ /dev/null @@ -1,23 +0,0 @@ -from share.disambiguation.criteria import MatchByManyToOne -from share.models.base import ShareObject, TypedShareObjectMeta -from share.models.fields import ShareForeignKey - -from share.util import ModelGenerator - - -class AbstractWorkRelation(ShareObject, metaclass=TypedShareObjectMeta): - subject = ShareForeignKey('AbstractCreativeWork', related_name='outgoing_creative_work_relations') - related = ShareForeignKey('AbstractCreativeWork', related_name='incoming_creative_work_relations') - - matching_criteria = MatchByManyToOne('subject', 'related', constrain_types=True) - - class Meta(ShareObject.Meta): - db_table = 'share_workrelation' - unique_together = ('subject', 'related', 'type') - - -generator = ModelGenerator() -globals().update(generator.subclasses_from_yaml(__file__, AbstractWorkRelation)) - - -__all__ = tuple(key for key, value in globals().items() if isinstance(value, type) and issubclass(value, ShareObject)) diff --git a/share/models/relations/creative.yaml b/share/models/relations/creative.yaml deleted file mode 100644 index 727fdd8b7..000000000 --- a/share/models/relations/creative.yaml +++ /dev/null @@ -1,139 +0,0 @@ -WorkRelation: - children: - IsPartOf: - verbose_name_plural: is part of - description: The subject work is a portion, component, or child of the related work. The subject work will be included in search results for the related work. - IsSupplementTo: - verbose_name_plural: is supplement to - description: The subject work supplements or enhances the related work. - IsDerivedFrom: - verbose_name_plural: is derived from - description: The subject work is derived from or based on the related work. - Cites: - description: The subject work cites the related work. - References: - description: The subject work uses the related work as a source of information. -# AgreesWith: -# description: The subject work agrees with statements, ideas or conclusions presented in the related work. -# uris: ['http://purl.org/spar/cito/agreesWith'] -# CitesAsAuthority: -# description: The subject work cites the related work as one that provides an authoritative description or definition of the subject under discussion. -# uris: ['http://purl.org/spar/cito/citesAsAuthority'] -# CitesAsDataSource: -# description: The subject work cites the related work as source of data. -# uris: ['http://purl.org/spar/cito/citesAsDataSource'] -# CitesAsEvidence: -# description: The subject work cites the related work as source of factual evidence for statements it contains. -# uris: ['http://purl.org/spar/cito/citesAsEvidence'] -# CitesAsMetadataDocument: -# description: The subject work cites the related work as being the container of metadata describing the subject work. -# uris: ['http://purl.org/spar/cito/citesAsMetadataDocument'] -# CitesAsPotentialSolution: -# description: The subject work cites the related work as providing or containing a possible solution to the issues being discussed. -# uris: ['http://purl.org/spar/cito/citesAsPotentialSolution'] -# CitesAsRecommendedReading: -# description: The subject work cites the related work as an item of recommended reading, e.g. in a lecture reading list or 'Suggested Further Reading' section. -# uris: ['http://purl.org/spar/cito/citesAsRecommendedReading'] -# CitesAsRelated: -# description: The subject work cites the related work as one that is related. -# uris: ['http://purl.org/spar/cito/citesAsRelated'] -# CitesAsSourceDocument: -# description: The subject work cites the related work as being the entity from which the subject work is derived, or about which the subject work contains metadata. -# uris: ['http://purl.org/spar/cito/citesAsSourceDocument'] -# CitesForInformation: -# description: The subject work cites the related work as a source of information on the subject under discussion. -# uris: ['http://purl.org/spar/cito/citesForInformation'] - Compiles: - description: The subject work is used to create or compile the related work. - uris: ['http://purl.org/spar/cito/compiles'] -# Confirms: -# description: The subject work confirms facts, ideas or statements presented in the related work. -# uris: ['http://purl.org/spar/cito/confirms'] -# ContainsAssertionFrom: -# description: The subject work contains a statement of fact or a logical assertion (or a collection of such facts and/or assertions) originally present in the related work. -# uris: ['http://purl.org/spar/cito/containsAssertionFrom'] - Corrects: - description: The subject work corrects statements, ideas or conclusions presented in the related work. - uris: ['http://purl.org/spar/cito/corrects'] -# Credits: -# description: The subject work acknowledges contributions made by the related work. -# uris: ['http://purl.org/spar/cito/credits'] -# Critiques: -# description: The subject work critiques statements, ideas or conclusions presented in the related work. -# uris: ['http://purl.org/spar/cito/critiques'] -# Derides: -# description: The subject work express derision for the related work, or for ideas or conclusions contained within it. -# uris: ['http://purl.org/spar/cito/derides'] -# Describes: -# description: The subject work describes the related work. -# uris: ['http://purl.org/spar/cito/describes'] -# DisagreesWith: -# description: The subject work disagrees with statements, ideas or conclusions presented in the related work. -# uris: ['http://purl.org/spar/cito/disagreesWith'] - Discusses: - description: The subject work discusses statements, ideas or conclusions presented in the related work. - uris: ['http://purl.org/spar/cito/discusses'] - Disputes: - description: The subject work disputes statements, ideas or conclusions presented in the related work. - uris: ['http://purl.org/spar/cito/disputes'] - Documents: - description: The subject work documents information about the related work. - uris: ['http://purl.org/spar/cito/documents'] - Extends: - description: The subject work extends facts, ideas or understandings presented in the related work. - uris: ['http://purl.org/spar/cito/extends'] -# IncludesExcerptFrom: -# description: The subject work includes one or more excerpts from the related work. -# uris: ['http://purl.org/spar/cito/includesExcerptFrom'] -# IncludesQuotationFrom: -# description: The subject work includes one or more quotations from the related work. -# uris: ['http://purl.org/spar/cito/includesQuotationFrom'] -# ObtainsBackgroundFrom: -# description: The subject work obtains background information from the related work. -# uris: ['http://purl.org/spar/cito/obtainsBackgroundFrom'] -# ObtainsSupportFrom: -# description: The subject work obtains intellectual or factual support from the related work. -# uris: ['http://purl.org/spar/cito/obtainsSupportFrom'] -# Parodies: -# description: The subject work imitates the characteristic style or content of the related work for comic effect, usually without explicit citation. -# uris: ['http://purl.org/spar/cito/parodies'] -# Plagiarizes: -# description: A property indicating that the author of the subject work plagiarizes the related work, by including textual or other elements from the related work without formal acknowledgement of their source. The subject work thus contains no explicit citation of the related work, according to the norms of scholarly practice, but cites it implicitly. -# uris: ['http://purl.org/spar/cito/plagiarizes'] -# Qualifies: -# description: The subject work qualifies or places conditions or restrictions upon statements, ideas or conclusions presented in the related work. -# uris: ['http://purl.org/spar/cito/qualifies'] -# Refutes: -# description: The subject work refutes statements, ideas or conclusions presented in the related work. -# uris: ['http://purl.org/spar/cito/refutes'] - RepliesTo: - verbose_name_plural: replies to - description: The subject work replies to statements, ideas or criticisms presented in the related work. - uris: ['http://purl.org/spar/cito/repliesTo'] - Retracts: - description: The subject work constitutes a formal retraction of the related work. The subject work will be included in search results for the related work. - uris: ['http://purl.org/spar/cito/retracts'] - Reviews: - description: The subject work reviews statements, ideas or conclusions presented in the related work. - uris: ['http://purl.org/spar/cito/reviews'] -# Ridicules: -# description: The subject work ridicules the related work or aspects of its contents. -# uris: ['http://purl.org/spar/cito/ridicules'] -# SpeculatesOn: -# description: The subject work speculates on something within or related to the related work, without firm evidence. -# uris: ['http://purl.org/spar/cito/speculatesOn'] -# Supports: -# description: The subject work provides intellectual or factual support for statements, ideas or conclusions presented in the related work. -# uris: ['http://purl.org/spar/cito/supports'] -# Updates: -# description: The subject work updates statements, ideas, hypotheses or understanding presented in the related work. -# uris: ['http://purl.org/spar/cito/updates'] -# UsesConclusionsFrom: -# description: The subject work describes work that uses conclusions presented in the related work. -# uris: ['http://purl.org/spar/cito/usesConclusionsFrom'] - UsesDataFrom: - description: The subject work describes work that uses data presented in the related work. - uris: ['http://purl.org/spar/cito/usesDataFrom'] -# UsesMethodIn: -# description: The subject work describes work that uses a method detailed in the related work. -# uris: ['http://purl.org/spar/cito/usesMethodIn'] diff --git a/share/models/sql.py b/share/models/sql.py index 9662fc21b..d4e723fc7 100644 --- a/share/models/sql.py +++ b/share/models/sql.py @@ -1,13 +1,5 @@ -from django.db import connections -from django.db.models.sql import InsertQuery -from django.db.models.sql.compiler import SQLInsertCompiler from django.db import models -from include import IncludeQuerySet - -from share.models.fuzzycount import FuzzyCountManager -from share.models.fuzzycount import FuzzyCountQuerySet - class GroupBy(models.Aggregate): template = '%(expressions)s' @@ -21,94 +13,3 @@ def __init__(self, expression, **extra): def get_group_by_cols(self): return self.source_expressions - - -class SQLInsertReturnVersionCompiler(SQLInsertCompiler): - - def as_sql(self): - # We don't need quote_name_unless_alias() here, since these are all - # going to be column names (so we can avoid the extra overhead). - qn = self.connection.ops.quote_name - opts = self.query.get_meta() - result = ['INSERT INTO %s' % qn(opts.db_table)] - - has_fields = bool(self.query.fields) - fields = self.query.fields if has_fields else [opts.pk] - result.append('(%s)' % ', '.join(qn(f.column) for f in fields)) - - if has_fields: - value_rows = [ - [self.prepare_value(field, self.pre_save_val(field, obj)) for field in fields] - for obj in self.query.objs - ] - else: - # An empty object. - value_rows = [[self.connection.ops.pk_default_value()] for _ in self.query.objs] - fields = [None] - - # Currently the backends just accept values when generating bulk - # queries and generate their own placeholders. Doing that isn't - # necessary and it should be possible to use placeholders and - # expressions in bulk inserts too. - can_bulk = (not self.return_id and self.connection.features.has_bulk_insert) - - placeholder_rows, param_rows = self.assemble_as_sql(fields, value_rows) - - if self.return_id and self.connection.features.can_return_id_from_insert: - params = param_rows[0] - result.append("VALUES (%s)" % ", ".join(placeholder_rows[0])) - - ### ACTUAL CHANGE HERE ### - result.append('RETURNING ("{0}"."{1}", "{0}"."{2}")'.format(opts.db_table, opts.pk.column, opts.get_field('version').column)) - ### /ACTUAL CHANGE HERE ### - - return [(" ".join(result), tuple(params))] - - if can_bulk: - result.append(self.connection.ops.bulk_insert_sql(fields, placeholder_rows)) - return [(" ".join(result), tuple(p for ps in param_rows for p in ps))] - else: - return [ - (" ".join(result + ["VALUES (%s)" % ", ".join(p)]), vals) - for p, vals in zip(placeholder_rows, param_rows) - ] - - -class InsertReturnVersionQuery(InsertQuery): - - def get_compiler(self, using=None, connection=None): - if using is None and connection is None: - raise ValueError("Need either using or connection") - if using: - connection = connections[using] - return SQLInsertReturnVersionCompiler(self, connection, using) - - -class InsertReturnVersionQuerySet(FuzzyCountQuerySet, IncludeQuerySet): - - def _insert(self, objs, fields, return_id=False, raw=False, using=None): - """ - Inserts a new record for the given model. This provides an interface to - the InsertQuery class and is how Model.save() is implemented. - """ - self._for_write = True - if using is None: - using = self.db - query = InsertReturnVersionQuery(self.model) - query.insert_values(fields, objs, raw=raw) - return query.get_compiler(using=using).execute_sql(return_id) - _insert.alters_data = True - _insert.queryset_only = False - - -class ShareObjectManager(FuzzyCountManager): - - def get_queryset(self): - return InsertReturnVersionQuerySet(self.model, using=self._db) - - -# ShareObjectManager = InsertReturnVersionQuerySet.as_manager() -# ShareObjectManager. - - -# class TypedShareObjectManager diff --git a/share/models/validators.py b/share/models/validators.py index 2f8c57703..026e44f99 100644 --- a/share/models/validators.py +++ b/share/models/validators.py @@ -24,7 +24,9 @@ def is_valid_jsonld(value): @deconstructible class JSONLDValidator: - __schema_cache = {} + __json_schema_cache = {} + __validator_cache = {} + with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'jsonld-schema.json')) as fobj: jsonld_schema = Draft4Validator(ujson.load(fobj)) @@ -124,9 +126,9 @@ def json_schema_for_field(self, share_field): return schema - def validator_for(self, share_schema_type): - if share_schema_type.name in JSONLDValidator.__schema_cache: - return JSONLDValidator.__schema_cache[share_schema_type.name] + def json_schema_for_type(self, share_schema_type): + if share_schema_type.name in JSONLDValidator.__json_schema_cache: + return JSONLDValidator.__json_schema_cache[share_schema_type.name] schema = { 'type': 'object', @@ -146,7 +148,18 @@ def validator_for(self, share_schema_type): schema['required'].append(share_field.name) schema['properties'][share_field.name] = self.json_schema_for_field(share_field) - return JSONLDValidator.__schema_cache.setdefault( + return JSONLDValidator.__json_schema_cache.setdefault( + share_schema_type.name, + schema, + ) + + def validator_for(self, share_schema_type): + if share_schema_type.name in JSONLDValidator.__validator_cache: + return JSONLDValidator.__validator_cache[share_schema_type.name] + + schema = self.json_schema_for_type(share_schema_type) + + return JSONLDValidator.__validator_cache.setdefault( share_schema_type.name, Draft4Validator(schema, format_checker=draft4_format_checker), ) diff --git a/share/oaipmh/legacy/renderers.py b/share/oaipmh/legacy/renderers.py deleted file mode 100644 index 89ec1b7b2..000000000 --- a/share/oaipmh/legacy/renderers.py +++ /dev/null @@ -1,212 +0,0 @@ -from datetime import datetime -from io import StringIO -from xml.etree.ElementTree import ElementTree, Element, SubElement - -from django.urls import reverse - -from share.models import Contributor -from share.oaipmh.util import format_datetime -from share.util.xml import strip_illegal_xml_chars - - -# For convenience -def SubEl(parent, tag, text=None, attrib={}): - element = SubElement(parent, tag, attrib) - if text: - element.text = strip_illegal_xml_chars(text) - return element - - -class OAIRenderer: - def __init__(self, repository, request): - self.repository = repository - self.request = request - self.kwargs = {} - - def identify(self, earliest_datestamp): - identify = Element('Identify') - SubEl(identify, 'repositoryName', self.repository.NAME) - SubEl(identify, 'baseURL', self.request.build_absolute_uri(reverse('oai-pmh'))) - SubEl(identify, 'protocolVersion', '2.0') - if earliest_datestamp: - SubEl(identify, 'earliestDatestamp', format_datetime(earliest_datestamp)) - SubEl(identify, 'deletedRecord', 'no') - SubEl(identify, 'granularity', self.repository.GRANULARITY) - for email in self.repository.ADMIN_EMAILS: - SubEl(identify, 'adminEmail', email) - - identifier = SubEl(SubEl(identify, 'description'), 'oai-identifier', attrib={ - 'xmlns': 'http://www.openarchives.org/OAI/2.0/oai-identifier', - 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', - 'xsi:schemaLocation': 'http://www.openarchives.org/OAI/2.0/oai-identifier http://www.openarchives.org/OAI/2.0/oai-identifier.xsd', - }) - SubEl(identifier, 'scheme', 'oai') - SubEl(identifier, 'repositoryIdentifier', self.repository.REPOSITORY_IDENTIFIER) - SubEl(identifier, 'delimiter', self.repository.IDENTIFER_DELIMITER) - SubEl(identifier, 'sampleIdentifier', self.repository.oai_identifier(1)) - - return self._render(identify) - - def listMetadataFormats(self, formats): - list_formats = Element('ListMetadataFormats') - for prefix, renderer in formats.items(): - format = SubEl(list_formats, 'metadataFormat') - SubEl(format, 'metadataPrefix', prefix) - SubEl(format, 'schema', renderer.schema) - SubEl(format, 'metadataNamespace', renderer.namespace) - return self._render(list_formats) - - def listSets(self, sets): - list_sets = Element('ListSets') - for spec, name in sets: - set = SubEl(list_sets, 'set') - SubEl(set, 'setSpec', spec) - SubEl(set, 'setName', name) - return self._render(list_sets) - - def listIdentifiers(self, works, next_token): - list_identifiers = Element('ListIdentifiers') - for work in works: - list_identifiers.append(self._header(work)) - SubEl(list_identifiers, 'resumptionToken', next_token) - return self._render(list_identifiers) - - def listRecords(self, works, next_token, metadataRenderer): - list_records = Element('ListRecords') - for work in works: - list_records.append(self._record(work, metadataRenderer)) - SubEl(list_records, 'resumptionToken', next_token) - return self._render(list_records) - - def getRecord(self, work, metadataRenderer): - get_record = Element('GetRecord') - get_record.append(self._record(work, metadataRenderer)) - return self._render(get_record) - - def errors(self, errors): - elements = [] - for error in errors: - element = Element('error', code=error.code) - element.text = error.description - elements.append(element) - return self._render(*elements) - - def _render(self, *elements): - root = Element('OAI-PMH', { - 'xmlns': 'http://www.openarchives.org/OAI/2.0/', - 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', - 'xsi:schemaLocation': 'http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd' - }) - SubEl(root, 'responseDate', format_datetime(datetime.now())) - request = SubEl(root, 'request', self.request.build_absolute_uri().rpartition('?')[0]) - verb = self.kwargs.pop('verb', None) - if verb: - request.set('verb', verb) - for k, v in self.kwargs.items(): - request.set(k, v) - - for element in elements: - root.append(element) - with StringIO() as stream: - ElementTree(root).write(stream, encoding='unicode', xml_declaration=True) - return stream.getvalue() - - def _header(self, work): - header = Element('header') - SubEl(header, 'identifier', self.repository.oai_identifier(work)) - SubEl(header, 'datestamp', format_datetime(work.date_modified)), - for spec in work.sources.all(): - SubEl(header, 'setSpec', spec.source.name) - return header - - def _record(self, work, metadataRenderer): - record = Element('record') - record.append(self._header(work)) - metadata = SubEl(record, 'metadata') - metadataRenderer.render_metadata(metadata, work) - # TODO SHARE-730 Add elements - return record - - -class MetadataRenderer: - def __init__(self, repository): - self.repository = repository - - @property - def prefix(self): - raise NotImplementedError() - - @property - def schema(self): - raise NotImplementedError() - - @property - def namespace(self): - raise NotImplementedError() - - def render_metadata(self, parent, work): - raise NotImplementedError() - - -class DublinCoreRenderer(MetadataRenderer): - schema = 'http://www.openarchives.org/OAI/2.0/oai_dc.xsd' - namespace = 'http://www.openarchives.org/OAI/2.0/oai_dc/' - - contributor_types = set(Contributor.get_types()) - set(['share.creator']) - - def render_metadata(self, parent, work): - dc = SubEl(parent, 'oai_dc:dc', attrib={ - 'xmlns:oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/', - 'xmlns:dc': 'http://purl.org/dc/elements/1.1/', - 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', - 'xsi:schemaLocation': 'http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd' - }) - if work.title: - SubEl(dc, 'dc:title', work.title) - - agent_relation_types = ['share.creator', 'share.publisher'] + list(self.contributor_types) - agent_relations = {k: [] for k in agent_relation_types} - - for relation in work.agent_relations.all(): - if relation.type not in agent_relations: - continue - agent_relations[relation.type].append(relation) - - for relation in sorted(agent_relations['share.creator'], key=lambda x: x.order_cited or -1): - SubEl(dc, 'dc:creator', relation.cited_as or relation.agent.name) - - for subject in work.subjects.all(): - SubEl(dc, 'dc:subject', subject.name) - - if work.description: - SubEl(dc, 'dc:description', work.description) - - for relation in agent_relations['share.publisher']: - SubEl(dc, 'dc:publisher', relation.cited_as or relation.agent.name) - - for relation in sorted([relation for type_ in self.contributor_types for relation in agent_relations[type_]], key=lambda x: x.order_cited or -1): - SubEl(dc, 'dc:contributor', relation.cited_as or relation.agent.name) - - date = work.date_published or work.date_updated - if date: - SubEl(dc, 'dc:date', format_datetime(date)) - - SubEl(dc, 'dc:type', work._meta.model_name) - - for identifier in work.identifiers.all(): - SubEl(dc, 'dc:identifier', identifier.uri) - - if work.language: - SubEl(dc, 'dc:language', work.language) - - for relation in work.incoming_creative_work_relations.all(): - if work.id == relation.subject_id: - SubEl(dc, 'dc:relation', self.repository.oai_identifier(relation.related_id)) - else: - SubEl(dc, 'dc:relation', self.repository.oai_identifier(relation.subject_id)) - - if work.rights: - SubEl(dc, 'dc:rights', work.rights) - - if work.free_to_read_type: - SubEl(dc, 'dc:rights', work.free_to_read_type) diff --git a/share/oaipmh/legacy/repository.py b/share/oaipmh/legacy/repository.py deleted file mode 100644 index 22c07be7e..000000000 --- a/share/oaipmh/legacy/repository.py +++ /dev/null @@ -1,251 +0,0 @@ -import dateutil - -from django.conf import settings -from django.db import connection - -from share.models import AbstractCreativeWork, Source, ShareUser -from share.oaipmh import errors as oai_errors -from share.oaipmh.verbs import OAIVerb -from share.oaipmh.legacy.renderers import OAIRenderer, DublinCoreRenderer -from share.oaipmh.util import format_datetime -from share.util import IDObfuscator, InvalidID - - -class OAIRepository: - NAME = 'SHARE' - REPOSITORY_IDENTIFIER = 'share.osf.io' - IDENTIFER_DELIMITER = ':' - GRANULARITY = 'YYYY-MM-DDThh:mm:ssZ' - ADMIN_EMAILS = ['share-support@osf.io'] - FORMATS = { - 'oai_dc': DublinCoreRenderer, - } - PAGE_SIZE = 20 - - # extracted from share/search/fetchers.py - VALID_IDS_QUERY = ''' - SELECT id - FROM share_creativework - WHERE id IN %(ids)s - AND title != '' - AND ( - SELECT COUNT(*) FROM ( - SELECT * FROM share_workidentifier - WHERE share_workidentifier.creative_work_id = share_creativework.id - LIMIT %(max_identifiers)s + 1 - ) AS identifiers - ) <= %(max_identifiers)s - AND ( - SELECT COUNT(*) FROM ( - SELECT * FROM share_agentworkrelation - WHERE share_agentworkrelation.creative_work_id = share_creativework.id - LIMIT %(max_agent_relations)s + 1 - ) AS agent_relations - ) <= %(max_agent_relations)s - ORDER BY id - ''' - - def handle_request(self, request, kwargs): - renderer = OAIRenderer(self, request) - verb, self.errors = OAIVerb.validate(**kwargs) - if not self.errors: - # No repeated arguments at this point - kwargs = {k: v[0] for k, v in kwargs.items()} - renderer.kwargs = kwargs - xml = getattr(self, '_do_{}'.format(verb.name.lower()))(kwargs, renderer) - - return xml if not self.errors else renderer.errors(self.errors) - - def resolve_oai_identifier(self, identifier): - try: - splid = identifier.split(self.IDENTIFER_DELIMITER) - if len(splid) != 3 or splid[:2] != ['oai', self.REPOSITORY_IDENTIFIER]: - raise InvalidID(identifier) - return IDObfuscator.resolve(splid[-1]) - except (AbstractCreativeWork.DoesNotExist, InvalidID): - self.errors.append(oai_errors.BadRecordID(identifier)) - return None - - def oai_identifier(self, work): - if isinstance(work, int): - share_id = IDObfuscator.encode_id(work, AbstractCreativeWork) - else: - share_id = IDObfuscator.encode(work) - return 'oai{delim}{repository}{delim}{id}'.format(id=share_id, repository=self.REPOSITORY_IDENTIFIER, delim=self.IDENTIFER_DELIMITER) - - def _do_identify(self, kwargs, renderer): - earliest = AbstractCreativeWork.objects.order_by('date_modified').values_list('date_modified', flat=True) - return renderer.identify(earliest[0] if earliest.exists() else None) - - def _do_listmetadataformats(self, kwargs, renderer): - if 'identifier' in kwargs: - self.resolve_oai_identifier(kwargs['identifier']) - return renderer.listMetadataFormats(self.FORMATS) - - def _do_listsets(self, kwargs, renderer): - if 'resumptionToken' in kwargs: - self.errors.append(oai_errors.BadResumptionToken(kwargs['resumptionToken'])) - return - return renderer.listSets(Source.objects.values_list('name', 'long_title')) - - def _do_listidentifiers(self, kwargs, renderer): - works, next_token, _ = self._load_page(kwargs) - if self.errors: - return - return renderer.listIdentifiers(works, next_token) - - def _do_listrecords(self, kwargs, renderer): - works, next_token, metadataRenderer = self._load_page(kwargs) - if self.errors: - return - return renderer.listRecords(works, next_token, metadataRenderer) - - def _do_getrecord(self, kwargs, renderer): - metadataRenderer = self._get_metadata_renderer(kwargs['metadataPrefix']) - work = self.resolve_oai_identifier(kwargs['identifier']) - if self.errors: - return - return renderer.getRecord(work, metadataRenderer) - - def _get_metadata_renderer(self, prefix, catch=True): - try: - return self.FORMATS[prefix](self) - except KeyError: - if not catch: - raise - self.errors.append(oai_errors.BadFormat(prefix)) - - def _load_page(self, kwargs): - if 'resumptionToken' in kwargs: - try: - work_ids, kwargs = self._resume(kwargs['resumptionToken']) - metadataRenderer = self._get_metadata_renderer(kwargs['metadataPrefix'], catch=False) - except (ValueError, KeyError): - self.errors.append(oai_errors.BadResumptionToken(kwargs['resumptionToken'])) - else: - work_ids = self._get_record_ids(kwargs) - metadataRenderer = self._get_metadata_renderer(kwargs['metadataPrefix']) - if self.errors: - return [], None, None - - works_queryset = AbstractCreativeWork.objects.filter( - id__in=work_ids, - ).include( - 'identifiers', - 'subjects', - 'sources__source', - 'incoming_creative_work_relations', - 'agent_relations__agent', - ).order_by('id') - - works = list(works_queryset) - - if not len(works): - self.errors.append(oai_errors.NoResults()) - return [], None, None - - if len(works) <= self.PAGE_SIZE: - # Last page - next_token = None - else: - works = works[:self.PAGE_SIZE] - next_token = self._get_resumption_token(kwargs, works[-1].id) - return works, next_token, metadataRenderer - - def _get_record_ids(self, kwargs, catch=True, last_id=None, page_size=None): - if page_size is None: - page_size = self.PAGE_SIZE - - queryset = AbstractCreativeWork.objects.filter( - is_deleted=False, - same_as_id__isnull=True, - ) - - if 'from' in kwargs: - try: - from_ = dateutil.parser.parse(kwargs['from']) - queryset = queryset.filter(date_modified__gte=from_) - except ValueError: - if not catch: - raise - self.errors.append(oai_errors.BadArgument('Invalid value for', 'from')) - if 'until' in kwargs: - try: - until = dateutil.parser.parse(kwargs['until']) - queryset = queryset.filter(date_modified__lte=until) - except ValueError: - if not catch: - raise - self.errors.append(oai_errors.BadArgument('Invalid value for', 'until')) - if 'set' in kwargs: - source_users = ShareUser.objects.filter(source__name=kwargs['set']).values_list('id', flat=True) - queryset = queryset.filter(sources__in=source_users) - - if last_id is not None: - queryset = queryset.filter(id__gt=last_id) - - queryset = queryset.order_by('id').values_list('id', flat=True) - - # get some extra, in case some are invalid - ids = tuple(queryset[:page_size + 10]) - - if not ids: - return [] - - # exclude untitled and franken works - # doing this in a separate query to avoid bad query plans with too much counting - with connection.cursor() as cursor: - cursor.execute( - self.VALID_IDS_QUERY, - { - 'ids': ids, - 'max_identifiers': settings.SHARE_LIMITS['MAX_IDENTIFIERS'], - 'max_agent_relations': settings.SHARE_LIMITS['MAX_AGENT_RELATIONS'], - }, - ) - valid_ids = [row[0] for row in cursor.fetchall()] - - # if there were invalid ids, get more to fill out the page - if len(ids) > page_size and len(valid_ids) <= page_size: - extra_ids = self._get_record_ids( - kwargs, - catch=catch, - last_id=ids[-1], - page_size=page_size - len(valid_ids) - ) - valid_ids.extend(extra_ids) - - return valid_ids[:page_size + 1] - - def _resume(self, token): - from_, until, set_spec, prefix, last_id = token.split('|') - kwargs = {} - if from_: - kwargs['from'] = from_ - if until: - kwargs['until'] = until - if set_spec: - kwargs['set'] = set_spec - kwargs['metadataPrefix'] = prefix - work_ids = self._get_record_ids(kwargs, catch=False, last_id=int(last_id)) - return work_ids, kwargs - - def _get_resumption_token(self, kwargs, last_id): - from_ = None - until = None - if 'from' in kwargs: - try: - from_ = dateutil.parser.parse(kwargs['from']) - except ValueError: - self.errors.append(oai_errors.BadArgument('Invalid value for', 'from')) - if 'until' in kwargs: - try: - until = dateutil.parser.parse(kwargs['until']) - except ValueError: - self.errors.append(oai_errors.BadArgument('Invalid value for', 'until')) - set_spec = kwargs.get('set', '') - return self._format_resumption_token(from_, until, set_spec, kwargs['metadataPrefix'], last_id) - - def _format_resumption_token(self, from_, until, set_spec, prefix, last_id): - # TODO something more opaque, maybe - return '{}|{}|{}|{}|{}'.format(format_datetime(from_) if from_ else '', format_datetime(until) if until else '', set_spec, prefix, last_id) diff --git a/share/oaipmh/views.py b/share/oaipmh/views.py index f37c81896..477bda294 100644 --- a/share/oaipmh/views.py +++ b/share/oaipmh/views.py @@ -1,10 +1,7 @@ -from django.conf import settings from django.views.generic.base import View from django.template.response import HttpResponse -from share.oaipmh.legacy.repository import OAIRepository as LegacyRepository from share.oaipmh.fmr_repository import OaiPmhRepository -from share.search.elastic_manager import ElasticManager class OAIPMHView(View): @@ -17,23 +14,6 @@ def post(self, request): return self.oai_response(**request.POST) def oai_response(self, **kwargs): - use_legacy_repository = self._should_use_legacy_repository(kwargs.pop('pls_trove', False)) - - if use_legacy_repository: - repository = LegacyRepository() - else: - repository = OaiPmhRepository() - + repository = OaiPmhRepository() xml = repository.handle_request(self.request, kwargs) return HttpResponse(xml, content_type=self.CONTENT_TYPE) - - def _should_use_legacy_repository(self, pls_trove): - if pls_trove or not settings.SHARE_LEGACY_PIPELINE: - return False - - # TEMPORARY HACK -- i mean it this time -- very specific to May 2021 - # check whether the primary elasticsearch alias points at the "old" or "new" index - primary_indexes = ElasticManager().get_primary_indexes() - - # old index name from settings.ELASTICSEARCH['INDEXES'] - return (primary_indexes == ['share_customtax_1']) diff --git a/share/regulate/steps/deduplicate.py b/share/regulate/steps/deduplicate.py index ac8613f77..65b4f85a6 100644 --- a/share/regulate/steps/deduplicate.py +++ b/share/regulate/steps/deduplicate.py @@ -1,5 +1,3 @@ -from share.disambiguation.matcher import Matcher -from share.disambiguation.strategies import GraphStrategy from share.regulate.steps import GraphStep @@ -14,7 +12,26 @@ class Deduplicate(GraphStep): """ MAX_MERGES = 100 + # map from concrete type to set of fields used to dedupe + DEDUPLICATION_CRITERIA = { + # works and agents may be merged if duplicate identifiers are merged + # 'abstractcreativework': {}, + # 'abstractagent': {}, + 'abstractagentworkrelation': {'creative_work', 'agent', 'type'}, + 'abstractagentrelation': {'subject', 'related', 'type'}, + 'abstractworkrelation': {'subject', 'related', 'type'}, + 'workidentifier': {'uri'}, + 'agentidentifier': {'uri'}, + 'subject': {'name', 'parent', 'central_synonym'}, + 'tag': {'name'}, + 'throughtags': {'tag', 'creative_work'}, + # 'award': {}, + 'throughawards': {'funder', 'award'}, + } + def regulate_graph(self, graph): + # naive algorithm, O(n*m) (n: number of nodes, m: number of merges) + # but merges shouldn't be common, so probably not worth optimizing count = 0 while self._merge_first_dupe(graph): count += 1 @@ -23,11 +40,30 @@ def regulate_graph(self, graph): return def _merge_first_dupe(self, graph): - matcher = Matcher(GraphStrategy(graph)) - for matches in matcher.chunk_matches(graph): - for node, dupes in matches.items(): - for dupe in dupes: - if dupe in graph and node in graph: - graph.merge_nodes(dupe, node) - return True + dupe_index = {} + for node in graph: + node_key = self._get_node_key(node) + if node_key: + other_node = dupe_index.get(node_key) + if other_node: + graph.merge_nodes(node, other_node) + return True + dupe_index[node_key] = node return False + + def _get_node_key(self, node): + criteria = self.DEDUPLICATION_CRITERIA.get(node.concrete_type) + if not criteria: + return None + return ( + node.concrete_type, + tuple( + self._get_criterion_value(node, criterion) + for criterion in criteria + ) + ) + + def _get_criterion_value(self, node, criterion_name): + if criterion_name == 'type': + return node.type + return node[criterion_name] diff --git a/share/search/__init__.py b/share/search/__init__.py index e76a4beed..8e1963222 100644 --- a/share/search/__init__.py +++ b/share/search/__init__.py @@ -1,10 +1,6 @@ import logging from contextlib import ExitStack -from django.apps import apps -from django.db.models import Q - -from share.models import AbstractCreativeWork from share.search.elastic_manager import ElasticManager from share.search.messages import MessageType @@ -87,62 +83,3 @@ def handle_messages_sync(self, message_type, target_ids, index_names=None): ] self.elastic_manager.send_actions_sync(elastic_actions) self.elastic_manager.refresh_indexes(index_names) - - # consider the below deprecated -- should be removed along with ShareObject - def index(self, model_name, *pks, index=None, urgent=False): - message_type = { - 'agent': MessageType.INDEX_AGENT, - 'creativework': MessageType.INDEX_CREATIVEWORK, - 'subject': MessageType.INDEX_SUBJECT, - 'tag': MessageType.INDEX_TAG, - }[model_name.lower()] - - if not pks: - return - - model = apps.get_model('share', model_name) - pks = self.pks_to_reindex(model, pks) - - self.send_messages( - message_type=message_type, - target_ids=pks, - urgent=urgent, - index_names=[index] if index else None, - ) - - def pks_to_reindex(self, model, pks): - """Get all pks that should be reindexed if the objects with the given ids were updated. - - The indexed payload may include related objects, which we don't want to get stale. - """ - pks = set(pks) - if model is AbstractCreativeWork: - # Reindex children/gchildren/ggchildren of any affected works - parent_relation = 'share.ispartof' - children = model.objects.filter(( - Q( - outgoing_creative_work_relations__type=parent_relation, - outgoing_creative_work_relations__related_id__in=pks - ) | Q( - outgoing_creative_work_relations__type=parent_relation, - outgoing_creative_work_relations__related__outgoing_creative_work_relations__type=parent_relation, - outgoing_creative_work_relations__related__outgoing_creative_work_relations__related_id__in=pks - ) | Q( - outgoing_creative_work_relations__type=parent_relation, - outgoing_creative_work_relations__related__outgoing_creative_work_relations__type=parent_relation, - outgoing_creative_work_relations__related__outgoing_creative_work_relations__related__outgoing_creative_work_relations__type=parent_relation, - outgoing_creative_work_relations__related__outgoing_creative_work_relations__related__outgoing_creative_work_relations__related_id__in=pks - )), - is_deleted=False - ).values_list('id', flat=True) - - # Reindex works retracted by any affected works - retraction_relation = 'share.retracts' - retracted = model.objects.filter( - incoming_creative_work_relations__type=retraction_relation, - incoming_creative_work_relations__subject_id__in=pks, - is_deleted=False - ).values_list('id', flat=True) - - return pks.union(children, retracted) - return pks diff --git a/share/search/fetchers.py b/share/search/fetchers.py deleted file mode 100644 index 5f4054774..000000000 --- a/share/search/fetchers.py +++ /dev/null @@ -1,573 +0,0 @@ -import sys -import bleach -import logging -import time - -# from stevedore.extension import ExtensionManager - -from django.apps import apps -from django.conf import settings -from django.db import connection - -from project.settings import ALLOWED_TAGS - -from share import models -from share.util import IDObfuscator - - -logger = logging.getLogger(__name__) - - -class Fetcher: - """ - All Queries must return 2 columns, the pk of the result and the _source that will be sent to elasticsearch, - formatted/named as (id, _source). To filter your selections by the ids given to the fetcher use: - `WHERE table_name.id IN (SELECT id FROM pks)` - """ - - QUERY = None - QUERY_TEMPLATE = ''' - WITH pks AS (SELECT * FROM UNNEST(%(ids)s::int[]) WITH ORDINALITY t(id, ord)), - results AS ({0}) - SELECT _source FROM results - RIGHT OUTER JOIN pks ON pks.id = results.id - ORDER BY pks.ord; - ''' - - DEFAULT_FETCHERS = { - 'agent': 'share.search.fetchers.AgentFetcher', - 'creativework': 'share.search.fetchers.CreativeWorkFetcher', - 'subject': 'share.search.fetchers.SubjectFetcher', - 'tag': 'share.search.fetchers.TagFetcher', - } - - @classmethod - def fetcher_for(cls, model, overrides=None): - if not model._meta.concrete_model.__subclasses__(): - model_name = model._meta.model_name.lower() - else: - model_name = model._meta.concrete_model.__subclasses__()[0]._meta.model_name.lower() - try: - fetcher_path = (overrides or {}).get(model_name) or cls.DEFAULT_FETCHERS[model_name] - except KeyError: - raise ValueError('No fetcher exists for {!r}'.format(model)) - module, _, name = fetcher_path.rpartition('.') - __import__(module) - return getattr(sys.modules[module], name)() - - def __call__(self, pks): - if self.QUERY is None: - raise NotImplementedError - - pks = tuple(pks) - if not pks: - return [] - - if connection.connection is None: - connection.cursor() - - with connection.connection.cursor() as c: - start = time.time() * 1000 - logger.debug('Fetching %d rows using %r', len(pks), self) - c.execute(self.QUERY_TEMPLATE.format(self.QUERY), self.query_parameters(pks)) - logger.debug('Main query execution of %r took %dms', self, time.time() * 1000 - start) - - while True: - data = c.fetchone() - - if not data: - logger.debug('Entire fetching processes of %r took %dms', self, time.time() * 1000 - start) - return - - if data[0] is None: - yield None - else: - yield self.post_process(data[0]) - - def post_process(self, data): - return self.populate_types(data) - - def populate_types(self, data): - model = apps.get_model(data['type']) - data['id'] = IDObfuscator.encode_id(data['id'], model) - data['type'] = model._meta.verbose_name - data['types'] = [] - for parent in model.__mro__: - if not parent._meta.proxy: - break - data['types'].append(parent._meta.verbose_name) - - return data - - def query_parameters(self, pks): - return {'ids': '{' + ','.join(str(pk) for pk in pks) + '}'} - - -# For ease of use -fetcher_for = Fetcher.fetcher_for - - -class CreativeWorkFetcher(Fetcher): - SUBJECT_DELIMITER = '|' - RETRACTION_RELATION = 'share.retracts' - PARENT_RELATION = 'share.ispartof' - WORK_ANCESTORS_DEPTH = 3 - MAX_IDENTIFIERS = 50 - MAX_AGENT_RELATIONS = 300 - - QUERY_TEMPLATE = ''' - WITH pks AS (SELECT * FROM UNNEST(%(ids)s::int[]) WITH ORDINALITY t(id, ord)), - {0} - SELECT _source FROM results - RIGHT OUTER JOIN pks ON pks.id = results.id - ORDER BY pks.ord; - ''' - - QUERY = ( - # Gather all the tags in one query, for use below - ''' - all_tags AS ( - SELECT - creative_work_id AS creativework_id, - array_agg(share_tag.name) AS tags - FROM share_tag - JOIN share_throughtags ON share_tag.id = share_throughtags.tag_id - WHERE share_throughtags.creative_work_id IN (SELECT id FROM pks) - GROUP BY creative_work_id - ), - ''' - - # Gather all the sources in one query, for use below - ''' - all_sources AS ( - SELECT - abstractcreativework_id AS creativework_id, - array_agg(DISTINCT share_source.long_title) AS sources - FROM share_creativework_sources - JOIN share_shareuser ON share_creativework_sources.shareuser_id = share_shareuser.id - JOIN share_source ON share_shareuser.id = share_source.user_id - WHERE share_creativework_sources.abstractcreativework_id IN (SELECT id FROM pks) - AND NOT share_source.is_deleted - GROUP BY abstractcreativework_id - ), - ''' - - # Gather all the agents in one query, for use below - ''' - all_related_agents AS ( - SELECT - agent_relation.creative_work_id AS creativework_id, - json_agg(json_strip_nulls(json_build_object( - 'id', agent.id - , 'type', agent.type - , 'name', agent.name - , 'given_name', agent.given_name - , 'family_name', agent.family_name - , 'additional_name', agent.additional_name - , 'suffix', agent.suffix - , 'identifiers', COALESCE(identifiers, '{}') - , 'relation_type', agent_relation.type - , 'order_cited', agent_relation.order_cited - , 'cited_as', agent_relation.cited_as - , 'affiliations', COALESCE(affiliations, '[]'::json) - , 'awards', COALESCE(awards, '[]'::json) - ))) AS related_agents - FROM share_agentworkrelation AS agent_relation - JOIN share_agent AS agent ON agent_relation.agent_id = agent.id - - LEFT JOIN LATERAL ( - SELECT - array_agg(identifier.uri) AS identifiers - FROM share_agentidentifier AS identifier - WHERE identifier.agent_id = agent.id - AND identifier.scheme != 'mailto' - LIMIT 51 - ) AS identifiers ON TRUE - - LEFT JOIN LATERAL ( - SELECT json_agg(json_strip_nulls(json_build_object( - 'id', affiliated_agent.id - , 'type', affiliated_agent.type - , 'name', affiliated_agent.name - , 'affiliation_type', affiliation.type - ))) AS affiliations - FROM share_agentrelation AS affiliation - JOIN share_agent AS affiliated_agent ON affiliation.related_id = affiliated_agent.id - WHERE affiliation.subject_id = agent.id AND affiliated_agent.type != 'share.person' - ) AS affiliations ON (agent.type = 'share.person') - - LEFT JOIN LATERAL ( - SELECT json_agg(json_strip_nulls(json_build_object( - 'id', award.id - , 'type', 'share.award' - , 'date', award.date - , 'name', award.name - , 'description', award.description - , 'uri', award.uri - , 'amount', award.award_amount - ))) AS awards - FROM share_throughawards AS throughaward - JOIN share_award AS award ON throughaward.award_id = award.id - WHERE throughaward.funder_id = agent_relation.id - ) AS awards ON agent_relation.type = 'share.funder' - - WHERE agent_relation.creative_work_id IN (SELECT id FROM pks) - GROUP BY agent_relation.creative_work_id - ), - ''' - - # Gather all the works we want, so postgres doesn't get confused by the huge query below - # Exclude works with empty titles, too many identifiers, or too many agent relations - ''' - all_creative_works AS ( - SELECT * - FROM share_creativework AS all_creative_works - WHERE id IN (SELECT id FROM pks) - AND title != '' - AND ( - SELECT COUNT(*) FROM ( - SELECT * FROM share_workidentifier - WHERE share_workidentifier.creative_work_id = all_creative_works.id - LIMIT %(max_identifiers)s + 1 - ) AS identifiers - ) <= %(max_identifiers)s - AND ( - SELECT COUNT(*) FROM ( - SELECT * FROM share_agentworkrelation - WHERE share_agentworkrelation.creative_work_id = all_creative_works.id - LIMIT %(max_agent_relations)s + 1 - ) AS agent_relations - ) <= %(max_agent_relations)s - ), - ''' - - # For each work, construct the JSON that (after post-processing) will be sent to elasticsearch - ''' - results AS ( - SELECT creativework.id, json_build_object( - 'id', creativework.id - , 'type', creativework.type - , 'title', creativework.title - , 'description', creativework.description - , 'is_deleted', creativework.is_deleted - , 'language', creativework.language - , 'date_created', creativework.date_created - , 'date_modified', creativework.date_modified - , 'date_updated', creativework.date_updated - , 'date_published', creativework.date_published - , 'registration_type', creativework.registration_type - , 'withdrawn', creativework.withdrawn - , 'justification', creativework.justification - , 'tags', COALESCE(tags, '{}') - , 'identifiers', COALESCE(identifiers, '{}') - , 'sources', COALESCE(sources, '{}') - , 'subjects', COALESCE(subjects.original, '{}') - , 'subject_synonyms', COALESCE(subjects.synonyms, '{}') - , 'related_agents', COALESCE(related_agents, '{}') - , 'retractions', COALESCE(retractions, '{}') - , 'lineage', COALESCE(lineage, '{}') - ) AS _source - FROM all_creative_works AS creativework - - LEFT JOIN all_sources ON all_sources.creativework_id = creativework.id - - LEFT JOIN all_tags ON all_tags.creativework_id = creativework.id - - LEFT JOIN all_related_agents ON all_related_agents.creativework_id = creativework.id - - LEFT JOIN LATERAL ( - SELECT array_agg(identifier.uri) AS identifiers - FROM share_workidentifier AS identifier - WHERE identifier.creative_work_id = creativework.id - ) AS identifiers ON TRUE - ''' - - # Get the full subject lineage for each subject on the work (subjects.original), - # as well as their synonyms in the central taxonomy (subjects.synonyms) - ''' - LEFT JOIN LATERAL ( - WITH RECURSIVE subject_names(synonym, taxonomy, parent_id, path) AS ( - SELECT - FALSE, - CASE WHEN source.name = %(system_user)s THEN %(central_taxonomy)s ELSE source.long_title END, - parent_id, - share_subject.name - FROM share_throughsubjects - JOIN share_subject ON share_subject.id = share_throughsubjects.subject_id - JOIN share_subjecttaxonomy AS taxonomy ON share_subject.taxonomy_id = taxonomy.id - JOIN share_source AS source ON taxonomy.source_id = source.id - WHERE share_throughsubjects.creative_work_id = creativework.id - AND NOT share_throughsubjects.is_deleted - UNION ALL - SELECT - TRUE, %(central_taxonomy)s, central.parent_id, central.name - FROM share_throughsubjects - JOIN share_subject ON share_subject.id = share_throughsubjects.subject_id - JOIN share_subject AS central ON central.id = share_subject.central_synonym_id - WHERE share_throughsubjects.creative_work_id = creativework.id - AND NOT share_throughsubjects.is_deleted - - UNION ALL - SELECT - synonym, - child.taxonomy, - parent.parent_id, - concat_ws(%(subject_delimiter)s, parent.name, path) - FROM subject_names AS child - INNER JOIN share_subject AS parent ON child.parent_id = parent.id - ) SELECT - ( - SELECT array_agg(DISTINCT concat_ws(%(subject_delimiter)s, taxonomy, path)) - FROM subject_names - WHERE NOT synonym AND parent_id IS NULL - ) AS original, - ( - SELECT array_agg(DISTINCT concat_ws(%(subject_delimiter)s, taxonomy, path)) - FROM subject_names - WHERE synonym AND parent_id IS NULL - ) AS synonyms - ) AS subjects ON TRUE - ''' - - # Find the work's parent/grandparent/great-grandparent - ''' - LEFT JOIN LATERAL ( - WITH RECURSIVE ancestors(depth, id, type, title, identifiers) AS ( - VALUES (0, creativework.id, NULL, NULL, NULL::text[]) - UNION ( - SELECT - child.depth + 1, parent.id, parent.type, parent.title, COALESCE(parent_identifiers.identifiers, '{}') - FROM ancestors AS child - JOIN share_workrelation AS work_relation ON child.id = work_relation.subject_id - JOIN share_creativework AS parent ON work_relation.related_id = parent.id - LEFT JOIN LATERAL ( - SELECT array_agg(identifier.uri) AS identifiers - FROM share_workidentifier AS identifier - WHERE identifier.creative_work_id = parent.id - ) AS parent_identifiers ON TRUE - WHERE work_relation.type = %(parent_relation)s - AND child.depth < %(work_ancestors_depth)s - AND NOT parent.is_deleted - LIMIT 1 - ) - ) - SELECT json_agg(json_strip_nulls(json_build_object( - 'id', id - , 'type', type - , 'title', title - , 'identifiers', identifiers - ))) AS lineage - FROM ancestors - WHERE depth > 0 - ) AS lineage ON TRUE - ''' - - # Find works that retract this work - ''' - LEFT JOIN LATERAL ( - SELECT json_agg(json_strip_nulls(json_build_object( - 'id', retraction.id - , 'type', retraction.type - , 'title', retraction.title - , 'description', retraction.description - , 'date_created', retraction.date_created - , 'date_modified', retraction.date_modified - , 'date_updated', retraction.date_updated - , 'date_published', retraction.date_published - , 'identifiers', COALESCE(identifiers, '{}') - ))) AS retractions - FROM share_workrelation AS work_relation - JOIN share_creativework AS retraction ON work_relation.subject_id = retraction.id - LEFT JOIN LATERAL ( - SELECT array_agg(identifier.uri) AS identifiers - FROM share_workidentifier AS identifier - WHERE identifier.creative_work_id = retraction.id - ) AS identifiers ON TRUE - WHERE work_relation.related_id = creativework.id - AND work_relation.type = %(retraction_relation)s - AND NOT retraction.is_deleted - ) AS retractions ON TRUE - ) - ''' - ) - - def query_parameters(self, pks): - return { - **super().query_parameters(pks), - 'central_taxonomy': settings.SUBJECTS_CENTRAL_TAXONOMY, - 'subject_delimiter': self.SUBJECT_DELIMITER, - 'system_user': settings.APPLICATION_USERNAME, - 'retraction_relation': self.RETRACTION_RELATION, - 'parent_relation': self.PARENT_RELATION, - 'work_ancestors_depth': self.WORK_ANCESTORS_DEPTH, - 'max_identifiers': settings.SHARE_LIMITS['MAX_IDENTIFIERS'], - 'max_agent_relations': settings.SHARE_LIMITS['MAX_AGENT_RELATIONS'], - } - - def post_process(self, data): - data['lists'] = {} - - if data['title']: - data['title'] = bleach.clean(data['title'], strip=True, tags=ALLOWED_TAGS) - - if data['description']: - data['description'] = bleach.clean(data['description'], strip=True, tags=ALLOWED_TAGS) - - # Related works will end up with 0 identifiers and end up clogging search with - # many broken-looking works. Rather than filtering them out, we're setting them to - # deleted to clean out any stragglers. - if not data['identifiers']: - data['is_deleted'] = True - - for agent in data.pop('related_agents'): - try: - # We have to try except this. Out of desperation to fix a problem - # some types got changed to random strings to dodge unique contraints - self.populate_types(agent) - except ValueError: - continue - - for award in agent.get('awards', []): - self.populate_types(award) - - for affiliation in agent.get('affiliations', []): - try: - # We have to try except this. Out of desperation to fix a problem - # some types got changed to random strings to dodge unique contraints - self.populate_types(affiliation) - affiliation['affiliation'] = apps.get_model(affiliation.pop('affiliation_type'))._meta.verbose_name - except ValueError: - continue - - try: - # We have to try except this. Out of desperation to fix a problem - # some types got changed to random strings to dodge unique contraints - relation_model = apps.get_model(agent.pop('relation_type')) - except ValueError: - pass - - parent_model = next(parent for parent in relation_model.__mro__ if not parent.__mro__[2]._meta.proxy) - parent_name = str(parent_model._meta.verbose_name_plural) - agent['relation'] = relation_model._meta.verbose_name - data['lists'].setdefault(parent_name, []).append(agent) - - if relation_model == models.AgentWorkRelation: - elastic_field = 'affiliations' - else: - elastic_field = parent_name - data.setdefault(elastic_field, []).append(agent.get('cited_as') or agent['name']) - - if parent_model == models.Contributor: - data.setdefault('affiliations', []).extend(a['name'] for a in agent['affiliations']) - - data['retracted'] = bool(data['retractions']) - for retraction in data.pop('retractions'): - self.populate_types(retraction) - data['lists'].setdefault('retractions', []).append(retraction) - - lineage = [] - for ancestor in data.pop('lineage'): - self.populate_types(ancestor) - lineage.insert(0, ancestor) - if lineage: - data['lists']['lineage'] = lineage - - data['date'] = (data['date_published'] or data['date_updated'] or data['date_created']) - - return super().post_process(data) - - -class CreativeWorkShortSubjectsFetcher(CreativeWorkFetcher): - def post_process(self, data): - subjects = set() - for subject in data['subjects']: - taxonomy, *lineage = subject.split(self.SUBJECT_DELIMITER) - if taxonomy == settings.SUBJECTS_CENTRAL_TAXONOMY: - subjects.update(lineage) - data['subjects'] = list(subjects) - - del data['subject_synonyms'] - - return super().post_process(data) - - -class AgentFetcher(Fetcher): - - QUERY = ''' - SELECT agent.id, json_strip_nulls(json_build_object( - 'id', agent.id - , 'type', agent.type - , 'name', agent.name - , 'family_name', agent.family_name - , 'given_name', agent.given_name - , 'additional_name', agent.additional_name - , 'suffix', agent.suffix - , 'location', agent.location - , 'sources', COALESCE(sources, '{}') - , 'identifiers', COALESCE(identifiers, '{}') - , 'related_types', COALESCE(related_types, '{}'))) AS _source - FROM share_agent AS agent - LEFT JOIN LATERAL ( - SELECT array_agg(DISTINCT source.long_title) AS sources - FROM share_agent_sources AS throughsources - JOIN share_shareuser AS shareuser ON throughsources.shareuser_id = shareuser.id - JOIN share_source AS source ON shareuser.id = source.user_id - WHERE throughsources.abstractagent_id = agent.id - AND NOT source.is_deleted - ) AS sources ON TRUE - LEFT JOIN LATERAL ( - SELECT array_agg(identifier.uri) AS identifiers - FROM share_agentidentifier AS identifier - WHERE identifier.agent_id = agent.id - AND identifier.scheme != 'mailto' - ) AS identifiers ON TRUE - LEFT JOIN LATERAL ( - SELECT array_agg(DISTINCT creative_work_relation.type) AS related_types - FROM share_agentworkrelation AS creative_work_relation - WHERE creative_work_relation.agent_id = agent.id - ) AS related_types ON TRUE - WHERE agent.id IN (SELECT id FROM pks) - ''' - - def post_process(self, data): - data = super().post_process(data) - - for rtype in data.pop('related_types'): - try: - # We have to try except this. Out of desperation to fix a problem - # some types got changed to random strings to dodge unique contraints - klass = apps.get_model(rtype) - except ValueError: - continue - - for relation_model in klass.__mro__: - if not relation_model.__mro__[1]._meta.proxy: - break - data['types'].append(relation_model._meta.verbose_name) - data['types'] = list(set(data['types'])) - - return data - - -class SubjectFetcher(Fetcher): - QUERY = ''' - SELECT subject.id, json_strip_nulls(json_build_object('id', subject.id , 'name', subject.name)) AS _source - FROM share_subject AS subject - WHERE subject.id IN (SELECT id FROM pks) - AND length(subject.name) < 2001 - ''' - - def post_process(self, data): - return {'id': IDObfuscator.encode_id(data['id'], models.Subject), 'type': 'subject', 'name': data['name']} - - -class TagFetcher(Fetcher): - QUERY = ''' - SELECT tag.id, json_strip_nulls(json_build_object('id', tag.id , 'name', tag.name)) AS _source - FROM share_tag AS tag - WHERE tag.id IN (SELECT id FROM pks) - AND length(tag.name) < 2001 - ''' - - def post_process(self, data): - return {'id': IDObfuscator.encode_id(data['id'], models.Tag), 'type': 'tag', 'name': data['name']} diff --git a/share/search/index_setup/__init__.py b/share/search/index_setup/__init__.py index e33bc3496..6f85e6560 100644 --- a/share/search/index_setup/__init__.py +++ b/share/search/index_setup/__init__.py @@ -1,8 +1,8 @@ from .postrend_backcompat import PostRendBackcompatIndexSetup -from .share_classic import ShareClassicIndexSetup +from .trove_v0 import TroveV0IndexSetup __all__ = ( 'PostRendBackcompatIndexSetup', - 'ShareClassicIndexSetup', + 'TroveV0IndexSetup', ) diff --git a/share/search/index_setup/postrend_backcompat.py b/share/search/index_setup/postrend_backcompat.py index 693b08c29..f31dd9b1c 100644 --- a/share/search/index_setup/postrend_backcompat.py +++ b/share/search/index_setup/postrend_backcompat.py @@ -6,13 +6,14 @@ class PostRendBackcompatIndexSetup(IndexSetup): + SUBJECT_DELIMITER = '|' + @property def supported_message_types(self): return {MessageType.INDEX_SUID} @property def index_settings(self): - from share.search.fetchers import CreativeWorkFetcher return { 'analysis': { 'filter': { @@ -56,7 +57,7 @@ def index_settings(self): 'tokenizer': { 'subject_tokenizer': { 'type': 'path_hierarchy', - 'delimiter': CreativeWorkFetcher.SUBJECT_DELIMITER, + 'delimiter': self.SUBJECT_DELIMITER, } } } diff --git a/share/search/index_setup/share_classic.py b/share/search/index_setup/share_classic.py deleted file mode 100644 index bc273c01a..000000000 --- a/share/search/index_setup/share_classic.py +++ /dev/null @@ -1,62 +0,0 @@ -from share.models import Agent, CreativeWork, Tag, Subject - -from share.search.exceptions import IndexSetupError -from share.search.fetchers import fetcher_for -from share.search.index_setup.base import IndexSetup -from share.search.index_setup.postrend_backcompat import PostRendBackcompatIndexSetup -from share.search.messages import MessageType - - -# composes PostRendBackcompatIndexSetup so we can explicitly reuse settings/mappings -# and easily delete this as soon as we don't need it -class ShareClassicIndexSetup(IndexSetup): - def __init__(self): - self.backcompat_setup = PostRendBackcompatIndexSetup() - - @property - def supported_message_types(self): - return { - MessageType.INDEX_AGENT, - MessageType.INDEX_CREATIVEWORK, - MessageType.INDEX_TAG, - MessageType.INDEX_SUBJECT, - } - - @property - def index_settings(self): - return self.backcompat_setup.index_settings - - @property - def index_mappings(self): - return self.backcompat_setup.index_mappings - - def build_action_generator(self, index_name, message_type): - if message_type not in self.supported_message_types: - raise IndexSetupError(f'Invalid message_type "{message_type}" (expected {self.supported_message_types})') - - model, doc_type = self._get_model_and_doc_type(message_type) - fetcher = fetcher_for(model) - - action_template = { - '_index': index_name, - '_type': doc_type - } - - def action_generator(target_id_iter): - for target_id, result in zip(target_id_iter, fetcher(target_id_iter)): - if result is None: - action = None - elif result.pop('is_deleted', False): - action = {'_id': result['id'], '_op_type': 'delete', **action_template} - else: - action = {'_id': result['id'], '_op_type': 'index', **action_template, '_source': result} - yield (target_id, action) - return action_generator - - def _get_model_and_doc_type(self, message_type): - return { - MessageType.INDEX_AGENT: (Agent, 'agents'), - MessageType.INDEX_CREATIVEWORK: (CreativeWork, 'creativeworks'), - MessageType.INDEX_SUBJECT: (Subject, 'subjects'), - MessageType.INDEX_TAG: (Tag, 'tags'), - }[message_type] diff --git a/share/search/index_setup/trove_v0.py b/share/search/index_setup/trove_v0.py new file mode 100644 index 000000000..375258aa1 --- /dev/null +++ b/share/search/index_setup/trove_v0.py @@ -0,0 +1,19 @@ +from share.search.index_setup.base import IndexSetup + + +# just a placeholder for now +class TroveV0IndexSetup(IndexSetup): + @property + def supported_message_types(self): + return set() + + @property + def index_settings(self): + raise NotImplementedError + + @property + def index_mappings(self): + raise NotImplementedError + + def build_action_generator(self, index_name, message_type): + raise NotImplementedError diff --git a/share/shell_util.py b/share/shell_util.py index 210738cad..76c07dc43 100644 --- a/share/shell_util.py +++ b/share/shell_util.py @@ -1,35 +1,15 @@ -from django.contrib.contenttypes.models import ContentType +"""grab-bag of handy utilities that will be available in the django shell -from share import tasks # noqa -from share.models import RawDatum +that is, the shell you get from `python manage.py shell_plus` +""" + +from share import tasks from share.search import SearchIndexer from share.util import IDObfuscator -def reindex_works(works, index=None, urgent=True): - if not isinstance(works, list): - works = [works] - work_ids = [work.id for work in works] - if (work_ids): - print('Indexing {} works'.format(len(work_ids))) - indexer = SearchIndexer() - indexer.index('creativework', *work_ids, index=index, urgent=urgent) - else: - print('No works to index') - - -def get_raws(obj): - if isinstance(obj, str): - model, id = IDObfuscator.decode(obj) - else: - model = obj._meta.model - id = obj.id - return RawDatum.objects.filter( - normalizeddata__changeset__changes__target_id=id, - normalizeddata__changeset__changes__target_type=ContentType.objects.get_for_model(model, for_concrete_model=True) - ) - - -def print_raws(obj): - for raw in get_raws(obj): - print(raw.data) +__all__ = ( + 'tasks', + 'SearchIndexer', + 'IDObfuscator', +) diff --git a/share/tasks/jobs.py b/share/tasks/jobs.py index c2c0eaf1f..c07292c71 100644 --- a/share/tasks/jobs.py +++ b/share/tasks/jobs.py @@ -1,18 +1,13 @@ import logging import random -from django.apps import apps from django.conf import settings -from django.contrib.contenttypes.models import ContentType from django.db import transaction, IntegrityError -from django.db.utils import OperationalError from django.utils import timezone from share import exceptions from share.harvest.exceptions import HarvesterConcurrencyError -from share.ingest.change_builder import ChangeSetBuilder from share.models import ( - AbstractCreativeWork, HarvestJob, IngestJob, NormalizedData, @@ -24,7 +19,6 @@ from share.search import SearchIndexer from share.search.messages import MessageType from share.util import chunked -from share.util.graph import MutableGraph logger = logging.getLogger(__name__) @@ -318,7 +312,6 @@ def _consume_job(self, job, superfluous, force, apply_changes=True, index=True, ) job.ingested_normalized_data.add(datum) - # new Suid-based process if pls_format_metadata: records = FormattedMetadataRecord.objects.save_formatted_records( job.suid, @@ -331,14 +324,6 @@ def _consume_job(self, job, superfluous, force, apply_changes=True, index=True, if records and index: self._queue_for_indexing(job.suid, urgent) - # soon-to-be-rended ShareObject-based process: - if settings.SHARE_LEGACY_PIPELINE and apply_changes: - if graph is None: - graph = MutableGraph.from_jsonld(datum.data) - updated_work_ids = self._apply_changes(job, graph, datum) - if index and updated_work_ids: - self._update_index(updated_work_ids, urgent) - def _transform(self, job, raw): transformer = job.suid.source_config.get_transformer() @@ -369,48 +354,3 @@ def _regulate(self, job, graph): def _queue_for_indexing(self, suid, urgent): indexer = SearchIndexer(self.task.app) if self.task else SearchIndexer() indexer.send_messages(MessageType.INDEX_SUID, [suid.id], urgent=urgent) - - def _apply_changes(self, job, graph, normalized_datum): - updated = None - matches = None - - try: - # Load all relevant ContentTypes in a single query - ContentType.objects.get_for_models(*apps.get_models('share'), for_concrete_models=False) - - with transaction.atomic(): - change_set_builder = ChangeSetBuilder(graph, normalized_datum, disambiguate=True) - change_set = change_set_builder.build_change_set() - - user = normalized_datum.source # "source" here is a user... - source = user.source - if change_set and (source or user.is_robot or user.is_trusted): - updated = change_set.accept() - matches = change_set_builder.matches - - # Retry if it was just the wrong place at the wrong time - except (exceptions.IngestConflict, OperationalError): - job.retries = (job.retries or 0) + 1 - job.save(update_fields=('retries',)) - if job.retries > self.MAX_RETRIES: - raise - job.reschedule() - return - - if not updated: - return - - updated_works = set( - x.id for x in (updated or []) - if isinstance(x, AbstractCreativeWork) - ) - existing_works = set( - x.id for x in (matches or {}).values() - if isinstance(x, AbstractCreativeWork) - ) - - return list(updated_works | existing_works) - - def _update_index(self, work_ids, urgent): - indexer = SearchIndexer(self.task.app) if self.task else SearchIndexer() - indexer.index('creativework', *work_ids, urgent=urgent) diff --git a/share/util/__init__.py b/share/util/__init__.py index 68d6e3b7d..07e04dea7 100644 --- a/share/util/__init__.py +++ b/share/util/__init__.py @@ -1,7 +1,5 @@ from collections import OrderedDict -import os import re -import yaml WHITESPACE_RE = r'\s+' @@ -167,50 +165,6 @@ def __get_node(self, key): return self.__node_map[key] if self.__node_map else key -class ModelGenerator: - """Generate model classes from yaml specs""" - - def __init__(self, field_types={}): - self.__field_types = field_types - - def subclasses_from_yaml(self, file_name, base): - yaml_file = re.sub(r'\.py$', '.yaml', os.path.abspath(file_name)) - with open(yaml_file) as fobj: - model_specs = yaml.load(fobj) - - return self.generate_subclasses(model_specs, base) - - def generate_subclasses(self, model_specs, base): - models = {} - for (name, mspec) in sorted(model_specs.items()): - fields = mspec.get('fields', {}) - verbose_name_plural = mspec.get('verbose_name_plural', None) - - model = type(name, (base,), { - **{fname: self._get_field(fspec) for (fname, fspec) in fields.items()}, - '__doc__': mspec.get('description'), - '__qualname__': name, - '__module__': base.__module__ - }) - models[name] = model - models[model.VersionModel.__name__] = model.VersionModel - - if verbose_name_plural: - model._meta.verbose_name_plural = verbose_name_plural - elif model._meta.verbose_name.endswith('s'): - model._meta.verbose_name_plural = model._meta.verbose_name - - children = mspec.get('children') - if children: - models.update(self.generate_subclasses(children, model)) - - return models - - def _get_field(self, field_spec): - field_class = self.__field_types[field_spec['type']] - return field_class(*field_spec.get('args', []), **field_spec.get('kwargs', {})) - - class DictHashingDict: # A wrapper around dicts that can have dicts as keys diff --git a/tests/share/normalize/test_models.py b/tests/share/normalize/test_models.py index 35ab6e4d2..ab2e37a0c 100644 --- a/tests/share/normalize/test_models.py +++ b/tests/share/normalize/test_models.py @@ -221,7 +221,8 @@ def test_normalize_agent(self, input, output, Graph, ExpectedGraph): Organization(name='Money Foundation', identifiers=[AgentIdentifier(1)]), Organization(name='Money Foundation', identifiers=[AgentIdentifier(2)]) ], [ - Organization(name='Money Foundation', identifiers=[AgentIdentifier(1), AgentIdentifier(2)]) + Organization(name='Money Foundation', identifiers=[AgentIdentifier(1)]), + Organization(name='Money Foundation', identifiers=[AgentIdentifier(2)]), ]), # same name, different identifiers, different capitilization ([ @@ -247,7 +248,10 @@ def test_normalize_agent(self, input, output, Graph, ExpectedGraph): ([ Organization(name='Timetables Inc.'), Organization(name='Timetables Inc.', identifiers=[AgentIdentifier(1)]) - ], [Organization(name='Timetables Inc.', identifiers=[AgentIdentifier(1)])]), + ], [ + Organization(name='Timetables Inc.'), + Organization(name='Timetables Inc.', identifiers=[AgentIdentifier(1)]) + ]), # same identifier, different name, accept longest alphabetize ([ Institution(name='Cooking Institute', identifiers=[AgentIdentifier(1)]), @@ -276,8 +280,8 @@ def test_normalize_organization_institution_name(self, input, output, Graph, Exp Host(cited_as='Money Foundation', agent=Organization(name='Money Foundation', identifiers=[AgentIdentifier(1)])), Funder(cited_as='Money Foundation', agent=Organization(id=1, name='Money Foundation', identifiers=[AgentIdentifier(2)])), ], [ - Host(cited_as='Money Foundation', agent=Organization(id=1, name='Money Foundation', identifiers=[AgentIdentifier(1), AgentIdentifier(2)])), - Funder(cited_as='Money Foundation', agent=Organization(id=1)), + Host(cited_as='Money Foundation', agent=Organization(name='Money Foundation', identifiers=[AgentIdentifier(1)])), + Funder(cited_as='Money Foundation', agent=Organization(id=1, name='Money Foundation', identifiers=[AgentIdentifier(2)])), ]), # same identifier, different type ([ @@ -300,8 +304,8 @@ def test_normalize_organization_institution_name(self, input, output, Graph, Exp Funder(cited_as='Timetables Inc.', agent=Organization(id=1, name='Timetables Inc.')), Publisher(cited_as='Timetables Inc.', agent=Organization(id=2, name='Timetables Inc.', identifiers=[AgentIdentifier(1)])) ], [ - Funder(cited_as='Timetables Inc.', agent=Organization(id=2, name='Timetables Inc.', identifiers=[AgentIdentifier(1)])), - Publisher(cited_as='Timetables Inc.', agent=Organization(id=2)) + Funder(cited_as='Timetables Inc.', agent=Organization(id=1, name='Timetables Inc.')), + Publisher(cited_as='Timetables Inc.', agent=Organization(id=2, name='Timetables Inc.', identifiers=[AgentIdentifier(1)])) ]), # same identifier, different name, accept longest alphabetize ([ @@ -337,14 +341,16 @@ def test_normalize_mixed_agent_relation(self, input, output, Graph, ExpectedGrap Creator(cited_as='American Heart Association', agent=Organization(id=0, name='American Heart Association', identifiers=[AgentIdentifier(1, id=1)])), Contributor(cited_as='American Heart Association', agent=Organization(id=1, name='American Heart Association', identifiers=[AgentIdentifier(1, id=2)])) ], [ - Creator(cited_as='American Heart Association', agent=Organization(id=1, name='American Heart Association', identifiers=[AgentIdentifier(1, id=2)])) + Creator(cited_as='American Heart Association', agent=Organization(id=1, name='American Heart Association', identifiers=[AgentIdentifier(1, id=2)])), + Contributor(cited_as='American Heart Association', agent=Organization(id=1)), ]), # same name, different identifiers, different type, same type tree ([ Creator(cited_as='Money Foundation', agent=Organization(id=1, name='Money Foundation', identifiers=[AgentIdentifier()])), Contributor(cited_as='Money Foundation', agent=Organization(id=2, name='Money Foundation', identifiers=[AgentIdentifier()])), ], [ - Creator(cited_as='Money Foundation', agent=Organization(id=2, name='Money Foundation', identifiers=[AgentIdentifier(), AgentIdentifier()])) + Creator(cited_as='Money Foundation', agent=Organization(id=1, name='Money Foundation', identifiers=[AgentIdentifier()])), + Contributor(cited_as='Money Foundation', agent=Organization(id=2, name='Money Foundation', identifiers=[AgentIdentifier()])), ]), # same identifier, same name, different type ([ @@ -359,20 +365,23 @@ def test_normalize_mixed_agent_relation(self, input, output, Graph, ExpectedGrap Creator(cited_as='Bob Dylan', agent=Person(id=0, name='Bob Dylan', identifiers=[AgentIdentifier(1, id=0)])), Contributor(cited_as='Bob Dylan', agent=Person(id=1, name='Bob Dylan', identifiers=[AgentIdentifier(1, id=1)])), ], [ - Creator(cited_as='Bob Dylan', agent=Person(id=0, name='Bob Dylan', identifiers=[AgentIdentifier(1, id=1)])) + Creator(cited_as='Bob Dylan', agent=Person(id=0, name='Bob Dylan', identifiers=[AgentIdentifier(1, id=0)])), + Contributor(cited_as='Bob Dylan', agent=Person(id=0)), ]), # same identifier, different name, different type ([ Creator(cited_as='B. Dylan', agent=Person(id=0, name='B. Dylan', identifiers=[AgentIdentifier(1, id=0)])), Contributor(cited_as='Bob Dylan', agent=Person(id=1, name='Bob Dylan', identifiers=[AgentIdentifier(1, id=1)])), ], [ - Creator(cited_as='Bob Dylan', agent=Person(id=0, name='Bob Dylan', identifiers=[AgentIdentifier(1, id=1)])) + Creator(cited_as='B. Dylan', agent=Person(id=0, name='Bob Dylan', identifiers=[AgentIdentifier(1, id=0)])), + Contributor(cited_as='Bob Dylan', agent=Person(id=0)), ]), # same name, one identifier, add identifier ([ Creator(1, id=0, order_cited=4, cited_as='Timetables Inc.', agent=Organization(id=0, name='Timetables Inc.')), Creator(1, id=1, order_cited=20, cited_as='Timetables Inc.', agent=Organization(id=1, name='Timetables Inc.', identifiers=[AgentIdentifier()])) ], [ + Creator(1, id=0, order_cited=4, cited_as='Timetables Inc.', agent=Organization(id=0, name='Timetables Inc.')), Creator(1, id=1, order_cited=20, cited_as='Timetables Inc.', agent=Organization(id=1, name='Timetables Inc.', identifiers=[AgentIdentifier()])) ]), # same identifier, different name, accept longest alphabetize @@ -382,6 +391,7 @@ def test_normalize_mixed_agent_relation(self, input, output, Graph, ExpectedGrap Funder(cited_as='Cook Institute', agent=Organization(id=3, name='Cook Institute', identifiers=[AgentIdentifier(1, id=3)])) ], [ Creator(cited_as='Cooking Institute', agent=Institution(id=1, name='Cooking Institute', identifiers=[AgentIdentifier(1, id=3)])), + Contributor(cited_as='Cooking Instituze', agent=Organization(id=1)), Funder(cited_as='Cook Institute', agent=Institution(id=1)) ]), # same identifier, different name, different type, accept longest alphabetize, more specific @@ -391,6 +401,7 @@ def test_normalize_mixed_agent_relation(self, input, output, Graph, ExpectedGrap Funder(cited_as='Cook Institute', agent=Institution(id=2, name='Cook Institute', identifiers=[AgentIdentifier(1, id=3)])) ], [ Creator(cited_as='Cooking Institute', order_cited=10, agent=Institution(id=0, name='Cooking Institute', identifiers=[AgentIdentifier(1, id=3)])), + Contributor(cited_as='Cooking Instituze', agent=Institution(id=0)), Funder(cited_as='Cook Institute', agent=Institution(id=0)) ]), # Related agent removed diff --git a/tests/share/regulate/steps/test_deduplicate.py b/tests/share/regulate/steps/test_deduplicate.py index c7084f55d..ba22aa711 100644 --- a/tests/share/regulate/steps/test_deduplicate.py +++ b/tests/share/regulate/steps/test_deduplicate.py @@ -2,7 +2,11 @@ from share.regulate.steps.deduplicate import Deduplicate -from tests.share.normalize.factories import * +from tests.share.normalize.factories import ( + CreativeWork, + Preprint, + WorkIdentifier, +) class TestDeduplicate: From e3e5eeb68044d1f9303a8b99cdf3a7ece3fcb7e0 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Wed, 17 Feb 2021 14:28:58 -0500 Subject: [PATCH 03/14] add big rend migration --- api/migrations/0001_create_application.py | 2 +- .../migrations/0002_application.py | 2 +- share/migrations/0001_initial.py | 103 - ...nitial_squashed_0020_auto_20170206_2114.py | 30 - .../migrations/0001_squashed_0058_big_rend.py | 634 +++++ share/migrations/0002_initial.py | 2252 ----------------- share/migrations/0003_create_share_user.py | 25 - ...4_unique_organization_institution_names.py | 20 - ...update_trigger_migrations_20161117_0127.py | 128 - share/migrations/0006_auto_20161118_1707.py | 24 - share/migrations/0007_auto_20161122_1810.py | 576 ----- share/migrations/0008_auto_20161207_1535.py | 119 - share/migrations/0009_auto_20161209_1945.py | 129 - share/migrations/0010_auto_20161212_1418_a.py | 228 -- share/migrations/0010_auto_20161212_1418_b.py | 228 -- share/migrations/0010_auto_20161212_1418_c.py | 228 -- share/migrations/0010_auto_20161212_1418_d.py | 528 ---- share/migrations/0010_auto_20161212_1418_e.py | 88 - share/migrations/0011_sitebanner.py | 32 - share/migrations/0012_auto_20161212_1555.py | 87 - share/migrations/0013_auto_20161214_1921.py | 91 - share/migrations/0014_auto_20170112_2143.py | 20 - share/migrations/0014_auto_20170127_1423.py | 54 - share/migrations/0015_auto_20170117_2050.py | 45 - share/migrations/0016_auto_20170130_2130.py | 26 - share/migrations/0016_merge.py | 16 - share/migrations/0017_merge.py | 17 - share/migrations/0018_fuzzycount.py | 34 - share/migrations/0018_store_favicons.py | 35 - share/migrations/0019_merge.py | 16 - share/migrations/0023_auto_20170315_0241.py | 2 +- share/migrations/0057_unconcurrent_index.py | 28 + share/migrations/0058_big_rend.py | 1702 +++++++++++++ share/models/celery.py | 39 - share/models/ingest.py | 7 +- 35 files changed, 2370 insertions(+), 5225 deletions(-) delete mode 100644 share/migrations/0001_initial.py create mode 100644 share/migrations/0001_squashed_0058_big_rend.py delete mode 100644 share/migrations/0002_initial.py delete mode 100644 share/migrations/0003_create_share_user.py delete mode 100644 share/migrations/0004_unique_organization_institution_names.py delete mode 100644 share/migrations/0005_update_trigger_migrations_20161117_0127.py delete mode 100644 share/migrations/0006_auto_20161118_1707.py delete mode 100644 share/migrations/0007_auto_20161122_1810.py delete mode 100644 share/migrations/0008_auto_20161207_1535.py delete mode 100644 share/migrations/0009_auto_20161209_1945.py delete mode 100644 share/migrations/0010_auto_20161212_1418_a.py delete mode 100644 share/migrations/0010_auto_20161212_1418_b.py delete mode 100644 share/migrations/0010_auto_20161212_1418_c.py delete mode 100644 share/migrations/0010_auto_20161212_1418_d.py delete mode 100644 share/migrations/0010_auto_20161212_1418_e.py delete mode 100644 share/migrations/0011_sitebanner.py delete mode 100644 share/migrations/0012_auto_20161212_1555.py delete mode 100644 share/migrations/0013_auto_20161214_1921.py delete mode 100644 share/migrations/0014_auto_20170112_2143.py delete mode 100644 share/migrations/0014_auto_20170127_1423.py delete mode 100644 share/migrations/0015_auto_20170117_2050.py delete mode 100644 share/migrations/0016_auto_20170130_2130.py delete mode 100644 share/migrations/0016_merge.py delete mode 100644 share/migrations/0017_merge.py delete mode 100644 share/migrations/0018_fuzzycount.py delete mode 100644 share/migrations/0018_store_favicons.py delete mode 100644 share/migrations/0019_merge.py create mode 100644 share/migrations/0057_unconcurrent_index.py create mode 100644 share/migrations/0058_big_rend.py diff --git a/api/migrations/0001_create_application.py b/api/migrations/0001_create_application.py index b9cd31ef8..cde04bfb0 100644 --- a/api/migrations/0001_create_application.py +++ b/api/migrations/0001_create_application.py @@ -22,7 +22,7 @@ class Migration(migrations.Migration): dependencies = [ ('oauth2_provider', '0002_08_updates'), - ('share', '0003_create_share_user'), + ('share', '0001_initial_squashed_0020_auto_20170206_2114'), ] operations = [ diff --git a/osf_oauth2_adapter/migrations/0002_application.py b/osf_oauth2_adapter/migrations/0002_application.py index 7d131de09..4376bdf28 100644 --- a/osf_oauth2_adapter/migrations/0002_application.py +++ b/osf_oauth2_adapter/migrations/0002_application.py @@ -32,7 +32,7 @@ class Migration(migrations.Migration): dependencies = [ ('osf_oauth2_adapter', '0001_make_human_group'), - ('share', '0003_create_share_user'), + ('share', '0001_initial_squashed_0020_auto_20170206_2114'), ('sites', '0002_alter_domain_unique'), ('socialaccount', '0003_extra_data_default_dict'), ] diff --git a/share/migrations/0001_initial.py b/share/migrations/0001_initial.py deleted file mode 100644 index 342d56924..000000000 --- a/share/migrations/0001_initial.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-11-04 16:46 -from __future__ import unicode_literals - -from django.conf import settings -import django.contrib.postgres.fields.jsonb -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion -import django.utils.timezone -import share.models.core -import share.models.fields -import share.models.validators - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('auth', '0007_alter_validators_add_error_messages'), - ] - - operations = [ - migrations.CreateModel( - name='ShareUser', - fields=[ - ('password', models.CharField(max_length=128, verbose_name='password')), - ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), - ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), - ('id', models.AutoField(primary_key=True, serialize=False)), - ('username', models.TextField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 30 characters or fewer. Letters, digits and @/./+/-/_ only.', unique=True, validators=[django.core.validators.MaxLengthValidator(64), django.core.validators.RegexValidator('^[\\w.@+-]+$', 'Enter a valid username. This value may contain only letters, numbers and @/./+/-/_ characters.')], verbose_name='username')), - ('first_name', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(64)], verbose_name='first name')), - ('last_name', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(64)], verbose_name='last name')), - ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), - ('gravatar', share.models.fields.ShareURLField(blank=True)), - ('time_zone', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(100)])), - ('locale', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(100)])), - ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), - ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), - ('is_trusted', models.BooleanField(default=False, help_text='Designates whether the user can push directly into the db.', verbose_name='trusted')), - ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), - ('robot', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(40)])), - ('long_title', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(100)])), - ('home_page', share.models.fields.ShareURLField(blank=True)), - ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')), - ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')), - ], - options={ - 'verbose_name_plural': 'Share users', - 'verbose_name': 'Share user', - }, - managers=[ - ('objects', share.models.core.ShareUserManager()), - ], - ), - migrations.CreateModel( - name='RawData', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('app_label', models.TextField(db_index=True)), - ('provider_doc_id', models.TextField()), - ('data', models.TextField()), - ('sha256', models.TextField(validators=[django.core.validators.MaxLengthValidator(64)])), - ('date_seen', models.DateTimeField(auto_now=True)), - ('date_harvested', models.DateTimeField(auto_now_add=True)), - ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'verbose_name_plural': 'Raw data', - }, - ), - migrations.CreateModel( - name='ProviderRegistration', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('status', models.IntegerField(choices=[(0, 'pending'), (1, 'accepted'), (2, 'implemented'), (3, 'rejected')], default=0)), - ('submitted_at', models.DateTimeField(auto_now_add=True, db_index=True)), - ('contact_name', models.TextField(max_length=300)), - ('contact_email', models.EmailField(max_length=254)), - ('contact_affiliation', models.TextField(max_length=300)), - ('direct_source', models.BooleanField(default=False)), - ('source_name', models.TextField(max_length=300)), - ('source_description', models.TextField(max_length=1000)), - ('source_rate_limit', models.TextField(blank=True, default='', max_length=300)), - ('source_documentation', models.TextField(blank=True, default='', max_length=300)), - ('source_preferred_metadata_prefix', models.TextField(blank=True, default='', max_length=300)), - ('source_oai', models.BooleanField(default=False)), - ('source_base_url', models.URLField(blank=True, default='')), - ('source_disallowed_sets', models.TextField(blank=True, default='', max_length=300)), - ('source_additional_info', models.TextField(blank=True, default='', max_length=1000)), - ('submitted_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'ordering': ['-submitted_at'], - }, - ), - migrations.AlterUniqueTogether( - name='rawdata', - unique_together=set([('provider_doc_id', 'app_label', 'source', 'sha256')]), - ), - ] diff --git a/share/migrations/0001_initial_squashed_0020_auto_20170206_2114.py b/share/migrations/0001_initial_squashed_0020_auto_20170206_2114.py index ee6b80a63..896501d98 100644 --- a/share/migrations/0001_initial_squashed_0020_auto_20170206_2114.py +++ b/share/migrations/0001_initial_squashed_0020_auto_20170206_2114.py @@ -37,36 +37,6 @@ class Migration(migrations.Migration): ('contenttypes', '0002_remove_content_type_name'), ] - replaces = [ - ('share', '0001_initial'), - ('share', '0002_initial'), - ('share', '0003_create_share_user'), - ('share', '0004_unique_organization_institution_names'), - ('share', '0005_update_trigger_migrations_20161117_0127'), - ('share', '0006_auto_20161118_1707'), - ('share', '0007_auto_20161122_1810'), - ('share', '0008_auto_20161207_1535'), - ('share', '0009_auto_20161209_1945'), - ('share', '0010_auto_20161212_1418_a'), - ('share', '0010_auto_20161212_1418_b'), - ('share', '0010_auto_20161212_1418_c'), - ('share', '0010_auto_20161212_1418_d'), - ('share', '0010_auto_20161212_1418_e'), - ('share', '0011_sitebanner'), - ('share', '0012_auto_20161212_1555'), - ('share', '0013_auto_20161214_1921'), - ('share', '0014_auto_20170112_2143'), - ('share', '0014_auto_20170127_1423'), - ('share', '0015_auto_20170117_2050'), - ('share', '0016_auto_20170130_2130'), - ('share', '0016_merge'), - ('share', '0017_merge'), - ('share', '0018_fuzzycount'), - ('share', '0018_store_favicons'), - ('share', '0019_merge'), - ('share', '0020_auto_20170206_2114'), - ] - operations = [ migrations.CreateModel( name='ShareUser', diff --git a/share/migrations/0001_squashed_0058_big_rend.py b/share/migrations/0001_squashed_0058_big_rend.py new file mode 100644 index 000000000..1397a5419 --- /dev/null +++ b/share/migrations/0001_squashed_0058_big_rend.py @@ -0,0 +1,634 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2021-02-23 15:54 +from __future__ import unicode_literals + +import datetime +import db.deletion +from django.conf import settings +import django.contrib.postgres.fields.jsonb +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import share.models.core +import share.models.fields +import share.models.indexes +import share.models.ingest +import share.models.jobs +import share.models.validators + + +def create_share_robot_user(apps, schema_editor): + ShareUser = apps.get_model('share', 'ShareUser') + Source = apps.get_model('share', 'Source') + + system_user = ShareUser.objects.db_manager(schema_editor.connection.alias).create_robot_user(username=settings.APPLICATION_USERNAME, robot='') + + system_user.is_trusted = True + system_user.save() + + Source.objects.update_or_create( + user=system_user, + defaults={ + 'name': settings.APPLICATION_USERNAME, + 'long_title': 'SHARE System', + 'canonical': True, + } + ) + + +def create_share_admin_user(apps, schema_editor): + import os + ShareUser = apps.get_model('share', 'ShareUser') + ShareUser.objects.db_manager(schema_editor.connection.alias).create_superuser('admin', os.environ.get('SHARE_ADMIN_PASSWORD', 'password')) + + +class Migration(migrations.Migration): + + replaces = [ + ('share', '0001_initial_squashed_0020_auto_20170206_2114'), + ('share', '0023_auto_20170315_0241'), + ('share', '0024_populate_suids'), + ('share', '0025_auto_20170315_1544'), + ('share', '0026_auto_20170324_1757'), + ('share', '0027_harvestlog_datetime_to_date'), + ('share', '0028_auto_20170329_1703'), + ('share', '0029_source_is_deleted'), + ('share', '0030_source_canonical'), + ('share', '0031_auto_20170504_1957'), + ('share', '0032_auto_20170508_1931'), + ('share', '0032_auto_20170508_1753'), + ('share', '0033_merge_20170509_1710'), + ('share', '0034_auto_20170512_2052'), + ('share', '0035_celery_upgrade'), + ('share', '0036_rawdatum_datestamp'), + ('share', '0037_pglock'), + ('share', '0038_auto_20170606_1857'), + ('share', '0038_trust_system_user'), + ('share', '0039_subject_taxonomy_a'), + ('share', '0040_subject_provenance'), + ('share', '0041_subject_taxonomy_b'), + ('share', '0039_auto_20170614_1825'), + ('share', '0042_merge_20170620_1330'), + ('share', '0039_sourcestat'), + ('share', '0043_merge_20170626_1516'), + ('share', '0040_rawdatum_no_output'), + ('share', '0041_no_change_index'), + ('share', '0044_merge_20170628_1811'), + ('share', '0045_subject_seq'), + ('share', '0046_auto_20170714_1547'), + ('share', '0047_auto_20171019_2018'), + ('share', '0048_auto_20171113_1852'), + ('share', '0049_jobs'), + ('share', '0050_harvest_job_index'), + ('share', '0051_auto_20180319_2145'), + ('share', '0051_auto_20180301_2032'), + ('share', '0052_merge_20180416_1412'), + ('share', '0053_auto_20180419_2033'), + ('share', '0054_formatted_metadata_record'), + ('share', '0055_job_status_created'), + ('share', '0056_auto_20210217_1923'), + ('share', '0057_unconcurrent_index'), + ('share', '0058_big_rend'), + ] + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('auth', '0007_alter_validators_add_error_messages'), + #('oauth2_provider', '0004_auto_20160525_1623'), + ] + + operations = [ + migrations.CreateModel( + name='ShareUser', + fields=[ + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('id', models.AutoField(primary_key=True, serialize=False)), + ('username', models.TextField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 30 characters or fewer. Letters, digits and @/./+/-/_ only.', unique=True, validators=[django.core.validators.MaxLengthValidator(64), django.core.validators.RegexValidator('^[\\w.@+-]+$', 'Enter a valid username. This value may contain only letters, numbers and @/./+/-/_ characters.')], verbose_name='username')), + ('first_name', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(64)], verbose_name='first name')), + ('last_name', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(64)], verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), + ('gravatar', share.models.fields.ShareURLField(blank=True)), + ('time_zone', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(100)])), + ('locale', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(100)])), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('is_trusted', models.BooleanField(default=False, help_text='Designates whether the user can push directly into the db.', verbose_name='trusted')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('robot', models.TextField(blank=True, validators=[django.core.validators.MaxLengthValidator(40)])), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')), + ], + options={ + 'verbose_name_plural': 'Share users', + 'verbose_name': 'Share user', + }, + managers=[ + ('objects', share.models.core.ShareUserManager()), + ], + ), + migrations.CreateModel( + name='NormalizedData', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True, null=True)), + ('data', share.models.fields.DateTimeAwareJSONField(validators=[share.models.validators.JSONLDValidator()])), + ], + ), + migrations.CreateModel( + name='ProviderRegistration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.IntegerField(choices=[(0, 'pending'), (1, 'accepted'), (2, 'implemented'), (3, 'rejected')], default=0)), + ('submitted_at', models.DateTimeField(auto_now_add=True, db_index=True)), + ('contact_name', models.TextField(max_length=300)), + ('contact_email', models.EmailField(max_length=254)), + ('contact_affiliation', models.TextField(max_length=300)), + ('direct_source', models.BooleanField(default=False)), + ('source_name', models.TextField(max_length=300)), + ('source_description', models.TextField(max_length=1000)), + ('source_rate_limit', models.TextField(blank=True, default='', max_length=300)), + ('source_documentation', models.TextField(blank=True, default='', max_length=300)), + ('source_preferred_metadata_prefix', models.TextField(blank=True, default='', max_length=300)), + ('source_oai', models.BooleanField(default=False)), + ('source_base_url', models.URLField(blank=True, default='')), + ('source_disallowed_sets', models.TextField(blank=True, default='', max_length=300)), + ('source_additional_info', models.TextField(blank=True, default='', max_length=1000)), + ('submitted_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-submitted_at'], + }, + ), + migrations.CreateModel( + name='RawDatum', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('datum', models.TextField()), + ('sha256', models.TextField(validators=[django.core.validators.MaxLengthValidator(64)])), + ('date_modified', models.DateTimeField(auto_now=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ], + options={ + 'verbose_name_plural': 'Raw Data', + }, + ), + migrations.CreateModel( + name='SiteBanner', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('active', models.BooleanField(db_index=True, default=True)), + ('title', models.CharField(max_length=300)), + ('description', models.TextField(blank=True)), + ('color', models.IntegerField(choices=[(0, 'success'), (1, 'info'), (2, 'warning'), (3, 'danger')], default=1)), + ('icon', models.CharField(blank=True, default='exclamation', max_length=31)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('last_modified_at', models.DateTimeField(auto_now=True)), + ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ('last_modified_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.AddField( + model_name='normalizeddata', + name='raw', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.RawDatum'), + ), + migrations.AddField( + model_name='normalizeddata', + name='source', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + migrations.CreateModel( + name='Harvester', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.TextField(unique=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ], + managers=[ + ('objects', share.models.ingest.NaturalKeyManager('key')), + ], + ), + migrations.CreateModel( + name='HarvestJob', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('task_id', models.UUIDField(null=True)), + ('status', models.IntegerField(choices=[(0, 'Created'), (1, 'Started'), (2, 'Failed'), (3, 'Succeeded'), (4, 'Rescheduled'), (6, 'Forced'), (7, 'Skipped'), (8, 'Retrying'), (9, 'Cancelled')], db_index=True, default=0)), + ('error_context', models.TextField(blank=True, db_column='context', default='')), + ('completions', models.IntegerField(default=0)), + ('date_started', models.DateTimeField(blank=True, null=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True, db_index=True)), + ('share_version', models.TextField(default=share.models.jobs.get_share_version, editable=False)), + ('source_config_version', models.PositiveIntegerField()), + ('end_date', models.DateTimeField(db_index=True)), + ('start_date', models.DateTimeField(db_index=True)), + ('harvester_version', models.PositiveIntegerField()), + ], + options={ + 'db_table': 'share_harvestlog', + }, + ), + migrations.CreateModel( + name='Source', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.TextField(unique=True)), + ('long_title', models.TextField(unique=True)), + ('home_page', models.URLField(null=True)), + ('icon', models.ImageField(null=True, storage=share.models.ingest.SourceIconStorage(), upload_to=share.models.ingest.icon_name)), + ], + managers=[ + ('objects', share.models.ingest.NaturalKeyManager('name')), + ], + ), + migrations.CreateModel( + name='SourceConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('label', models.TextField(unique=True)), + ('version', models.PositiveIntegerField(default=1)), + ('base_url', models.URLField(null=True)), + ('earliest_date', models.DateField(null=True)), + ('rate_limit_allowance', models.PositiveIntegerField(default=5)), + ('rate_limit_period', models.PositiveIntegerField(default=1)), + ('harvester_kwargs', django.contrib.postgres.fields.jsonb.JSONField(null=True)), + ('transformer_kwargs', django.contrib.postgres.fields.jsonb.JSONField(null=True)), + ('disabled', models.BooleanField(default=False)), + ('harvester', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.Harvester')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='share.Source')), + ], + managers=[ + ('objects', share.models.ingest.NaturalKeyManager('label')), + ], + ), + migrations.CreateModel( + name='SourceIcon', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('image', models.BinaryField()), + ('source_name', models.TextField(unique=True)), + ], + ), + migrations.CreateModel( + name='SourceUniqueIdentifier', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('identifier', models.TextField()), + ('source_config', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='share.SourceConfig')), + ], + ), + migrations.CreateModel( + name='Transformer', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.TextField(unique=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True)), + ], + managers=[ + ('objects', share.models.ingest.NaturalKeyManager('key')), + ], + ), + migrations.CreateModel( + name='RawDatumJob', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('datum', models.ForeignKey(db_column='rawdatum_id', on_delete=django.db.models.deletion.CASCADE, to='share.RawDatum')), + ('job', models.ForeignKey(db_column='harvestlog_id', on_delete=django.db.models.deletion.CASCADE, to='share.HarvestJob')), + ], + options={ + 'db_table': 'share_rawdatum_logs', + }, + ), + migrations.AddField( + model_name='rawdatum', + name='jobs', + field=models.ManyToManyField(related_name='raw_data', through='share.RawDatumJob', to='share.HarvestJob'), + ), + migrations.AddField( + model_name='sourceconfig', + name='transformer', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.Transformer'), + ), + migrations.AddField( + model_name='source', + name='user', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='harvestjob', + name='source_config', + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='harvest_jobs', to='share.SourceConfig'), + ), + migrations.AddField( + model_name='rawdatum', + name='suid', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='share.SourceUniqueIdentifier'), + ), + migrations.AlterUniqueTogether( + name='sourceuniqueidentifier', + unique_together=set([('identifier', 'source_config')]), + ), + migrations.AlterUniqueTogether( + name='harvestjob', + unique_together=set([('source_config', 'start_date', 'end_date', 'harvester_version', 'source_config_version')]), + ), + migrations.AlterUniqueTogether( + name='rawdatum', + unique_together=set([('suid', 'sha256')]), + ), + migrations.AlterField( + model_name='sourceconfig', + name='earliest_date', + field=models.DateField(blank=True, null=True), + ), + migrations.AlterField( + model_name='sourceconfig', + name='harvester_kwargs', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + migrations.AlterField( + model_name='sourceconfig', + name='transformer_kwargs', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + migrations.AlterField( + model_name='harvestjob', + name='end_date', + field=models.DateField(db_index=True), + ), + migrations.AlterField( + model_name='harvestjob', + name='start_date', + field=models.DateField(db_index=True), + ), + migrations.AlterField( + model_name='source', + name='icon', + field=models.ImageField(blank=True, default='', storage=share.models.ingest.SourceIconStorage(), upload_to=share.models.ingest.icon_name), + preserve_default=False, + ), + migrations.AddField( + model_name='source', + name='is_deleted', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='source', + name='canonical', + field=models.BooleanField(db_index=True, default=False), + ), + migrations.CreateModel( + name='CeleryTaskResult', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('correlation_id', models.TextField(blank=True)), + ('status', models.CharField(choices=[('FAILURE', 'FAILURE'), ('PENDING', 'PENDING'), ('RECEIVED', 'RECEIVED'), ('RETRY', 'RETRY'), ('REVOKED', 'REVOKED'), ('STARTED', 'STARTED'), ('SUCCESS', 'SUCCESS')], db_index=True, default='PENDING', max_length=50)), + ('task_id', models.UUIDField(db_index=True, unique=True)), + ('meta', share.models.fields.DateTimeAwareJSONField(editable=False, null=True)), + ('result', share.models.fields.DateTimeAwareJSONField(editable=False, null=True)), + ('task_name', models.TextField(blank=True, db_index=True, editable=False, null=True)), + ('traceback', models.TextField(blank=True, editable=False, null=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True, db_index=True)), + ('share_version', models.TextField(default=share.models.jobs.get_share_version, editable=False)), + ], + options={ + 'verbose_name': 'Celery Task Result', + 'verbose_name_plural': 'Celery Task Results', + }, + ), + migrations.AddField( + model_name='sourceconfig', + name='full_harvest', + field=models.BooleanField(default=False, help_text='Whether or not this SourceConfig should be fully harvested. Requires earliest_date to be set. The schedule harvests task will create all logs necessary if this flag is set. This should never be set to True by default. '), + ), + migrations.AddField( + model_name='sourceconfig', + name='harvest_after', + field=models.TimeField(default='02:00'), + ), + migrations.AddField( + model_name='sourceconfig', + name='harvest_interval', + field=models.DurationField(default='1 day'), + ), + migrations.AddField( + model_name='normalizeddata', + name='tasks', + field=models.ManyToManyField(to='share.CeleryTaskResult'), + ), + migrations.AddIndex( + model_name='celerytaskresult', + index=models.Index(fields=['-date_modified', '-id'], name='share_celer_date_mo_686d4d_idx'), + ), + migrations.AddField( + model_name='rawdatum', + name='datestamp', + field=models.DateTimeField(help_text="The most relevant datetime that can be extracted from this RawDatum. This may be, but is not limitted to, a deletion, modification, publication, or creation datestamp. Ideally, this datetime should be appropriate for determining the chronological order it's data will be applied.", null=True), + ), + migrations.CreateModel( + name='PGLock', + fields=[ + ('pid', models.IntegerField(primary_key=True, serialize=False)), + ('locktype', models.TextField()), + ('objid', models.IntegerField()), + ('classid', models.IntegerField()), + ], + options={ + 'db_table': 'pg_locks', + 'managed': False, + }, + ), + migrations.AlterField( + model_name='sourceconfig', + name='harvest_interval', + field=models.DurationField(default=datetime.timedelta(1)), + ), + migrations.CreateModel( + name='SourceStat', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('is_deleted', models.BooleanField(default=False)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('response_status_code', models.SmallIntegerField(blank=True, null=True)), + ('response_elapsed_time', models.FloatField(blank=True, null=True)), + ('response_exception', models.TextField(blank=True, null=True)), + ('earliest_datestamp_config', models.DateField(blank=True, null=True)), + ('base_url_config', models.TextField()), + ('admin_note', models.TextField(blank=True)), + ('grade', models.FloatField()), + ('earliest_datestamp_source', models.DateField(blank=True, null=True)), + ('earliest_datestamps_match', models.BooleanField(default=False)), + ('base_url_source', models.TextField(blank=True, null=True)), + ('base_urls_match', models.BooleanField(default=False)), + ('config', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='share.SourceConfig')), + ], + ), + migrations.AddField( + model_name='rawdatum', + name='no_output', + field=models.NullBooleanField(help_text='Indicates that this RawDatum resulted in an empty graph when transformed. This allows the RawDataJanitor to find records that have not been processed. Records that result in an empty graph will not have a NormalizedData associated with them, which would otherwise look like data that has not yet been processed.'), + ), + migrations.AddIndex( + model_name='rawdatum', + index=models.Index(fields=['no_output'], name='share_rawda_no_outp_f0330f_idx'), + ), + migrations.AlterField( + model_name='source', + name='user', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='sourceconfig', + name='source', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='source_configs', to='share.Source'), + ), + migrations.AddField( + model_name='sourceconfig', + name='private_harvester_kwargs', + field=share.models.fields.EncryptedJSONField(blank=True, null=True), + ), + migrations.AddField( + model_name='sourceconfig', + name='private_transformer_kwargs', + field=share.models.fields.EncryptedJSONField(blank=True, null=True), + ), + migrations.CreateModel( + name='IngestJob', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('task_id', models.UUIDField(null=True)), + ('status', models.IntegerField(choices=[(0, 'Created'), (1, 'Started'), (2, 'Failed'), (3, 'Succeeded'), (4, 'Rescheduled'), (6, 'Forced'), (7, 'Skipped'), (8, 'Retrying'), (9, 'Cancelled')], db_index=True, default=0)), + ('claimed', models.NullBooleanField()), + ('error_type', models.TextField(blank=True, db_index=True, null=True)), + ('error_message', models.TextField(blank=True, db_column='message', null=True)), + ('error_context', models.TextField(blank=True, db_column='context', default='')), + ('completions', models.IntegerField(default=0)), + ('date_started', models.DateTimeField(blank=True, null=True)), + ('date_created', models.DateTimeField(auto_now_add=True)), + ('date_modified', models.DateTimeField(auto_now=True, db_index=True)), + ('share_version', models.TextField(default=share.models.jobs.get_share_version, editable=False)), + ('source_config_version', models.PositiveIntegerField()), + ('transformer_version', models.PositiveIntegerField()), + ('regulator_version', models.PositiveIntegerField()), + ('retries', models.IntegerField(null=True)), + ], + ), + migrations.CreateModel( + name='RegulatorLog', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('description', models.TextField()), + ('node_id', models.TextField(null=True)), + ('rejected', models.BooleanField()), + ('ingest_job', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='regulator_logs', to='share.IngestJob')), + ], + ), + migrations.AlterModelManagers( + name='sourceconfig', + managers=[ + ('objects', share.models.ingest.SourceConfigManager('label')), + ], + ), + migrations.AddField( + model_name='harvestjob', + name='claimed', + field=models.NullBooleanField(), + ), + migrations.AddField( + model_name='harvestjob', + name='error_message', + field=models.TextField(blank=True, db_column='message', null=True), + ), + migrations.AddField( + model_name='harvestjob', + name='error_type', + field=models.TextField(blank=True, db_index=True, null=True), + ), + migrations.AlterField( + model_name='rawdatum', + name='datestamp', + field=models.DateTimeField(help_text='The most relevant datetime that can be extracted from this RawDatum. This may be, but is not limited to, a deletion, modification, publication, or creation datestamp. Ideally, this datetime should be appropriate for determining the chronological order its data will be applied.', null=True), + ), + migrations.AlterField( + model_name='source', + name='home_page', + field=models.URLField(blank=True, null=True), + ), + migrations.AlterField( + model_name='sourceconfig', + name='full_harvest', + field=models.BooleanField(default=False, help_text='Whether or not this SourceConfig should be fully harvested. Requires earliest_date to be set. The schedule harvests task will create all jobs necessary if this flag is set. This should never be set to True by default. '), + ), + migrations.AddField( + model_name='ingestjob', + name='raw', + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='ingest_jobs', to='share.RawDatum'), + ), + migrations.AddField( + model_name='ingestjob', + name='source_config', + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='ingest_jobs', to='share.SourceConfig'), + ), + migrations.AddField( + model_name='ingestjob', + name='suid', + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='ingest_jobs', to='share.SourceUniqueIdentifier'), + ), + migrations.AddField( + model_name='ingestjob', + name='ingested_normalized_data', + field=models.ManyToManyField(related_name='ingest_jobs', to='share.NormalizedData'), + ), + migrations.AlterUniqueTogether( + name='ingestjob', + unique_together=set([('raw', 'source_config_version', 'transformer_version', 'regulator_version')]), + ), + migrations.AlterField( + model_name='rawdatum', + name='suid', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='raw_data', to='share.SourceUniqueIdentifier'), + ), + migrations.AddField( + model_name='sourceconfig', + name='regulator_steps', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + migrations.CreateModel( + name='FormattedMetadataRecord', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('record_format', models.TextField()), + ('date_modified', models.DateTimeField(auto_now=True)), + ('formatted_metadata', models.TextField()), + ('suid', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='share.SourceUniqueIdentifier')), + ], + ), + migrations.AlterUniqueTogether( + name='formattedmetadatarecord', + unique_together=set([('suid', 'record_format')]), + ), + migrations.AlterField( + model_name='formattedmetadatarecord', + name='record_format', + field=models.TextField(choices=[('oai_dc', 'oai_dc'), ('sharev2_elastic', 'sharev2_elastic')]), + ), + migrations.RunPython( + code=create_share_robot_user, + ), + migrations.RunPython( + code=create_share_admin_user, + ), + migrations.RunSQL( + sql='\n CREATE OR REPLACE FUNCTION count_estimate(query text) RETURNS INTEGER AS\n $func$\n DECLARE\n rec record;\n ROWS INTEGER;\n BEGIN\n FOR rec IN EXECUTE \'EXPLAIN \' || query LOOP\n ROWS := SUBSTRING(rec."QUERY PLAN" FROM \' rows=([[:digit:]]+)\');\n EXIT WHEN ROWS IS NOT NULL;\n END LOOP;\n\n RETURN ROWS - 1;\n END\n $func$ LANGUAGE plpgsql;\n ', + reverse_sql='DROP FUNCTION count_estimate();', + ), + ] diff --git a/share/migrations/0002_initial.py b/share/migrations/0002_initial.py deleted file mode 100644 index b3b746e62..000000000 --- a/share/migrations/0002_initial.py +++ /dev/null @@ -1,2252 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-11-17 01:26 -from __future__ import unicode_literals - -from django.conf import settings -import django.contrib.postgres.fields.jsonb -from django.db import migrations, models -import django.db.models.deletion -import share.models.fields -import share.models.validators - - -class Migration(migrations.Migration): - - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('share', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='AbstractAgent', - fields=[ - ('type', models.CharField(choices=[('share.agent', 'agent'), ('share.organization', 'organization'), ('share.consortium', 'consortium'), ('share.institution', 'institution'), ('share.person', 'person')], db_index=True, max_length=255)), - ('id', models.AutoField(primary_key=True, serialize=False)), - ('name', models.TextField(blank=True)), - ('location', models.TextField(blank=True)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('additional_name', models.TextField(blank=True, null=True)), - ('given_name', models.TextField(blank=True, null=True)), - ('family_name', models.TextField(blank=True, null=True)), - ('suffix', models.TextField(blank=True, null=True)), - ], - options={ - 'db_table': 'share_agent', - }, - ), - migrations.CreateModel( - name='AbstractAgentRelation', - fields=[ - ('type', models.CharField(choices=[('share.agentrelation', 'agent relation'), ('share.isaffiliatedwith', 'is affiliated with'), ('share.ismemberof', 'is member of'), ('share.isemployedby', 'is employed by')], db_index=True, max_length=255)), - ('id', models.AutoField(primary_key=True, serialize=False)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ], - options={ - 'db_table': 'share_agentrelation', - }, - ), - migrations.CreateModel( - name='AbstractAgentRelationVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('type', models.CharField(choices=[('share.agentrelationversion', 'agent relation version'), ('share.isaffiliatedwithversion', 'is affiliated with version'), ('share.ismemberofversion', 'is member of version'), ('share.isemployedbyversion', 'is employed by version')], db_index=True, max_length=255)), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ], - options={ - 'db_table': 'share_agentrelationversion', - }, - ), - migrations.CreateModel( - name='AbstractAgentVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('type', models.CharField(choices=[('share.agentversion', 'agent version'), ('share.organizationversion', 'organization version'), ('share.consortiumversion', 'consortium version'), ('share.institutionversion', 'institution version'), ('share.personversion', 'person version')], db_index=True, max_length=255)), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('name', models.TextField(blank=True)), - ('location', models.TextField(blank=True)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('additional_name', models.TextField(blank=True, null=True)), - ('given_name', models.TextField(blank=True, null=True)), - ('family_name', models.TextField(blank=True, null=True)), - ('suffix', models.TextField(blank=True, null=True)), - ], - options={ - 'db_table': 'share_agentversion', - }, - ), - migrations.CreateModel( - name='AbstractAgentWorkRelation', - fields=[ - ('type', models.CharField(choices=[('share.agentworkrelation', 'agent work relation'), ('share.host', 'host'), ('share.contributor', 'contributor'), ('share.principalinvestigator', 'principal investigator'), ('share.creator', 'creator'), ('share.publisher', 'publisher'), ('share.funder', 'funder')], db_index=True, max_length=255)), - ('id', models.AutoField(primary_key=True, serialize=False)), - ('cited_as', models.TextField(blank=True)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('order_cited', models.PositiveIntegerField(null=True)), - ('agent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='work_relations', to='share.AbstractAgent')), - ('agent_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion')), - ], - options={ - 'db_table': 'share_agentworkrelation', - }, - ), - migrations.CreateModel( - name='AbstractAgentWorkRelationVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('type', models.CharField(choices=[('share.agentworkrelationversion', 'agent work relation version'), ('share.hostversion', 'host version'), ('share.contributorversion', 'contributor version'), ('share.principalinvestigatorversion', 'principal investigator version'), ('share.creatorversion', 'creator version'), ('share.publisherversion', 'publisher version'), ('share.funderversion', 'funder version')], db_index=True, max_length=255)), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('cited_as', models.TextField(blank=True)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('order_cited', models.PositiveIntegerField(null=True)), - ('agent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent')), - ('agent_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion')), - ], - options={ - 'db_table': 'share_agentworkrelationversion', - }, - ), - migrations.CreateModel( - name='AbstractCreativeWork', - fields=[ - ('type', models.CharField(choices=[('share.creativework', 'creative work'), ('share.patent', 'patent'), ('share.dataset', 'data set'), ('share.presentation', 'presentation'), ('share.software', 'software'), ('share.poster', 'poster'), ('share.publication', 'publication'), ('share.preprint', 'preprint'), ('share.project', 'project'), ('share.registration', 'registration'), ('share.article', 'article'), ('share.book', 'book'), ('share.thesis', 'thesis'), ('share.report', 'report'), ('share.conferencepaper', 'conference paper'), ('share.dissertation', 'dissertation'), ('share.workingpaper', 'working paper')], db_index=True, max_length=255)), - ('id', models.AutoField(primary_key=True, serialize=False)), - ('title', models.TextField(blank=True)), - ('description', models.TextField(blank=True)), - ('is_deleted', models.BooleanField(default=False, help_text='Determines whether or not this record will be discoverable via search.')), - ('date_published', models.DateTimeField(null=True)), - ('date_updated', models.DateTimeField(null=True)), - ('free_to_read_type', share.models.fields.ShareURLField(blank=True)), - ('free_to_read_date', models.DateTimeField(null=True)), - ('rights', models.TextField(blank=True, null=True)), - ('language', models.TextField(blank=True, help_text='The ISO 3166-1 alpha-2 country code indicating the language of this record.', null=True)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ], - options={ - 'db_table': 'share_creativework', - }, - ), - migrations.CreateModel( - name='AbstractCreativeWorkVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('type', models.CharField(choices=[('share.creativeworkversion', 'creative work version'), ('share.patentversion', 'patent version'), ('share.datasetversion', 'data set version'), ('share.presentationversion', 'presentation version'), ('share.softwareversion', 'software version'), ('share.posterversion', 'poster version'), ('share.publicationversion', 'publication version'), ('share.preprintversion', 'preprint version'), ('share.projectversion', 'project version'), ('share.registrationversion', 'registration version'), ('share.articleversion', 'article version'), ('share.bookversion', 'book version'), ('share.thesisversion', 'thesis version'), ('share.reportversion', 'report version'), ('share.conferencepaperversion', 'conference paper version'), ('share.dissertationversion', 'dissertation version'), ('share.workingpaperversion', 'working paper version')], db_index=True, max_length=255)), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('title', models.TextField(blank=True)), - ('description', models.TextField(blank=True)), - ('is_deleted', models.BooleanField(default=False, help_text='Determines whether or not this record will be discoverable via search.')), - ('date_published', models.DateTimeField(null=True)), - ('date_updated', models.DateTimeField(null=True)), - ('free_to_read_type', share.models.fields.ShareURLField(blank=True)), - ('free_to_read_date', models.DateTimeField(null=True)), - ('rights', models.TextField(blank=True, null=True)), - ('language', models.TextField(blank=True, help_text='The ISO 3166-1 alpha-2 country code indicating the language of this record.', null=True)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ], - options={ - 'db_table': 'share_creativeworkversion', - }, - ), - migrations.CreateModel( - name='AbstractWorkRelation', - fields=[ - ('type', models.CharField(choices=[('share.workrelation', 'work relation'), ('share.isidenticalto', 'is identical to'), ('share.reviews', 'reviews'), ('share.isderivedfrom', 'is derived from'), ('share.ispartof', 'is part of'), ('share.references', 'references'), ('share.compiles', 'compiles'), ('share.issupplementto', 'is supplement to'), ('share.documents', 'documents'), ('share.extends', 'extends'), ('share.cites', 'cites')], db_index=True, max_length=255)), - ('id', models.AutoField(primary_key=True, serialize=False)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ], - options={ - 'db_table': 'share_workrelation', - }, - ), - migrations.CreateModel( - name='AbstractWorkRelationVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('type', models.CharField(choices=[('share.workrelationversion', 'work relation version'), ('share.isidenticaltoversion', 'is identical to version'), ('share.reviewsversion', 'reviews version'), ('share.isderivedfromversion', 'is derived from version'), ('share.ispartofversion', 'is part of version'), ('share.referencesversion', 'references version'), ('share.compilesversion', 'compiles version'), ('share.issupplementtoversion', 'is supplement to version'), ('share.documentsversion', 'documents version'), ('share.extendsversion', 'extends version'), ('share.citesversion', 'cites version')], db_index=True, max_length=255)), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ], - options={ - 'db_table': 'share_workrelationversion', - }, - ), - migrations.CreateModel( - name='AgentIdentifier', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('uri', share.models.fields.ShareURLField(unique=True)), - ('host', models.TextField(editable=False)), - ('scheme', models.TextField(editable=False)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('agent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='identifiers', to='share.AbstractAgent')), - ('agent_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion')), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='AgentIdentifierVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('uri', share.models.fields.ShareURLField()), - ('host', models.TextField(editable=False)), - ('scheme', models.TextField(editable=False)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('agent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent')), - ('agent_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion')), - ], - options={ - 'ordering': ('-date_modified',), - 'abstract': False, - }, - ), - migrations.CreateModel( - name='Award', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('name', models.TextField(blank=True)), - ('description', models.TextField(blank=True)), - ('uri', share.models.fields.ShareURLField(unique=True)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='AwardVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('name', models.TextField(blank=True)), - ('description', models.TextField(blank=True)), - ('uri', share.models.fields.ShareURLField()), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ], - options={ - 'ordering': ('-date_modified',), - 'abstract': False, - }, - ), - migrations.CreateModel( - name='CeleryTask', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('type', models.CharField(choices=[('share.celeryprovidertask', 'celery provider task')], db_index=True, max_length=255)), - ('uuid', models.UUIDField(db_index=True, unique=True)), - ('name', models.TextField(blank=True, db_index=True)), - ('args', models.TextField(blank=True)), - ('kwargs', models.TextField(blank=True)), - ('timestamp', models.DateTimeField(auto_now_add=True, db_index=True)), - ('status', models.IntegerField(choices=[(0, 'started'), (1, 'retried'), (2, 'failed'), (3, 'succeeded')])), - ('app_label', models.TextField(blank=True, db_index=True, null=True)), - ('app_version', models.TextField(blank=True, db_index=True, null=True)), - ('provider', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='provider', to=settings.AUTH_USER_MODEL)), - ('started_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='started_by', to=settings.AUTH_USER_MODEL)), - ], - options={ - 'ordering': ('-timestamp',), - }, - ), - migrations.CreateModel( - name='Change', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('change', django.contrib.postgres.fields.jsonb.JSONField()), - ('node_id', models.TextField(db_index=True)), - ('type', models.IntegerField(choices=[(0, 'create'), (1, 'merge'), (2, 'update')], editable=False)), - ('target_id', models.PositiveIntegerField(null=True)), - ('target_version_id', models.PositiveIntegerField(null=True)), - ], - options={ - 'ordering': ('pk',), - }, - ), - migrations.CreateModel( - name='ChangeSet', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('status', models.IntegerField(choices=[(0, 'pending'), (1, 'accepted'), (2, 'rejected')], default=0)), - ('submitted_at', models.DateTimeField(auto_now_add=True)), - ], - ), - migrations.CreateModel( - name='ExtraData', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('data', share.models.fields.DateTimeAwareJSONField(default=dict)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_extradata', to='share.Change')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraData')), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='ExtraDataVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('data', share.models.fields.DateTimeAwareJSONField(default=dict)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_extradataversion', to='share.Change')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraData')), - ('same_as_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion')), - ], - options={ - 'db_table': None, - 'abstract': False, - }, - ), - migrations.CreateModel( - name='NormalizedData', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('created_at', models.DateTimeField(null=True)), - ('data', share.models.fields.DateTimeAwareJSONField(validators=[share.models.validators.JSONLDValidator()])), - ('raw', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.RawData')), - ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - ), - migrations.CreateModel( - name='Subject', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.TextField(db_index=True, unique=True)), - ('parent', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.Subject')), - ], - ), - migrations.CreateModel( - name='Tag', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('name', models.TextField(unique=True)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_tag', to='share.Change')), - ('extra', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.Tag')), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='TagVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('name', models.TextField()), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_tagversion', to='share.Change')), - ('extra', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.Tag')), - ('same_as_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.TagVersion')), - ], - options={ - 'ordering': ('-date_modified',), - 'abstract': False, - }, - ), - migrations.CreateModel( - name='ThroughAwards', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('award', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='share.Award')), - ('award_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AwardVersion')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_throughawards', to='share.Change')), - ('extra', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion')), - ('funder', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='share.AbstractAgentWorkRelation')), - ('funder_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughAwards')), - ], - ), - migrations.CreateModel( - name='ThroughAwardsVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('award', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.Award')), - ('award_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AwardVersion')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_throughawardsversion', to='share.Change')), - ('extra', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion')), - ('funder', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelation')), - ('funder_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughAwards')), - ('same_as_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughAwardsVersion')), - ], - options={ - 'db_table': None, - }, - ), - migrations.CreateModel( - name='ThroughContributor', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_throughcontributor', to='share.Change')), - ('extra', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion')), - ('related', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelation')), - ('related_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughContributor')), - ], - ), - migrations.CreateModel( - name='ThroughContributorVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_throughcontributorversion', to='share.Change')), - ('extra', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion')), - ('related', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelation')), - ('related_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughContributor')), - ('same_as_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughContributorVersion')), - ('subject', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelation')), - ('subject_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion')), - ], - options={ - 'db_table': None, - }, - ), - migrations.CreateModel( - name='ThroughSubjects', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_throughsubjects', to='share.Change')), - ('creative_work', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='subject_relations', to='share.AbstractCreativeWork')), - ('creative_work_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion')), - ('extra', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughSubjects')), - ], - ), - migrations.CreateModel( - name='ThroughSubjectsVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_throughsubjectsversion', to='share.Change')), - ('creative_work', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork')), - ('creative_work_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion')), - ('extra', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughSubjects')), - ('same_as_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughSubjectsVersion')), - ('subject', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.Subject')), - ], - options={ - 'db_table': None, - }, - ), - migrations.CreateModel( - name='ThroughTags', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_throughtags', to='share.Change')), - ('creative_work', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tag_relations', to='share.AbstractCreativeWork')), - ('creative_work_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion')), - ('extra', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughTags')), - ], - ), - migrations.CreateModel( - name='ThroughTagsVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_throughtagsversion', to='share.Change')), - ('creative_work', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork')), - ('creative_work_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion')), - ('extra', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughTags')), - ('same_as_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughTagsVersion')), - ('tag', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.Tag')), - ('tag_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.TagVersion')), - ], - options={ - 'db_table': None, - }, - ), - migrations.CreateModel( - name='WorkIdentifier', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('uri', share.models.fields.ShareURLField(unique=True)), - ('host', models.TextField(editable=False)), - ('scheme', models.TextField(editable=False, help_text='A prefix to URI indicating how the following data should be interpreted.')), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_workidentifier', to='share.Change')), - ('creative_work', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='identifiers', to='share.AbstractCreativeWork')), - ('creative_work_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion')), - ('extra', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.WorkIdentifier')), - ], - options={ - 'abstract': False, - }, - ), - migrations.CreateModel( - name='WorkIdentifierVersion', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('action', models.TextField(max_length=10)), - ('persistent_id', models.PositiveIntegerField()), - ('uri', share.models.fields.ShareURLField()), - ('host', models.TextField(editable=False)), - ('scheme', models.TextField(editable=False, help_text='A prefix to URI indicating how the following data should be interpreted.')), - ('date_created', models.DateTimeField(auto_now_add=True, help_text='The date of ingress to SHARE.')), - ('date_modified', models.DateTimeField(auto_now=True, db_index=True, help_text='The date this record was modified by SHARE.')), - ('change', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_workidentifierversion', to='share.Change')), - ('creative_work', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork')), - ('creative_work_version', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion')), - ('extra', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData')), - ('extra_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion')), - ('same_as', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.WorkIdentifier')), - ('same_as_version', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.WorkIdentifierVersion')), - ], - options={ - 'ordering': ('-date_modified',), - 'abstract': False, - }, - ), - migrations.AddField( - model_name='workidentifier', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.WorkIdentifierVersion'), - ), - migrations.AddField( - model_name='workidentifier', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_workidentifier', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='workidentifier', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_workidentifier_version', to='share.WorkIdentifierVersion'), - ), - migrations.AddField( - model_name='throughtags', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughTagsVersion'), - ), - migrations.AddField( - model_name='throughtags', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_throughtags', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='throughtags', - name='tag', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='work_relations', to='share.Tag'), - ), - migrations.AddField( - model_name='throughtags', - name='tag_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.TagVersion'), - ), - migrations.AddField( - model_name='throughtags', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_throughtags_version', to='share.ThroughTagsVersion'), - ), - migrations.AddField( - model_name='throughsubjects', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughSubjectsVersion'), - ), - migrations.AddField( - model_name='throughsubjects', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_throughsubjects', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='throughsubjects', - name='subject', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='work_relations', to='share.Subject'), - ), - migrations.AddField( - model_name='throughsubjects', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_throughsubjects_version', to='share.ThroughSubjectsVersion'), - ), - migrations.AddField( - model_name='throughcontributor', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughContributorVersion'), - ), - migrations.AddField( - model_name='throughcontributor', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_throughcontributor', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='throughcontributor', - name='subject', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AddField( - model_name='throughcontributor', - name='subject_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AddField( - model_name='throughcontributor', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_throughcontributor_version', to='share.ThroughContributorVersion'), - ), - migrations.AddField( - model_name='throughawards', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughAwardsVersion'), - ), - migrations.AddField( - model_name='throughawards', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_throughawards', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='throughawards', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_throughawards_version', to='share.ThroughAwardsVersion'), - ), - migrations.AddField( - model_name='tag', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.TagVersion'), - ), - migrations.AddField( - model_name='tag', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_tag', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='tag', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_tag_version', to='share.TagVersion'), - ), - migrations.AddField( - model_name='extradata', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='extradata', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_extradata', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='extradata', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_extradata_version', to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='changeset', - name='normalized_data', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='share.NormalizedData'), - ), - migrations.AddField( - model_name='change', - name='change_set', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='changes', to='share.ChangeSet'), - ), - migrations.AddField( - model_name='change', - name='model_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.ContentType'), - ), - migrations.AddField( - model_name='change', - name='target_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='target_change', to='contenttypes.ContentType'), - ), - migrations.AddField( - model_name='change', - name='target_version_type', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='target_version_change', to='contenttypes.ContentType'), - ), - migrations.AddField( - model_name='awardversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_awardversion', to='share.Change'), - ), - migrations.AddField( - model_name='awardversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='awardversion', - name='extra_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='awardversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.Award'), - ), - migrations.AddField( - model_name='awardversion', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AwardVersion'), - ), - migrations.AddField( - model_name='award', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_award', to='share.Change'), - ), - migrations.AddField( - model_name='award', - name='extra', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='award', - name='extra_version', - field=models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='award', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.Award'), - ), - migrations.AddField( - model_name='award', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AwardVersion'), - ), - migrations.AddField( - model_name='award', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_award', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='award', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_award_version', to='share.AwardVersion'), - ), - migrations.AddField( - model_name='agentidentifierversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_agentidentifierversion', to='share.Change'), - ), - migrations.AddField( - model_name='agentidentifierversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='agentidentifierversion', - name='extra_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='agentidentifierversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AgentIdentifier'), - ), - migrations.AddField( - model_name='agentidentifierversion', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AgentIdentifierVersion'), - ), - migrations.AddField( - model_name='agentidentifier', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_agentidentifier', to='share.Change'), - ), - migrations.AddField( - model_name='agentidentifier', - name='extra', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='agentidentifier', - name='extra_version', - field=models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='agentidentifier', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AgentIdentifier'), - ), - migrations.AddField( - model_name='agentidentifier', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AgentIdentifierVersion'), - ), - migrations.AddField( - model_name='agentidentifier', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_agentidentifier', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='agentidentifier', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_agentidentifier_version', to='share.AgentIdentifierVersion'), - ), - migrations.AddField( - model_name='abstractworkrelationversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_abstractworkrelationversion', to='share.Change'), - ), - migrations.AddField( - model_name='abstractworkrelationversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='abstractworkrelationversion', - name='extra_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='abstractworkrelationversion', - name='related', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractworkrelationversion', - name='related_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractworkrelationversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractWorkRelation'), - ), - migrations.AddField( - model_name='abstractworkrelationversion', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractWorkRelationVersion'), - ), - migrations.AddField( - model_name='abstractworkrelationversion', - name='subject', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractworkrelationversion', - name='subject_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_abstractworkrelation', to='share.Change'), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='extra', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='extra_version', - field=models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='related', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='incoming_creative_work_relations', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='related_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractWorkRelation'), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractWorkRelationVersion'), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_abstractworkrelation', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='subject', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='outgoing_creative_work_relations', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='subject_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractworkrelation', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_abstractworkrelation_version', to='share.AbstractWorkRelationVersion'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_abstractcreativeworkversion', to='share.Change'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='extra_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='related_agent_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractcreativeworkversion_related_agent_versions_+', through='share.AbstractAgentWorkRelation', to='share.AbstractAgentVersion'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='related_agents', - field=share.models.fields.TypedManyToManyField(related_name='_abstractcreativeworkversion_related_agents_+', through='share.AbstractAgentWorkRelation', to='share.AbstractAgent'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='related_work_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractcreativeworkversion_related_work_versions_+', through='share.AbstractWorkRelation', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='related_works', - field=share.models.fields.TypedManyToManyField(related_name='_abstractcreativeworkversion_related_works_+', through='share.AbstractWorkRelation', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='subjects', - field=share.models.fields.TypedManyToManyField(related_name='_abstractcreativeworkversion_subjects_+', through='share.ThroughSubjects', to='share.Subject'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='tag_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractcreativeworkversion_tag_versions_+', through='share.ThroughTags', to='share.TagVersion'), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='tags', - field=share.models.fields.TypedManyToManyField(related_name='_abstractcreativeworkversion_tags_+', through='share.ThroughTags', to='share.Tag'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_abstractcreativework', to='share.Change'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='extra', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='extra_version', - field=models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='related_agent_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractcreativework_related_agent_versions_+', through='share.AbstractAgentWorkRelation', to='share.AbstractAgentVersion'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='related_agents', - field=share.models.fields.TypedManyToManyField(through='share.AbstractAgentWorkRelation', to='share.AbstractAgent'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='related_work_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractcreativework_related_work_versions_+', through='share.AbstractWorkRelation', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='related_works', - field=share.models.fields.TypedManyToManyField(through='share.AbstractWorkRelation', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_abstractcreativework', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='abstractcreativework', - name='subjects', - field=share.models.fields.TypedManyToManyField(related_name='subjected_works', through='share.ThroughSubjects', to='share.Subject'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='tag_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractcreativework_tag_versions_+', through='share.ThroughTags', to='share.TagVersion'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='tags', - field=share.models.fields.TypedManyToManyField(related_name='tagged_works', through='share.ThroughTags', to='share.Tag'), - ), - migrations.AddField( - model_name='abstractcreativework', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_abstractcreativework_version', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='award_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractagentworkrelationversion_award_versions_+', through='share.ThroughAwards', to='share.AwardVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='awards', - field=share.models.fields.TypedManyToManyField(through='share.ThroughAwards', to='share.Award'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_abstractagentworkrelationversion', to='share.Change'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='contributed_throug_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractagentworkrelationversion_contributed_throug_versions_+', through='share.ThroughContributor', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='contributed_through', - field=share.models.fields.TypedManyToManyField(through='share.ThroughContributor', to='share.AbstractAgentWorkRelation'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='creative_work', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='creative_work_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='extra_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='award_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractagentworkrelation_award_versions_+', through='share.ThroughAwards', to='share.AwardVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='awards', - field=share.models.fields.TypedManyToManyField(through='share.ThroughAwards', to='share.Award'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_abstractagentworkrelation', to='share.Change'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='contributed_throug_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractagentworkrelation_contributed_throug_versions_+', through='share.ThroughContributor', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='contributed_through', - field=share.models.fields.TypedManyToManyField(through='share.ThroughContributor', to='share.AbstractAgentWorkRelation'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='creative_work', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='agent_relations', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='creative_work_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='extra', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='extra_version', - field=models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_abstractagentworkrelation', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_abstractagentworkrelation_version', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AddField( - model_name='abstractagentversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_abstractagentversion', to='share.Change'), - ), - migrations.AddField( - model_name='abstractagentversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='abstractagentversion', - name='extra_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='abstractagentversion', - name='related_agent_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractagentversion_related_agent_versions_+', through='share.AbstractAgentRelation', to='share.AbstractAgentVersion'), - ), - migrations.AddField( - model_name='abstractagentversion', - name='related_agents', - field=share.models.fields.TypedManyToManyField(related_name='_abstractagentversion_related_agents_+', through='share.AbstractAgentRelation', to='share.AbstractAgent'), - ), - migrations.AddField( - model_name='abstractagentversion', - name='related_work_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractagentversion_related_work_versions_+', through='share.AbstractAgentWorkRelation', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractagentversion', - name='related_works', - field=share.models.fields.TypedManyToManyField(related_name='_abstractagentversion_related_works_+', through='share.AbstractAgentWorkRelation', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractagentversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent'), - ), - migrations.AddField( - model_name='abstractagentversion', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AddField( - model_name='abstractagentrelationversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_abstractagentrelationversion', to='share.Change'), - ), - migrations.AddField( - model_name='abstractagentrelationversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='abstractagentrelationversion', - name='extra_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='abstractagentrelationversion', - name='related', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent'), - ), - migrations.AddField( - model_name='abstractagentrelationversion', - name='related_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AddField( - model_name='abstractagentrelationversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentRelation'), - ), - migrations.AddField( - model_name='abstractagentrelationversion', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentRelationVersion'), - ), - migrations.AddField( - model_name='abstractagentrelationversion', - name='subject', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent'), - ), - migrations.AddField( - model_name='abstractagentrelationversion', - name='subject_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_abstractagentrelation', to='share.Change'), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='extra', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='extra_version', - field=models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='related', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent'), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='related_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentRelation'), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentRelationVersion'), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_abstractagentrelation', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='subject', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent'), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='subject_version', - field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AddField( - model_name='abstractagentrelation', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_abstractagentrelation_version', to='share.AbstractAgentRelationVersion'), - ), - migrations.AddField( - model_name='abstractagent', - name='change', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='affected_abstractagent', to='share.Change'), - ), - migrations.AddField( - model_name='abstractagent', - name='extra', - field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraData'), - ), - migrations.AddField( - model_name='abstractagent', - name='extra_version', - field=models.OneToOneField(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AddField( - model_name='abstractagent', - name='related_agent_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractagent_related_agent_versions_+', through='share.AbstractAgentRelation', to='share.AbstractAgentVersion'), - ), - migrations.AddField( - model_name='abstractagent', - name='related_agents', - field=share.models.fields.TypedManyToManyField(through='share.AbstractAgentRelation', to='share.AbstractAgent'), - ), - migrations.AddField( - model_name='abstractagent', - name='related_work_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractagent_related_work_versions_+', through='share.AbstractAgentWorkRelation', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AddField( - model_name='abstractagent', - name='related_works', - field=share.models.fields.TypedManyToManyField(through='share.AbstractAgentWorkRelation', to='share.AbstractCreativeWork'), - ), - migrations.AddField( - model_name='abstractagent', - name='same_as', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent'), - ), - migrations.AddField( - model_name='abstractagent', - name='same_as_version', - field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AddField( - model_name='abstractagent', - name='sources', - field=models.ManyToManyField(editable=False, related_name='source_abstractagent', to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='abstractagent', - name='version', - field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='share_abstractagent_version', to='share.AbstractAgentVersion'), - ), - migrations.CreateModel( - name='Agent', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.abstractagent',), - ), - migrations.CreateModel( - name='AgentRelation', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.abstractagentrelation',), - ), - migrations.CreateModel( - name='AgentRelationVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.abstractagentrelationversion',), - ), - migrations.CreateModel( - name='AgentVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.abstractagentversion',), - ), - migrations.CreateModel( - name='AgentWorkRelation', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.abstractagentworkrelation',), - ), - migrations.CreateModel( - name='AgentWorkRelationVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.abstractagentworkrelationversion',), - ), - migrations.CreateModel( - name='CeleryProviderTask', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.celerytask',), - ), - migrations.CreateModel( - name='CreativeWork', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.abstractcreativework',), - ), - migrations.CreateModel( - name='CreativeWorkVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.abstractcreativeworkversion',), - ), - migrations.CreateModel( - name='WorkRelation', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.abstractworkrelation',), - ), - migrations.CreateModel( - name='WorkRelationVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.abstractworkrelationversion',), - ), - migrations.AlterUniqueTogether( - name='throughtags', - unique_together=set([('tag', 'creative_work')]), - ), - migrations.AlterUniqueTogether( - name='throughsubjects', - unique_together=set([('subject', 'creative_work')]), - ), - migrations.AlterUniqueTogether( - name='throughcontributor', - unique_together=set([('subject', 'related')]), - ), - migrations.AlterUniqueTogether( - name='throughawards', - unique_together=set([('funder', 'award')]), - ), - migrations.AddField( - model_name='normalizeddata', - name='tasks', - field=models.ManyToManyField(to='share.CeleryProviderTask'), - ), - migrations.AlterIndexTogether( - name='change', - index_together=set([('node_id', 'change_set', 'target_type'), ('target_type', 'target_id')]), - ), - migrations.AlterIndexTogether( - name='celerytask', - index_together=set([('type', 'name', 'app_label', 'timestamp')]), - ), - migrations.AlterUniqueTogether( - name='abstractworkrelation', - unique_together=set([('subject', 'related', 'type')]), - ), - migrations.AlterUniqueTogether( - name='abstractagentworkrelation', - unique_together=set([('agent', 'creative_work', 'type')]), - ), - migrations.AlterIndexTogether( - name='abstractagentversion', - index_together=set([('type', 'name')]), - ), - migrations.AlterUniqueTogether( - name='abstractagentrelation', - unique_together=set([('subject', 'related', 'type')]), - ), - migrations.AlterIndexTogether( - name='abstractagent', - index_together=set([('type', 'name')]), - ), - migrations.CreateModel( - name='Cites', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='CitesVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='Compiles', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='CompilesVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='Contributor', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentworkrelation',), - ), - migrations.CreateModel( - name='ContributorVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentworkrelationversion',), - ), - migrations.CreateModel( - name='DataSet', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativework',), - ), - migrations.CreateModel( - name='DataSetVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativeworkversion',), - ), - migrations.CreateModel( - name='Documents', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='DocumentsVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='Extends', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='ExtendsVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='Funder', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentworkrelation',), - ), - migrations.CreateModel( - name='FunderVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentworkrelationversion',), - ), - migrations.CreateModel( - name='Host', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentworkrelation',), - ), - migrations.CreateModel( - name='HostVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentworkrelationversion',), - ), - migrations.CreateModel( - name='IsAffiliatedWith', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentrelation',), - ), - migrations.CreateModel( - name='IsAffiliatedWithVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentrelationversion',), - ), - migrations.CreateModel( - name='IsDerivedFrom', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='IsDerivedFromVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='IsIdenticalTo', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='IsIdenticalToVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='IsPartOf', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='IsPartOfVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='IsSupplementTo', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='IsSupplementToVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='Organization', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agent',), - ), - migrations.CreateModel( - name='OrganizationVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentversion',), - ), - migrations.CreateModel( - name='Patent', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativework',), - ), - migrations.CreateModel( - name='PatentVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativeworkversion',), - ), - migrations.CreateModel( - name='Person', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agent',), - ), - migrations.CreateModel( - name='PersonVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentversion',), - ), - migrations.CreateModel( - name='Poster', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativework',), - ), - migrations.CreateModel( - name='PosterVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativeworkversion',), - ), - migrations.CreateModel( - name='Presentation', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativework',), - ), - migrations.CreateModel( - name='PresentationVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativeworkversion',), - ), - migrations.CreateModel( - name='Publication', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativework',), - ), - migrations.CreateModel( - name='PublicationVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativeworkversion',), - ), - migrations.CreateModel( - name='Publisher', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentworkrelation',), - ), - migrations.CreateModel( - name='PublisherVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.agentworkrelationversion',), - ), - migrations.CreateModel( - name='References', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='ReferencesVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='Reviews', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='ReviewsVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='Software', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativework',), - ), - migrations.CreateModel( - name='SoftwareVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativeworkversion',), - ), - migrations.AddField( - model_name='rawdata', - name='tasks', - field=models.ManyToManyField(to='share.CeleryProviderTask'), - ), - migrations.CreateModel( - name='Article', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publication',), - ), - migrations.CreateModel( - name='ArticleVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publicationversion',), - ), - migrations.CreateModel( - name='Book', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publication',), - ), - migrations.CreateModel( - name='BookVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publicationversion',), - ), - migrations.CreateModel( - name='ConferencePaper', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publication',), - ), - migrations.CreateModel( - name='ConferencePaperVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publicationversion',), - ), - migrations.CreateModel( - name='Consortium', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.organization',), - ), - migrations.CreateModel( - name='ConsortiumVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.organizationversion',), - ), - migrations.CreateModel( - name='Creator', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.contributor',), - ), - migrations.CreateModel( - name='CreatorVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.contributorversion',), - ), - migrations.CreateModel( - name='Dissertation', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publication',), - ), - migrations.CreateModel( - name='DissertationVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publicationversion',), - ), - migrations.CreateModel( - name='Institution', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.organization',), - ), - migrations.CreateModel( - name='InstitutionVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.organizationversion',), - ), - migrations.CreateModel( - name='IsEmployedBy', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.isaffiliatedwith',), - ), - migrations.CreateModel( - name='IsEmployedByVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.isaffiliatedwithversion',), - ), - migrations.CreateModel( - name='IsMemberOf', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.isaffiliatedwith',), - ), - migrations.CreateModel( - name='IsMemberOfVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.isaffiliatedwithversion',), - ), - migrations.CreateModel( - name='Preprint', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publication',), - ), - migrations.CreateModel( - name='PreprintVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publicationversion',), - ), - migrations.CreateModel( - name='PrincipalInvestigator', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.contributor',), - ), - migrations.CreateModel( - name='PrincipalInvestigatorVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.contributorversion',), - ), - migrations.CreateModel( - name='Project', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publication',), - ), - migrations.CreateModel( - name='ProjectVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publicationversion',), - ), - migrations.CreateModel( - name='Registration', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publication',), - ), - migrations.CreateModel( - name='RegistrationVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publicationversion',), - ), - migrations.CreateModel( - name='Report', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publication',), - ), - migrations.CreateModel( - name='ReportVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publicationversion',), - ), - migrations.CreateModel( - name='Thesis', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publication',), - ), - migrations.CreateModel( - name='ThesisVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publicationversion',), - ), - migrations.CreateModel( - name='WorkingPaper', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publication',), - ), - migrations.CreateModel( - name='WorkingPaperVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.publicationversion',), - ), - ] diff --git a/share/migrations/0003_create_share_user.py b/share/migrations/0003_create_share_user.py deleted file mode 100644 index 90135540f..000000000 --- a/share/migrations/0003_create_share_user.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-06-22 00:27 -from __future__ import unicode_literals - -from django.db import migrations -from django.conf import settings - -def create_share_robot_user(apps, schema_editor): - ShareUser = apps.get_model('share', 'ShareUser') - try: - ShareUser.objects.create_robot_user(username=settings.APPLICATION_USERNAME, robot='') - except AssertionError: - pass - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0002_initial'), - ('osf_oauth2_adapter', '0001_make_human_group') - ] - - operations = [ - migrations.RunPython(create_share_robot_user), - ] diff --git a/share/migrations/0004_unique_organization_institution_names.py b/share/migrations/0004_unique_organization_institution_names.py deleted file mode 100644 index 1449117a0..000000000 --- a/share/migrations/0004_unique_organization_institution_names.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-10-21 13:16 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0003_create_share_user'), - ] - - # Meant to be run on an empty database, so don't worry about existing duplicates - operations = [ - migrations.RunSQL( - "CREATE UNIQUE INDEX share_agent_unique_institution_organization_names ON share_agent (name) WHERE type in ('share.institution', 'share.organization', 'share.consortium');", - reverse_sql="DROP INDEX IF EXISTS share_agent_unique_institution_organization_names;" - ) - ] diff --git a/share/migrations/0005_update_trigger_migrations_20161117_0127.py b/share/migrations/0005_update_trigger_migrations_20161117_0127.py deleted file mode 100644 index 09e1d18c8..000000000 --- a/share/migrations/0005_update_trigger_migrations_20161117_0127.py +++ /dev/null @@ -1,128 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-11-17 01:27 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0003_create_share_user'), - ('share', '0004_unique_organization_institution_names'), - ] - - operations = [ - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_extradata_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_extradataversion(persistent_id, action, change_id, data, date_created, date_modified, same_as_id, same_as_version_id) VALUES (NEW.id, TG_OP, NEW.change_id, NEW.data, NEW.date_created, NEW.date_modified, NEW.same_as_id, NEW.same_as_version_id) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_extradata_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_extradata_change ON share_extradata;\n\n CREATE TRIGGER share_extradata_change\n BEFORE INSERT OR UPDATE ON share_extradata\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_extradata_change();', - reverse_sql='DROP TRIGGER share_extradata_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_tag_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_tagversion(persistent_id, action, change_id, date_created, date_modified, extra_id, extra_version_id, name, same_as_id, same_as_version_id) VALUES (NEW.id, TG_OP, NEW.change_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.name, NEW.same_as_id, NEW.same_as_version_id) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_tag_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_tag_change ON share_tag;\n\n CREATE TRIGGER share_tag_change\n BEFORE INSERT OR UPDATE ON share_tag\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_tag_change();', - reverse_sql='DROP TRIGGER share_tag_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_throughtags_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_throughtagsversion(persistent_id, action, change_id, creative_work_id, creative_work_version_id, date_created, date_modified, extra_id, extra_version_id, same_as_id, same_as_version_id, tag_id, tag_version_id) VALUES (NEW.id, TG_OP, NEW.change_id, NEW.creative_work_id, NEW.creative_work_version_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.same_as_id, NEW.same_as_version_id, NEW.tag_id, NEW.tag_version_id) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_throughtags_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_throughtags_change ON share_throughtags;\n\n CREATE TRIGGER share_throughtags_change\n BEFORE INSERT OR UPDATE ON share_throughtags\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_throughtags_change();', - reverse_sql='DROP TRIGGER share_throughtags_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_throughsubjects_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_throughsubjectsversion(persistent_id, action, change_id, creative_work_id, creative_work_version_id, date_created, date_modified, extra_id, extra_version_id, same_as_id, same_as_version_id, subject_id) VALUES (NEW.id, TG_OP, NEW.change_id, NEW.creative_work_id, NEW.creative_work_version_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.same_as_id, NEW.same_as_version_id, NEW.subject_id) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_throughsubjects_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_throughsubjects_change ON share_throughsubjects;\n\n CREATE TRIGGER share_throughsubjects_change\n BEFORE INSERT OR UPDATE ON share_throughsubjects\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_throughsubjects_change();', - reverse_sql='DROP TRIGGER share_throughsubjects_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_agent_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_agentversion(persistent_id, action, additional_name, change_id, date_created, date_modified, extra_id, extra_version_id, family_name, given_name, location, name, same_as_id, same_as_version_id, suffix, type) VALUES (NEW.id, TG_OP, NEW.additional_name, NEW.change_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.family_name, NEW.given_name, NEW.location, NEW.name, NEW.same_as_id, NEW.same_as_version_id, NEW.suffix, NEW.type) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_agent_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_agent_change ON share_agent;\n\n CREATE TRIGGER share_agent_change\n BEFORE INSERT OR UPDATE ON share_agent\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_agent_change();', - reverse_sql='DROP TRIGGER share_agent_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_creativework_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_creativeworkversion(persistent_id, action, change_id, date_created, date_modified, date_published, date_updated, description, extra_id, extra_version_id, free_to_read_date, free_to_read_type, is_deleted, language, rights, same_as_id, same_as_version_id, title, type) VALUES (NEW.id, TG_OP, NEW.change_id, NEW.date_created, NEW.date_modified, NEW.date_published, NEW.date_updated, NEW.description, NEW.extra_id, NEW.extra_version_id, NEW.free_to_read_date, NEW.free_to_read_type, NEW.is_deleted, NEW.language, NEW.rights, NEW.same_as_id, NEW.same_as_version_id, NEW.title, NEW.type) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_creativework_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_creativework_change ON share_creativework;\n\n CREATE TRIGGER share_creativework_change\n BEFORE INSERT OR UPDATE ON share_creativework\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_creativework_change();', - reverse_sql='DROP TRIGGER share_creativework_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_workidentifier_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_workidentifierversion(persistent_id, action, change_id, creative_work_id, creative_work_version_id, date_created, date_modified, extra_id, extra_version_id, host, same_as_id, same_as_version_id, scheme, uri) VALUES (NEW.id, TG_OP, NEW.change_id, NEW.creative_work_id, NEW.creative_work_version_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.host, NEW.same_as_id, NEW.same_as_version_id, NEW.scheme, NEW.uri) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_workidentifier_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_workidentifier_change ON share_workidentifier;\n\n CREATE TRIGGER share_workidentifier_change\n BEFORE INSERT OR UPDATE ON share_workidentifier\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_workidentifier_change();', - reverse_sql='DROP TRIGGER share_workidentifier_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_agentidentifier_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_agentidentifierversion(persistent_id, action, agent_id, agent_version_id, change_id, date_created, date_modified, extra_id, extra_version_id, host, same_as_id, same_as_version_id, scheme, uri) VALUES (NEW.id, TG_OP, NEW.agent_id, NEW.agent_version_id, NEW.change_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.host, NEW.same_as_id, NEW.same_as_version_id, NEW.scheme, NEW.uri) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_agentidentifier_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_agentidentifier_change ON share_agentidentifier;\n\n CREATE TRIGGER share_agentidentifier_change\n BEFORE INSERT OR UPDATE ON share_agentidentifier\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_agentidentifier_change();', - reverse_sql='DROP TRIGGER share_agentidentifier_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_agentworkrelation_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_agentworkrelationversion(persistent_id, action, agent_id, agent_version_id, change_id, cited_as, creative_work_id, creative_work_version_id, date_created, date_modified, extra_id, extra_version_id, order_cited, same_as_id, same_as_version_id, type) VALUES (NEW.id, TG_OP, NEW.agent_id, NEW.agent_version_id, NEW.change_id, NEW.cited_as, NEW.creative_work_id, NEW.creative_work_version_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.order_cited, NEW.same_as_id, NEW.same_as_version_id, NEW.type) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_agentworkrelation_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_agentworkrelation_change ON share_agentworkrelation;\n\n CREATE TRIGGER share_agentworkrelation_change\n BEFORE INSERT OR UPDATE ON share_agentworkrelation\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_agentworkrelation_change();', - reverse_sql='DROP TRIGGER share_agentworkrelation_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_throughcontributor_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_throughcontributorversion(persistent_id, action, change_id, date_created, date_modified, extra_id, extra_version_id, related_id, related_version_id, same_as_id, same_as_version_id, subject_id, subject_version_id) VALUES (NEW.id, TG_OP, NEW.change_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.related_id, NEW.related_version_id, NEW.same_as_id, NEW.same_as_version_id, NEW.subject_id, NEW.subject_version_id) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_throughcontributor_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_throughcontributor_change ON share_throughcontributor;\n\n CREATE TRIGGER share_throughcontributor_change\n BEFORE INSERT OR UPDATE ON share_throughcontributor\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_throughcontributor_change();', - reverse_sql='DROP TRIGGER share_throughcontributor_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_award_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_awardversion(persistent_id, action, change_id, date_created, date_modified, description, extra_id, extra_version_id, name, same_as_id, same_as_version_id, uri) VALUES (NEW.id, TG_OP, NEW.change_id, NEW.date_created, NEW.date_modified, NEW.description, NEW.extra_id, NEW.extra_version_id, NEW.name, NEW.same_as_id, NEW.same_as_version_id, NEW.uri) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_award_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_award_change ON share_award;\n\n CREATE TRIGGER share_award_change\n BEFORE INSERT OR UPDATE ON share_award\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_award_change();', - reverse_sql='DROP TRIGGER share_award_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_throughawards_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_throughawardsversion(persistent_id, action, award_id, award_version_id, change_id, date_created, date_modified, extra_id, extra_version_id, funder_id, funder_version_id, same_as_id, same_as_version_id) VALUES (NEW.id, TG_OP, NEW.award_id, NEW.award_version_id, NEW.change_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.funder_id, NEW.funder_version_id, NEW.same_as_id, NEW.same_as_version_id) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_throughawards_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_throughawards_change ON share_throughawards;\n\n CREATE TRIGGER share_throughawards_change\n BEFORE INSERT OR UPDATE ON share_throughawards\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_throughawards_change();', - reverse_sql='DROP TRIGGER share_throughawards_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_workrelation_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_workrelationversion(persistent_id, action, change_id, date_created, date_modified, extra_id, extra_version_id, related_id, related_version_id, same_as_id, same_as_version_id, subject_id, subject_version_id, type) VALUES (NEW.id, TG_OP, NEW.change_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.related_id, NEW.related_version_id, NEW.same_as_id, NEW.same_as_version_id, NEW.subject_id, NEW.subject_version_id, NEW.type) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_workrelation_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_workrelation_change ON share_workrelation;\n\n CREATE TRIGGER share_workrelation_change\n BEFORE INSERT OR UPDATE ON share_workrelation\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_workrelation_change();', - reverse_sql='DROP TRIGGER share_workrelation_change', - ), - migrations.RunSQL( - sql='CREATE OR REPLACE FUNCTION before_share_agentrelation_change() RETURNS trigger AS $$\n DECLARE\n vid INTEGER;\n BEGIN\n INSERT INTO share_agentrelationversion(persistent_id, action, change_id, date_created, date_modified, extra_id, extra_version_id, related_id, related_version_id, same_as_id, same_as_version_id, subject_id, subject_version_id, type) VALUES (NEW.id, TG_OP, NEW.change_id, NEW.date_created, NEW.date_modified, NEW.extra_id, NEW.extra_version_id, NEW.related_id, NEW.related_version_id, NEW.same_as_id, NEW.same_as_version_id, NEW.subject_id, NEW.subject_version_id, NEW.type) RETURNING (id) INTO vid;\n NEW.version_id = vid;\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;', - reverse_sql='DROP FUNCTION before_share_agentrelation_change();', - ), - migrations.RunSQL( - sql='DROP TRIGGER IF EXISTS share_agentrelation_change ON share_agentrelation;\n\n CREATE TRIGGER share_agentrelation_change\n BEFORE INSERT OR UPDATE ON share_agentrelation\n FOR EACH ROW\n EXECUTE PROCEDURE before_share_agentrelation_change();', - reverse_sql='DROP TRIGGER share_agentrelation_change', - ), - ] diff --git a/share/migrations/0006_auto_20161118_1707.py b/share/migrations/0006_auto_20161118_1707.py deleted file mode 100644 index 9e0754c53..000000000 --- a/share/migrations/0006_auto_20161118_1707.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-11-18 17:07 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0005_update_trigger_migrations_20161117_0127'), - ] - - operations = [ - migrations.AlterField( - model_name='abstractagent', - name='name', - field=models.TextField(blank=True, db_index=True), - ), - migrations.AlterIndexTogether( - name='change', - index_together=set([('target_type', 'target_id')]), - ), - ] diff --git a/share/migrations/0007_auto_20161122_1810.py b/share/migrations/0007_auto_20161122_1810.py deleted file mode 100644 index decb7f5c0..000000000 --- a/share/migrations/0007_auto_20161122_1810.py +++ /dev/null @@ -1,576 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-11-22 18:10 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0006_auto_20161118_1707'), - ] - - operations = [ - migrations.AlterField( - model_name='abstractagent', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagent', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagent', - name='type', - field=models.CharField(choices=[('share.agent', 'agent'), ('share.organization', 'organization'), ('share.consortium', 'consortium'), ('share.institution', 'institution'), ('share.person', 'person')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentRelationVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='type', - field=models.CharField(choices=[('share.agentrelation', 'agent relation'), ('share.isaffiliatedwith', 'is affiliated with'), ('share.isemployedby', 'is employed by'), ('share.ismemberof', 'is member of')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='related', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentRelationVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='subject', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='type', - field=models.CharField(choices=[('share.agentrelationversion', 'agent relation version'), ('share.isaffiliatedwithversion', 'is affiliated with version'), ('share.isemployedbyversion', 'is employed by version'), ('share.ismemberofversion', 'is member of version')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractagentversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentversion', - name='type', - field=models.CharField(choices=[('share.agentversion', 'agent version'), ('share.organizationversion', 'organization version'), ('share.consortiumversion', 'consortium version'), ('share.institutionversion', 'institution version'), ('share.personversion', 'person version')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='agent_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='type', - field=models.CharField(choices=[('share.agentworkrelation', 'agent work relation'), ('share.contributor', 'contributor'), ('share.creator', 'creator'), ('share.principalinvestigator', 'principal investigator'), ('share.funder', 'funder'), ('share.host', 'host'), ('share.publisher', 'publisher')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='agent', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='agent_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='creative_work', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='type', - field=models.CharField(choices=[('share.agentworkrelationversion', 'agent work relation version'), ('share.contributorversion', 'contributor version'), ('share.creatorversion', 'creator version'), ('share.principalinvestigatorversion', 'principal investigator version'), ('share.funderversion', 'funder version'), ('share.hostversion', 'host version'), ('share.publisherversion', 'publisher version')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='type', - field=models.CharField(choices=[('share.creativework', 'creative work'), ('share.dataset', 'data set'), ('share.patent', 'patent'), ('share.poster', 'poster'), ('share.presentation', 'presentation'), ('share.publication', 'publication'), ('share.article', 'article'), ('share.book', 'book'), ('share.conferencepaper', 'conference paper'), ('share.dissertation', 'dissertation'), ('share.preprint', 'preprint'), ('share.project', 'project'), ('share.registration', 'registration'), ('share.report', 'report'), ('share.thesis', 'thesis'), ('share.workingpaper', 'working paper'), ('share.software', 'software')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='type', - field=models.CharField(choices=[('share.creativeworkversion', 'creative work version'), ('share.datasetversion', 'data set version'), ('share.patentversion', 'patent version'), ('share.posterversion', 'poster version'), ('share.presentationversion', 'presentation version'), ('share.publicationversion', 'publication version'), ('share.articleversion', 'article version'), ('share.bookversion', 'book version'), ('share.conferencepaperversion', 'conference paper version'), ('share.dissertationversion', 'dissertation version'), ('share.preprintversion', 'preprint version'), ('share.projectversion', 'project version'), ('share.registrationversion', 'registration version'), ('share.reportversion', 'report version'), ('share.thesisversion', 'thesis version'), ('share.workingpaperversion', 'working paper version'), ('share.softwareversion', 'software version')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='type', - field=models.CharField(choices=[('share.workrelation', 'work relation'), ('share.cites', 'cites'), ('share.compiles', 'compiles'), ('share.documents', 'documents'), ('share.extends', 'extends'), ('share.isderivedfrom', 'is derived from'), ('share.isidenticalto', 'is identical to'), ('share.ispartof', 'is part of'), ('share.issupplementto', 'is supplement to'), ('share.references', 'references'), ('share.reviews', 'reviews')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='related', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='subject', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='type', - field=models.CharField(choices=[('share.workrelationversion', 'work relation version'), ('share.citesversion', 'cites version'), ('share.compilesversion', 'compiles version'), ('share.documentsversion', 'documents version'), ('share.extendsversion', 'extends version'), ('share.isderivedfromversion', 'is derived from version'), ('share.isidenticaltoversion', 'is identical to version'), ('share.ispartofversion', 'is part of version'), ('share.issupplementtoversion', 'is supplement to version'), ('share.referencesversion', 'references version'), ('share.reviewsversion', 'reviews version')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='agentidentifier', - name='agent_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='agentidentifier', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='agentidentifier', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AgentIdentifierVersion'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='agent', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='agent_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AgentIdentifierVersion'), - ), - migrations.AlterField( - model_name='award', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='award', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AwardVersion'), - ), - migrations.AlterField( - model_name='awardversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='awardversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AwardVersion'), - ), - migrations.AlterField( - model_name='change', - name='model_type', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.ContentType'), - ), - migrations.AlterField( - model_name='change', - name='node_id', - field=models.TextField(), - ), - migrations.AlterField( - model_name='change', - name='target_version_type', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='target_version_change', to='contenttypes.ContentType'), - ), - migrations.AlterField( - model_name='extradata', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='extradataversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='tag', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='tag', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.TagVersion'), - ), - migrations.AlterField( - model_name='tagversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='tagversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.TagVersion'), - ), - migrations.AlterField( - model_name='throughawards', - name='award_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AwardVersion'), - ), - migrations.AlterField( - model_name='throughawards', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughawards', - name='funder_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughawards', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughAwardsVersion'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='award', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.Award'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='award_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AwardVersion'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='funder', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='funder_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughAwardsVersion'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughContributorVersion'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='related', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughContributorVersion'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='subject', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughSubjectsVersion'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='creative_work', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughSubjectsVersion'), - ), - migrations.AlterField( - model_name='throughtags', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='throughtags', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughtags', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughTagsVersion'), - ), - migrations.AlterField( - model_name='throughtags', - name='tag_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.TagVersion'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='creative_work', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ThroughTagsVersion'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='tag', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.Tag'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='tag_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.TagVersion'), - ), - migrations.AlterField( - model_name='workidentifier', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='workidentifier', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='workidentifier', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.WorkIdentifierVersion'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='creative_work', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.WorkIdentifierVersion'), - ), - ] diff --git a/share/migrations/0008_auto_20161207_1535.py b/share/migrations/0008_auto_20161207_1535.py deleted file mode 100644 index 5f2063b14..000000000 --- a/share/migrations/0008_auto_20161207_1535.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-12-07 15:35 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion -import share.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0007_auto_20161122_1810'), - ] - - operations = [ - migrations.RemoveField( - model_name='abstractagentworkrelation', - name='contributed_throug_versions', - ), - migrations.RemoveField( - model_name='abstractagentworkrelationversion', - name='contributed_throug_versions', - ), - migrations.AddField( - model_name='abstractagentworkrelation', - name='contributed_through_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractagentworkrelation_contributed_through_versions_+', through='share.ThroughContributor', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AddField( - model_name='abstractagentworkrelationversion', - name='contributed_through_versions', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='_abstractagentworkrelationversion_contributed_through_versions_+', through='share.ThroughContributor', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractagent', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='related', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='incoming_agent_relations', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='subject', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='outgoing_agent_relations', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='agentidentifier', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='award', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='tag', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughawards', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='subject', - field=models.ForeignKey(db_index=False, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.Subject'), - ), - migrations.AlterField( - model_name='throughtags', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='workidentifier', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterIndexTogether( - name='change', - index_together=set([]), - ), - ] diff --git a/share/migrations/0009_auto_20161209_1945.py b/share/migrations/0009_auto_20161209_1945.py deleted file mode 100644 index 70d6f1a32..000000000 --- a/share/migrations/0009_auto_20161209_1945.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-12-09 19:45 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0008_auto_20161207_1535'), - ] - - operations = [ - migrations.DeleteModel( - name='IsIdenticalTo', - ), - migrations.DeleteModel( - name='IsIdenticalToVersion', - ), - migrations.CreateModel( - name='Corrects', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='CorrectsVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='Discusses', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='DiscussesVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='Disputes', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='DisputesVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='RepliesTo', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='RepliesToVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - migrations.CreateModel( - name='Retraction', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativework',), - ), - migrations.CreateModel( - name='RetractionVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativeworkversion',), - ), - migrations.CreateModel( - name='Retracts', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelation',), - ), - migrations.CreateModel( - name='RetractsVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.workrelationversion',), - ), - ] diff --git a/share/migrations/0010_auto_20161212_1418_a.py b/share/migrations/0010_auto_20161212_1418_a.py deleted file mode 100644 index fa44b5b4a..000000000 --- a/share/migrations/0010_auto_20161212_1418_a.py +++ /dev/null @@ -1,228 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-12-12 14:18 -from __future__ import unicode_literals - -import db.deletion -from django.conf import settings -from django.db import migrations, models -import share.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0009_auto_20161209_1945'), - ] - - operations = [ - migrations.AlterField( - model_name='abstractagent', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_abstractagent', to='share.Change'), - ), - migrations.AlterField( - model_name='abstractagent', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='abstractagent', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagent', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagent', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagent', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_abstractagent', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='abstractagent', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_abstractagent_version', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_abstractagentrelation', to='share.Change'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='related', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='incoming_agent_relations', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentRelation'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentRelationVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_abstractagentrelation', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='subject', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='outgoing_agent_relations', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_abstractagentrelation_version', to='share.AbstractAgentRelationVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_abstractagentrelationversion', to='share.Change'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='related', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentRelation'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentRelationVersion'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='subject', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_abstractagentversion', to='share.Change'), - ), - migrations.AlterField( - model_name='abstractagentversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='abstractagentversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='agent', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='work_relations', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='agent_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_abstractagentworkrelation', to='share.Change'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='creative_work', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='agent_relations', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_abstractagentworkrelation', to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/share/migrations/0010_auto_20161212_1418_b.py b/share/migrations/0010_auto_20161212_1418_b.py deleted file mode 100644 index 9425ee8d7..000000000 --- a/share/migrations/0010_auto_20161212_1418_b.py +++ /dev/null @@ -1,228 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-12-12 14:18 -from __future__ import unicode_literals - -import db.deletion -from django.conf import settings -from django.db import migrations, models -import share.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0010_auto_20161212_1418_a'), - ] - - operations = [ - migrations.AlterField( - model_name='abstractagentworkrelation', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_abstractagentworkrelation_version', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='agent', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='agent_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_abstractagentworkrelationversion', to='share.Change'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='creative_work', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_abstractcreativework', to='share.Change'), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_abstractcreativework', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='type', - field=models.CharField(choices=[('share.creativework', 'creative work'), ('share.dataset', 'data set'), ('share.patent', 'patent'), ('share.poster', 'poster'), ('share.presentation', 'presentation'), ('share.publication', 'publication'), ('share.article', 'article'), ('share.book', 'book'), ('share.conferencepaper', 'conference paper'), ('share.dissertation', 'dissertation'), ('share.preprint', 'preprint'), ('share.project', 'project'), ('share.registration', 'registration'), ('share.report', 'report'), ('share.thesis', 'thesis'), ('share.workingpaper', 'working paper'), ('share.retraction', 'retraction'), ('share.software', 'software')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_abstractcreativework_version', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_abstractcreativeworkversion', to='share.Change'), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='type', - field=models.CharField(choices=[('share.creativeworkversion', 'creative work version'), ('share.datasetversion', 'data set version'), ('share.patentversion', 'patent version'), ('share.posterversion', 'poster version'), ('share.presentationversion', 'presentation version'), ('share.publicationversion', 'publication version'), ('share.articleversion', 'article version'), ('share.bookversion', 'book version'), ('share.conferencepaperversion', 'conference paper version'), ('share.dissertationversion', 'dissertation version'), ('share.preprintversion', 'preprint version'), ('share.projectversion', 'project version'), ('share.registrationversion', 'registration version'), ('share.reportversion', 'report version'), ('share.thesisversion', 'thesis version'), ('share.workingpaperversion', 'working paper version'), ('share.retractionversion', 'retraction version'), ('share.softwareversion', 'software version')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_abstractworkrelation', to='share.Change'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='related', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='incoming_creative_work_relations', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractWorkRelation'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_abstractworkrelation', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='subject', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='outgoing_creative_work_relations', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='type', - field=models.CharField(choices=[('share.workrelation', 'work relation'), ('share.cites', 'cites'), ('share.compiles', 'compiles'), ('share.corrects', 'corrects'), ('share.discusses', 'discusses'), ('share.disputes', 'disputes'), ('share.documents', 'documents'), ('share.extends', 'extends'), ('share.isderivedfrom', 'is derived from'), ('share.ispartof', 'is part of'), ('share.issupplementto', 'is supplement to'), ('share.references', 'references'), ('share.repliesto', 'replies to'), ('share.retracts', 'retracts'), ('share.reviews', 'reviews')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_abstractworkrelation_version', to='share.AbstractWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_abstractworkrelationversion', to='share.Change'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='related', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractWorkRelation'), - ), - ] diff --git a/share/migrations/0010_auto_20161212_1418_c.py b/share/migrations/0010_auto_20161212_1418_c.py deleted file mode 100644 index e26088838..000000000 --- a/share/migrations/0010_auto_20161212_1418_c.py +++ /dev/null @@ -1,228 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-12-12 14:18 -from __future__ import unicode_literals - -import db.deletion -from django.conf import settings -from django.db import migrations, models -import share.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0010_auto_20161212_1418_b'), - ] - - operations = [ - migrations.AlterField( - model_name='abstractworkrelationversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractWorkRelationVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='subject', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='type', - field=models.CharField(choices=[('share.workrelationversion', 'work relation version'), ('share.citesversion', 'cites version'), ('share.compilesversion', 'compiles version'), ('share.correctsversion', 'corrects version'), ('share.discussesversion', 'discusses version'), ('share.disputesversion', 'disputes version'), ('share.documentsversion', 'documents version'), ('share.extendsversion', 'extends version'), ('share.isderivedfromversion', 'is derived from version'), ('share.ispartofversion', 'is part of version'), ('share.issupplementtoversion', 'is supplement to version'), ('share.referencesversion', 'references version'), ('share.repliestoversion', 'replies to version'), ('share.retractsversion', 'retracts version'), ('share.reviewsversion', 'reviews version')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='agentidentifier', - name='agent', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='identifiers', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='agentidentifier', - name='agent_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='agentidentifier', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_agentidentifier', to='share.Change'), - ), - migrations.AlterField( - model_name='agentidentifier', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='agentidentifier', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='agentidentifier', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AgentIdentifier'), - ), - migrations.AlterField( - model_name='agentidentifier', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AgentIdentifierVersion'), - ), - migrations.AlterField( - model_name='agentidentifier', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_agentidentifier', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='agentidentifier', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_agentidentifier_version', to='share.AgentIdentifierVersion'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='agent', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='agent_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentVersion'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_agentidentifierversion', to='share.Change'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AgentIdentifier'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AgentIdentifierVersion'), - ), - migrations.AlterField( - model_name='award', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_award', to='share.Change'), - ), - migrations.AlterField( - model_name='award', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='award', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='award', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.Award'), - ), - migrations.AlterField( - model_name='award', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AwardVersion'), - ), - migrations.AlterField( - model_name='award', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_award', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='award', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_award_version', to='share.AwardVersion'), - ), - migrations.AlterField( - model_name='awardversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_awardversion', to='share.Change'), - ), - migrations.AlterField( - model_name='awardversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='awardversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='awardversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.Award'), - ), - migrations.AlterField( - model_name='awardversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AwardVersion'), - ), - migrations.AlterField( - model_name='extradata', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_extradata', to='share.Change'), - ), - migrations.AlterField( - model_name='extradata', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraData'), - ), - migrations.AlterField( - model_name='extradata', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='extradata', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_extradata', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='extradata', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_extradata_version', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='extradataversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_extradataversion', to='share.Change'), - ), - migrations.AlterField( - model_name='extradataversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraData'), - ), - migrations.AlterField( - model_name='extradataversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='tag', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_tag', to='share.Change'), - ), - migrations.AlterField( - model_name='tag', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - ] diff --git a/share/migrations/0010_auto_20161212_1418_d.py b/share/migrations/0010_auto_20161212_1418_d.py deleted file mode 100644 index 039ad982c..000000000 --- a/share/migrations/0010_auto_20161212_1418_d.py +++ /dev/null @@ -1,528 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-12-12 14:18 -from __future__ import unicode_literals - -import db.deletion -from django.conf import settings -from django.db import migrations, models -import share.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0010_auto_20161212_1418_c'), - ] - - operations = [ - migrations.AlterField( - model_name='tag', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='tag', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.Tag'), - ), - migrations.AlterField( - model_name='tag', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.TagVersion'), - ), - migrations.AlterField( - model_name='tag', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_tag', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='tag', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_tag_version', to='share.TagVersion'), - ), - migrations.AlterField( - model_name='tagversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_tagversion', to='share.Change'), - ), - migrations.AlterField( - model_name='tagversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='tagversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='tagversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.Tag'), - ), - migrations.AlterField( - model_name='tagversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.TagVersion'), - ), - migrations.AlterField( - model_name='throughawards', - name='award', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.Award'), - ), - migrations.AlterField( - model_name='throughawards', - name='award_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AwardVersion'), - ), - migrations.AlterField( - model_name='throughawards', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_throughawards', to='share.Change'), - ), - migrations.AlterField( - model_name='throughawards', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='throughawards', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughawards', - name='funder', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='throughawards', - name='funder_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughawards', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughAwards'), - ), - migrations.AlterField( - model_name='throughawards', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughAwardsVersion'), - ), - migrations.AlterField( - model_name='throughawards', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_throughawards', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='throughawards', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_throughawards_version', to='share.ThroughAwardsVersion'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='award', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.Award'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='award_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AwardVersion'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_throughawardsversion', to='share.Change'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='funder', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='funder_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughAwards'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughAwardsVersion'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_throughcontributor', to='share.Change'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='related', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughContributor'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughContributorVersion'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_throughcontributor', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='throughcontributor', - name='subject', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughcontributor', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_throughcontributor_version', to='share.ThroughContributorVersion'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_throughcontributorversion', to='share.Change'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='related', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='related_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughContributor'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughContributorVersion'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='subject', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='subject_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelationVersion'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_throughsubjects', to='share.Change'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='creative_work', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='subject_relations', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughSubjects'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughSubjectsVersion'), - ), - migrations.AlterField( - model_name='throughsubjects', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_throughsubjects', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='throughsubjects', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_throughsubjects_version', to='share.ThroughSubjectsVersion'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_throughsubjectsversion', to='share.Change'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='creative_work', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughSubjects'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughSubjectsVersion'), - ), - migrations.AlterField( - model_name='throughtags', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_throughtags', to='share.Change'), - ), - migrations.AlterField( - model_name='throughtags', - name='creative_work', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='tag_relations', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='throughtags', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='throughtags', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='throughtags', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughtags', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughTags'), - ), - migrations.AlterField( - model_name='throughtags', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughTagsVersion'), - ), - migrations.AlterField( - model_name='throughtags', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_throughtags', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='throughtags', - name='tag', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='work_relations', to='share.Tag'), - ), - migrations.AlterField( - model_name='throughtags', - name='tag_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.TagVersion'), - ), - migrations.AlterField( - model_name='throughtags', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_throughtags_version', to='share.ThroughTagsVersion'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_throughtagsversion', to='share.Change'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='creative_work', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughTags'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughTagsVersion'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='tag', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.Tag'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='tag_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.TagVersion'), - ), - migrations.AlterField( - model_name='workidentifier', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_workidentifier', to='share.Change'), - ), - migrations.AlterField( - model_name='workidentifier', - name='creative_work', - field=models.ForeignKey(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='identifiers', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='workidentifier', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='workidentifier', - name='extra', - field=models.OneToOneField(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='workidentifier', - name='extra_version', - field=models.OneToOneField(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='workidentifier', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.WorkIdentifier'), - ), - migrations.AlterField( - model_name='workidentifier', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.WorkIdentifierVersion'), - ), - migrations.AlterField( - model_name='workidentifier', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_workidentifier', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='workidentifier', - name='version', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='share_workidentifier_version', to='share.WorkIdentifierVersion'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='change', - field=models.OneToOneField(editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='affected_workidentifierversion', to='share.Change'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='creative_work', - field=models.ForeignKey(db_index=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='creative_work_version', - field=models.ForeignKey(db_index=False, editable=False, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWorkVersion'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='extra', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to='share.ExtraData'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='extra_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraDataVersion'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='same_as', - field=models.ForeignKey(null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.WorkIdentifier'), - ), - migrations.AlterField( - model_name='workidentifierversion', - name='same_as_version', - field=models.ForeignKey(db_index=False, editable=False, null=True, on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.WorkIdentifierVersion'), - ), - ] diff --git a/share/migrations/0010_auto_20161212_1418_e.py b/share/migrations/0010_auto_20161212_1418_e.py deleted file mode 100644 index f9772d5c2..000000000 --- a/share/migrations/0010_auto_20161212_1418_e.py +++ /dev/null @@ -1,88 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-12-12 14:18 -from __future__ import unicode_literals - -import db.deletion -from django.conf import settings -from django.db import migrations, models -import share.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0010_auto_20161212_1418_d'), - ] - - operations = [ - migrations.AlterField( - model_name='workidentifierversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.WorkIdentifier'), - ), - migrations.AlterField( - model_name='abstractagentrelationversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentRelation'), - ), - migrations.AlterField( - model_name='abstractagentversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgent'), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractAgentWorkRelation'), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractCreativeWork'), - ), - migrations.AlterField( - model_name='abstractworkrelationversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AbstractWorkRelation'), - ), - migrations.AlterField( - model_name='agentidentifierversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.AgentIdentifier'), - ), - migrations.AlterField( - model_name='awardversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.Award'), - ), - migrations.AlterField( - model_name='extradataversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ExtraData'), - ), - migrations.AlterField( - model_name='tagversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.Tag'), - ), - migrations.AlterField( - model_name='throughawardsversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughAwards'), - ), - migrations.AlterField( - model_name='throughcontributorversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughContributor'), - ), - migrations.AlterField( - model_name='throughsubjectsversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughSubjects'), - ), - migrations.AlterField( - model_name='throughtagsversion', - name='persistent_id', - field=models.ForeignKey(db_column='persistent_id', on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), related_name='+', to='share.ThroughTags'), - ), - ] diff --git a/share/migrations/0011_sitebanner.py b/share/migrations/0011_sitebanner.py deleted file mode 100644 index f8412de5b..000000000 --- a/share/migrations/0011_sitebanner.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-12-12 15:40 -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0010_auto_20161212_1418_e'), - ] - - operations = [ - migrations.CreateModel( - name='SiteBanner', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('active', models.BooleanField(db_index=True, default=True)), - ('title', models.CharField(max_length=300)), - ('description', models.TextField(blank=True)), - ('color', models.IntegerField(choices=[(0, 'success'), (1, 'info'), (2, 'warning'), (3, 'danger')], default=1)), - ('icon', models.CharField(blank=True, default='exclamation', max_length=31)), - ('created_at', models.DateTimeField(auto_now_add=True)), - ('last_modified_at', models.DateTimeField(auto_now=True)), - ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), - ('last_modified_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)), - ], - ), - ] diff --git a/share/migrations/0012_auto_20161212_1555.py b/share/migrations/0012_auto_20161212_1555.py deleted file mode 100644 index ccd83f6df..000000000 --- a/share/migrations/0012_auto_20161212_1555.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-12-12 15:55 -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations -import share.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0011_sitebanner'), - ] - - operations = [ - migrations.AlterField( - model_name='abstractagent', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_abstractagent', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='abstractagentrelation', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_abstractagentrelation', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_abstractagentworkrelation', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_abstractcreativework', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='abstractworkrelation', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_abstractworkrelation', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='agentidentifier', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_agentidentifier', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='award', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_award', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='extradata', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_extradata', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='tag', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_tag', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='throughawards', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_throughawards', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='throughcontributor', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_throughcontributor', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='throughsubjects', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_throughsubjects', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='throughtags', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_throughtags', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='workidentifier', - name='sources', - field=share.models.fields.TypedManyToManyField(editable=False, related_name='source_workidentifier', to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/share/migrations/0013_auto_20161214_1921.py b/share/migrations/0013_auto_20161214_1921.py deleted file mode 100644 index 94d36e497..000000000 --- a/share/migrations/0013_auto_20161214_1921.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-12-14 19:21 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0012_auto_20161212_1555'), - ] - - operations = [ - migrations.CreateModel( - name='Department', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.organization',), - ), - migrations.CreateModel( - name='DepartmentVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.organizationversion',), - ), - migrations.CreateModel( - name='PrincipalInvestigatorContact', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.principalinvestigator',), - ), - migrations.CreateModel( - name='PrincipalInvestigatorContactVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.principalinvestigatorversion',), - ), - migrations.AddField( - model_name='award', - name='award_amount', - field=models.PositiveIntegerField(blank=True, null=True), - ), - migrations.AddField( - model_name='award', - name='date', - field=models.DateTimeField(blank=True, null=True), - ), - migrations.AddField( - model_name='awardversion', - name='award_amount', - field=models.PositiveIntegerField(blank=True, null=True), - ), - migrations.AddField( - model_name='awardversion', - name='date', - field=models.DateTimeField(blank=True, null=True), - ), - migrations.AlterField( - model_name='abstractagent', - name='type', - field=models.CharField(choices=[('share.agent', 'agent'), ('share.organization', 'organization'), ('share.consortium', 'consortium'), ('share.department', 'department'), ('share.institution', 'institution'), ('share.person', 'person')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractagentversion', - name='type', - field=models.CharField(choices=[('share.agentversion', 'agent version'), ('share.organizationversion', 'organization version'), ('share.consortiumversion', 'consortium version'), ('share.departmentversion', 'department version'), ('share.institutionversion', 'institution version'), ('share.personversion', 'person version')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractagentworkrelation', - name='type', - field=models.CharField(choices=[('share.agentworkrelation', 'agent work relation'), ('share.contributor', 'contributor'), ('share.creator', 'creator'), ('share.principalinvestigator', 'principal investigator'), ('share.principalinvestigatorcontact', 'principal investigator contact'), ('share.funder', 'funder'), ('share.host', 'host'), ('share.publisher', 'publisher')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractagentworkrelationversion', - name='type', - field=models.CharField(choices=[('share.agentworkrelationversion', 'agent work relation version'), ('share.contributorversion', 'contributor version'), ('share.creatorversion', 'creator version'), ('share.principalinvestigatorversion', 'principal investigator version'), ('share.principalinvestigatorcontactversion', 'principal investigator contact version'), ('share.funderversion', 'funder version'), ('share.hostversion', 'host version'), ('share.publisherversion', 'publisher version')], db_index=True, max_length=255), - ), - ] diff --git a/share/migrations/0014_auto_20170112_2143.py b/share/migrations/0014_auto_20170112_2143.py deleted file mode 100644 index c37d03dc7..000000000 --- a/share/migrations/0014_auto_20170112_2143.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2017-01-12 21:43 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0013_auto_20161214_1921'), - ] - - operations = [ - migrations.AlterField( - model_name='normalizeddata', - name='created_at', - field=models.DateTimeField(auto_now_add=True, null=True), - ), - ] diff --git a/share/migrations/0014_auto_20170127_1423.py b/share/migrations/0014_auto_20170127_1423.py deleted file mode 100644 index f57cd6127..000000000 --- a/share/migrations/0014_auto_20170127_1423.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2017-01-27 14:23 -from __future__ import unicode_literals - -from django.db import migrations, models -import share.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0013_auto_20161214_1921'), - ] - - operations = [ - migrations.AlterField( - model_name='award', - name='uri', - field=share.models.fields.ShareURLField(blank=True, unique=True), - ), - migrations.AlterField( - model_name='awardversion', - name='uri', - field=share.models.fields.ShareURLField(blank=True), - ), - migrations.CreateModel( - name='Repository', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativework',), - ), - migrations.CreateModel( - name='RepositoryVersion', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('share.creativeworkversion',), - ), - migrations.AlterField( - model_name='abstractcreativework', - name='type', - field=models.CharField(choices=[('share.creativework', 'creative work'), ('share.dataset', 'data set'), ('share.patent', 'patent'), ('share.poster', 'poster'), ('share.presentation', 'presentation'), ('share.publication', 'publication'), ('share.article', 'article'), ('share.book', 'book'), ('share.conferencepaper', 'conference paper'), ('share.dissertation', 'dissertation'), ('share.preprint', 'preprint'), ('share.project', 'project'), ('share.registration', 'registration'), ('share.report', 'report'), ('share.thesis', 'thesis'), ('share.workingpaper', 'working paper'), ('share.repository', 'repository'), ('share.retraction', 'retraction'), ('share.software', 'software')], db_index=True, max_length=255), - ), - migrations.AlterField( - model_name='abstractcreativeworkversion', - name='type', - field=models.CharField(choices=[('share.creativeworkversion', 'creative work version'), ('share.datasetversion', 'data set version'), ('share.patentversion', 'patent version'), ('share.posterversion', 'poster version'), ('share.presentationversion', 'presentation version'), ('share.publicationversion', 'publication version'), ('share.articleversion', 'article version'), ('share.bookversion', 'book version'), ('share.conferencepaperversion', 'conference paper version'), ('share.dissertationversion', 'dissertation version'), ('share.preprintversion', 'preprint version'), ('share.projectversion', 'project version'), ('share.registrationversion', 'registration version'), ('share.reportversion', 'report version'), ('share.thesisversion', 'thesis version'), ('share.workingpaperversion', 'working paper version'), ('share.repositoryversion', 'repository version'), ('share.retractionversion', 'retraction version'), ('share.softwareversion', 'software version')], db_index=True, max_length=255), - ), - ] diff --git a/share/migrations/0015_auto_20170117_2050.py b/share/migrations/0015_auto_20170117_2050.py deleted file mode 100644 index ba1d93443..000000000 --- a/share/migrations/0015_auto_20170117_2050.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2017-01-17 20:50 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0014_auto_20170127_1423'), - ] - - operations = [ - migrations.AddField( - model_name='abstractcreativework', - name='justification', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='abstractcreativework', - name='registration_type', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='abstractcreativework', - name='withdrawn', - field=models.NullBooleanField(), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='justification', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='registration_type', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='abstractcreativeworkversion', - name='withdrawn', - field=models.NullBooleanField(), - ), - ] diff --git a/share/migrations/0016_auto_20170130_2130.py b/share/migrations/0016_auto_20170130_2130.py deleted file mode 100644 index ec70b0210..000000000 --- a/share/migrations/0016_auto_20170130_2130.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2017-01-30 21:30 -from __future__ import unicode_literals - -from django.db import migrations -import share.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0015_auto_20170117_2050'), - ] - - operations = [ - migrations.AlterField( - model_name='award', - name='uri', - field=share.models.fields.ShareURLField(blank=True, null=True, unique=True), - ), - migrations.AlterField( - model_name='awardversion', - name='uri', - field=share.models.fields.ShareURLField(blank=True, null=True), - ), - ] diff --git a/share/migrations/0016_merge.py b/share/migrations/0016_merge.py deleted file mode 100644 index 2e79bcceb..000000000 --- a/share/migrations/0016_merge.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2017-01-27 19:31 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0014_auto_20170127_1423'), - ('share', '0015_auto_20170117_2050'), - ] - - operations = [ - ] diff --git a/share/migrations/0017_merge.py b/share/migrations/0017_merge.py deleted file mode 100644 index 98756ce5f..000000000 --- a/share/migrations/0017_merge.py +++ /dev/null @@ -1,17 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2017-01-30 21:47 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0014_auto_20170112_2143'), - ('share', '0016_merge'), - ('share', '0016_auto_20170130_2130'), - ] - - operations = [ - ] diff --git a/share/migrations/0018_fuzzycount.py b/share/migrations/0018_fuzzycount.py deleted file mode 100644 index 7e56e1ae7..000000000 --- a/share/migrations/0018_fuzzycount.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2017-02-03 16:22 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0017_merge'), - ] - - operations = [ - migrations.RunSQL( - sql=''' - CREATE OR REPLACE FUNCTION count_estimate(query text) RETURNS INTEGER AS - $func$ - DECLARE - rec record; - ROWS INTEGER; - BEGIN - FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP - ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); - EXIT WHEN ROWS IS NOT NULL; - END LOOP; - - RETURN ROWS - 1; - END - $func$ LANGUAGE plpgsql; - ''', - reverse_sql='DROP FUNCTION count_estimate();', - ), - ] diff --git a/share/migrations/0018_store_favicons.py b/share/migrations/0018_store_favicons.py deleted file mode 100644 index 2dc239de2..000000000 --- a/share/migrations/0018_store_favicons.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2017-01-19 16:18 -from __future__ import unicode_literals - -import db.deletion -from django.conf import settings -from django.db import migrations, models -import share.models.core - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0017_merge'), - ] - - operations = [ - migrations.CreateModel( - name='FaviconImage', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('image', models.BinaryField()), - ], - ), - migrations.AddField( - model_name='shareuser', - name='favicon', - field=models.ImageField(null=True, storage=share.models.ingest.SourceIconStorage(), upload_to=share.models.ingest.icon_name), - ), - migrations.AddField( - model_name='faviconimage', - name='user', - field=models.OneToOneField(on_delete=db.deletion.DatabaseOnDelete(clause='CASCADE'), to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/share/migrations/0019_merge.py b/share/migrations/0019_merge.py deleted file mode 100644 index ace705353..000000000 --- a/share/migrations/0019_merge.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2017-02-03 19:26 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('share', '0018_fuzzycount'), - ('share', '0018_store_favicons'), - ] - - operations = [ - ] diff --git a/share/migrations/0023_auto_20170315_0241.py b/share/migrations/0023_auto_20170315_0241.py index df9d6ab60..52ddd73bf 100644 --- a/share/migrations/0023_auto_20170315_0241.py +++ b/share/migrations/0023_auto_20170315_0241.py @@ -42,7 +42,7 @@ def migrate_push_sources(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('share', '0019_merge'), + ('share', '0001_initial_squashed_0020_auto_20170206_2114'), ] operations = [ diff --git a/share/migrations/0057_unconcurrent_index.py b/share/migrations/0057_unconcurrent_index.py new file mode 100644 index 000000000..3991dc92e --- /dev/null +++ b/share/migrations/0057_unconcurrent_index.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2021-02-24 19:25 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('share', '0056_auto_20210217_1923'), + ] + + operations = [ + # no changes in the database, just trying to disentangle the `ConcurrentIndex` concept + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.RemoveIndex( + model_name='rawdatum', + name='share_rawda_no_outp_f0330f_idx', + ), + migrations.AddIndex( + model_name='rawdatum', + index=models.Index(fields=['no_output'], name='share_rawda_no_outp_f0330f_idx'), + ), + ], + ), + ] diff --git a/share/migrations/0058_big_rend.py b/share/migrations/0058_big_rend.py new file mode 100644 index 000000000..471114647 --- /dev/null +++ b/share/migrations/0058_big_rend.py @@ -0,0 +1,1702 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2021-02-17 19:27 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('share', '0057_unconcurrent_index'), + ] + + operations = [ + migrations.AlterIndexTogether( + name='abstractagent', + index_together=set([]), + ), + migrations.RemoveField( + model_name='abstractagent', + name='change', + ), + migrations.RemoveField( + model_name='abstractagent', + name='extra', + ), + migrations.RemoveField( + model_name='abstractagent', + name='extra_version', + ), + migrations.RemoveField( + model_name='abstractagent', + name='related_agent_versions', + ), + migrations.RemoveField( + model_name='abstractagent', + name='related_agents', + ), + migrations.RemoveField( + model_name='abstractagent', + name='related_work_versions', + ), + migrations.RemoveField( + model_name='abstractagent', + name='related_works', + ), + migrations.RemoveField( + model_name='abstractagent', + name='same_as', + ), + migrations.RemoveField( + model_name='abstractagent', + name='same_as_version', + ), + migrations.RemoveField( + model_name='abstractagent', + name='sources', + ), + migrations.RemoveField( + model_name='abstractagent', + name='version', + ), + migrations.AlterUniqueTogether( + name='abstractagentrelation', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='change', + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='extra', + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='extra_version', + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='related', + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='related_version', + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='same_as', + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='same_as_version', + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='sources', + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='subject', + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='subject_version', + ), + migrations.RemoveField( + model_name='abstractagentrelation', + name='version', + ), + migrations.RemoveField( + model_name='abstractagentrelationversion', + name='change', + ), + migrations.RemoveField( + model_name='abstractagentrelationversion', + name='extra', + ), + migrations.RemoveField( + model_name='abstractagentrelationversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='abstractagentrelationversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='abstractagentrelationversion', + name='related', + ), + migrations.RemoveField( + model_name='abstractagentrelationversion', + name='related_version', + ), + migrations.RemoveField( + model_name='abstractagentrelationversion', + name='same_as', + ), + migrations.RemoveField( + model_name='abstractagentrelationversion', + name='same_as_version', + ), + migrations.RemoveField( + model_name='abstractagentrelationversion', + name='subject', + ), + migrations.RemoveField( + model_name='abstractagentrelationversion', + name='subject_version', + ), + migrations.AlterIndexTogether( + name='abstractagentversion', + index_together=set([]), + ), + migrations.RemoveField( + model_name='abstractagentversion', + name='change', + ), + migrations.RemoveField( + model_name='abstractagentversion', + name='extra', + ), + migrations.RemoveField( + model_name='abstractagentversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='abstractagentversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='abstractagentversion', + name='related_agent_versions', + ), + migrations.RemoveField( + model_name='abstractagentversion', + name='related_agents', + ), + migrations.RemoveField( + model_name='abstractagentversion', + name='related_work_versions', + ), + migrations.RemoveField( + model_name='abstractagentversion', + name='related_works', + ), + migrations.RemoveField( + model_name='abstractagentversion', + name='same_as', + ), + migrations.RemoveField( + model_name='abstractagentversion', + name='same_as_version', + ), + migrations.AlterUniqueTogether( + name='abstractagentworkrelation', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='agent', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='agent_version', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='award_versions', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='awards', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='change', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='contributed_through', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='contributed_through_versions', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='creative_work', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='creative_work_version', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='extra', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='extra_version', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='same_as', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='same_as_version', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='sources', + ), + migrations.RemoveField( + model_name='abstractagentworkrelation', + name='version', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='agent', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='agent_version', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='award_versions', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='awards', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='change', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='contributed_through', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='contributed_through_versions', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='creative_work', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='creative_work_version', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='extra', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='same_as', + ), + migrations.RemoveField( + model_name='abstractagentworkrelationversion', + name='same_as_version', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='change', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='extra', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='extra_version', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='related_agent_versions', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='related_agents', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='related_work_versions', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='related_works', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='same_as', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='same_as_version', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='sources', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='subject_versions', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='subjects', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='tag_versions', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='tags', + ), + migrations.RemoveField( + model_name='abstractcreativework', + name='version', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='change', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='extra', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='related_agent_versions', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='related_agents', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='related_work_versions', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='related_works', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='same_as', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='same_as_version', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='subject_versions', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='subjects', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='tag_versions', + ), + migrations.RemoveField( + model_name='abstractcreativeworkversion', + name='tags', + ), + migrations.AlterUniqueTogether( + name='abstractworkrelation', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='change', + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='extra', + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='extra_version', + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='related', + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='related_version', + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='same_as', + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='same_as_version', + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='sources', + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='subject', + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='subject_version', + ), + migrations.RemoveField( + model_name='abstractworkrelation', + name='version', + ), + migrations.RemoveField( + model_name='abstractworkrelationversion', + name='change', + ), + migrations.RemoveField( + model_name='abstractworkrelationversion', + name='extra', + ), + migrations.RemoveField( + model_name='abstractworkrelationversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='abstractworkrelationversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='abstractworkrelationversion', + name='related', + ), + migrations.RemoveField( + model_name='abstractworkrelationversion', + name='related_version', + ), + migrations.RemoveField( + model_name='abstractworkrelationversion', + name='same_as', + ), + migrations.RemoveField( + model_name='abstractworkrelationversion', + name='same_as_version', + ), + migrations.RemoveField( + model_name='abstractworkrelationversion', + name='subject', + ), + migrations.RemoveField( + model_name='abstractworkrelationversion', + name='subject_version', + ), + migrations.RemoveField( + model_name='agentidentifier', + name='agent', + ), + migrations.RemoveField( + model_name='agentidentifier', + name='agent_version', + ), + migrations.RemoveField( + model_name='agentidentifier', + name='change', + ), + migrations.RemoveField( + model_name='agentidentifier', + name='extra', + ), + migrations.RemoveField( + model_name='agentidentifier', + name='extra_version', + ), + migrations.RemoveField( + model_name='agentidentifier', + name='same_as', + ), + migrations.RemoveField( + model_name='agentidentifier', + name='same_as_version', + ), + migrations.RemoveField( + model_name='agentidentifier', + name='sources', + ), + migrations.RemoveField( + model_name='agentidentifier', + name='version', + ), + migrations.RemoveField( + model_name='agentidentifierversion', + name='agent', + ), + migrations.RemoveField( + model_name='agentidentifierversion', + name='agent_version', + ), + migrations.RemoveField( + model_name='agentidentifierversion', + name='change', + ), + migrations.RemoveField( + model_name='agentidentifierversion', + name='extra', + ), + migrations.RemoveField( + model_name='agentidentifierversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='agentidentifierversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='agentidentifierversion', + name='same_as', + ), + migrations.RemoveField( + model_name='agentidentifierversion', + name='same_as_version', + ), + migrations.RemoveField( + model_name='award', + name='change', + ), + migrations.RemoveField( + model_name='award', + name='extra', + ), + migrations.RemoveField( + model_name='award', + name='extra_version', + ), + migrations.RemoveField( + model_name='award', + name='same_as', + ), + migrations.RemoveField( + model_name='award', + name='same_as_version', + ), + migrations.RemoveField( + model_name='award', + name='sources', + ), + migrations.RemoveField( + model_name='award', + name='version', + ), + migrations.RemoveField( + model_name='awardversion', + name='change', + ), + migrations.RemoveField( + model_name='awardversion', + name='extra', + ), + migrations.RemoveField( + model_name='awardversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='awardversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='awardversion', + name='same_as', + ), + migrations.RemoveField( + model_name='awardversion', + name='same_as_version', + ), + migrations.RemoveField( + model_name='change', + name='change_set', + ), + migrations.RemoveField( + model_name='change', + name='model_type', + ), + migrations.RemoveField( + model_name='change', + name='target_type', + ), + migrations.RemoveField( + model_name='change', + name='target_version_type', + ), + migrations.RemoveField( + model_name='changeset', + name='normalized_data', + ), + migrations.RemoveField( + model_name='extradata', + name='change', + ), + migrations.RemoveField( + model_name='extradata', + name='same_as', + ), + migrations.RemoveField( + model_name='extradata', + name='same_as_version', + ), + migrations.RemoveField( + model_name='extradata', + name='sources', + ), + migrations.RemoveField( + model_name='extradata', + name='version', + ), + migrations.RemoveField( + model_name='extradataversion', + name='change', + ), + migrations.RemoveField( + model_name='extradataversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='extradataversion', + name='same_as', + ), + migrations.RemoveField( + model_name='extradataversion', + name='same_as_version', + ), + migrations.AlterUniqueTogether( + name='subject', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='subject', + name='central_synonym', + ), + migrations.RemoveField( + model_name='subject', + name='central_synonym_version', + ), + migrations.RemoveField( + model_name='subject', + name='change', + ), + migrations.RemoveField( + model_name='subject', + name='extra', + ), + migrations.RemoveField( + model_name='subject', + name='extra_version', + ), + migrations.RemoveField( + model_name='subject', + name='parent', + ), + migrations.RemoveField( + model_name='subject', + name='parent_version', + ), + migrations.RemoveField( + model_name='subject', + name='same_as', + ), + migrations.RemoveField( + model_name='subject', + name='same_as_version', + ), + migrations.RemoveField( + model_name='subject', + name='sources', + ), + migrations.RemoveField( + model_name='subject', + name='taxonomy', + ), + migrations.RemoveField( + model_name='subject', + name='version', + ), + migrations.RemoveField( + model_name='subjecttaxonomy', + name='source', + ), + migrations.RemoveField( + model_name='subjectversion', + name='central_synonym', + ), + migrations.RemoveField( + model_name='subjectversion', + name='central_synonym_version', + ), + migrations.RemoveField( + model_name='subjectversion', + name='change', + ), + migrations.RemoveField( + model_name='subjectversion', + name='extra', + ), + migrations.RemoveField( + model_name='subjectversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='subjectversion', + name='parent', + ), + migrations.RemoveField( + model_name='subjectversion', + name='parent_version', + ), + migrations.RemoveField( + model_name='subjectversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='subjectversion', + name='same_as', + ), + migrations.RemoveField( + model_name='subjectversion', + name='same_as_version', + ), + migrations.RemoveField( + model_name='subjectversion', + name='taxonomy', + ), + migrations.RemoveField( + model_name='tag', + name='change', + ), + migrations.RemoveField( + model_name='tag', + name='extra', + ), + migrations.RemoveField( + model_name='tag', + name='extra_version', + ), + migrations.RemoveField( + model_name='tag', + name='same_as', + ), + migrations.RemoveField( + model_name='tag', + name='same_as_version', + ), + migrations.RemoveField( + model_name='tag', + name='sources', + ), + migrations.RemoveField( + model_name='tag', + name='version', + ), + migrations.RemoveField( + model_name='tagversion', + name='change', + ), + migrations.RemoveField( + model_name='tagversion', + name='extra', + ), + migrations.RemoveField( + model_name='tagversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='tagversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='tagversion', + name='same_as', + ), + migrations.RemoveField( + model_name='tagversion', + name='same_as_version', + ), + migrations.AlterUniqueTogether( + name='throughawards', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='throughawards', + name='award', + ), + migrations.RemoveField( + model_name='throughawards', + name='award_version', + ), + migrations.RemoveField( + model_name='throughawards', + name='change', + ), + migrations.RemoveField( + model_name='throughawards', + name='extra', + ), + migrations.RemoveField( + model_name='throughawards', + name='extra_version', + ), + migrations.RemoveField( + model_name='throughawards', + name='funder', + ), + migrations.RemoveField( + model_name='throughawards', + name='funder_version', + ), + migrations.RemoveField( + model_name='throughawards', + name='same_as', + ), + migrations.RemoveField( + model_name='throughawards', + name='same_as_version', + ), + migrations.RemoveField( + model_name='throughawards', + name='sources', + ), + migrations.RemoveField( + model_name='throughawards', + name='version', + ), + migrations.RemoveField( + model_name='throughawardsversion', + name='award', + ), + migrations.RemoveField( + model_name='throughawardsversion', + name='award_version', + ), + migrations.RemoveField( + model_name='throughawardsversion', + name='change', + ), + migrations.RemoveField( + model_name='throughawardsversion', + name='extra', + ), + migrations.RemoveField( + model_name='throughawardsversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='throughawardsversion', + name='funder', + ), + migrations.RemoveField( + model_name='throughawardsversion', + name='funder_version', + ), + migrations.RemoveField( + model_name='throughawardsversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='throughawardsversion', + name='same_as', + ), + migrations.RemoveField( + model_name='throughawardsversion', + name='same_as_version', + ), + migrations.AlterUniqueTogether( + name='throughcontributor', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='throughcontributor', + name='change', + ), + migrations.RemoveField( + model_name='throughcontributor', + name='extra', + ), + migrations.RemoveField( + model_name='throughcontributor', + name='extra_version', + ), + migrations.RemoveField( + model_name='throughcontributor', + name='related', + ), + migrations.RemoveField( + model_name='throughcontributor', + name='related_version', + ), + migrations.RemoveField( + model_name='throughcontributor', + name='same_as', + ), + migrations.RemoveField( + model_name='throughcontributor', + name='same_as_version', + ), + migrations.RemoveField( + model_name='throughcontributor', + name='sources', + ), + migrations.RemoveField( + model_name='throughcontributor', + name='subject', + ), + migrations.RemoveField( + model_name='throughcontributor', + name='subject_version', + ), + migrations.RemoveField( + model_name='throughcontributor', + name='version', + ), + migrations.RemoveField( + model_name='throughcontributorversion', + name='change', + ), + migrations.RemoveField( + model_name='throughcontributorversion', + name='extra', + ), + migrations.RemoveField( + model_name='throughcontributorversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='throughcontributorversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='throughcontributorversion', + name='related', + ), + migrations.RemoveField( + model_name='throughcontributorversion', + name='related_version', + ), + migrations.RemoveField( + model_name='throughcontributorversion', + name='same_as', + ), + migrations.RemoveField( + model_name='throughcontributorversion', + name='same_as_version', + ), + migrations.RemoveField( + model_name='throughcontributorversion', + name='subject', + ), + migrations.RemoveField( + model_name='throughcontributorversion', + name='subject_version', + ), + migrations.AlterUniqueTogether( + name='throughsubjects', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='throughsubjects', + name='change', + ), + migrations.RemoveField( + model_name='throughsubjects', + name='creative_work', + ), + migrations.RemoveField( + model_name='throughsubjects', + name='creative_work_version', + ), + migrations.RemoveField( + model_name='throughsubjects', + name='extra', + ), + migrations.RemoveField( + model_name='throughsubjects', + name='extra_version', + ), + migrations.RemoveField( + model_name='throughsubjects', + name='same_as', + ), + migrations.RemoveField( + model_name='throughsubjects', + name='same_as_version', + ), + migrations.RemoveField( + model_name='throughsubjects', + name='sources', + ), + migrations.RemoveField( + model_name='throughsubjects', + name='subject', + ), + migrations.RemoveField( + model_name='throughsubjects', + name='subject_version', + ), + migrations.RemoveField( + model_name='throughsubjects', + name='version', + ), + migrations.RemoveField( + model_name='throughsubjectsversion', + name='change', + ), + migrations.RemoveField( + model_name='throughsubjectsversion', + name='creative_work', + ), + migrations.RemoveField( + model_name='throughsubjectsversion', + name='creative_work_version', + ), + migrations.RemoveField( + model_name='throughsubjectsversion', + name='extra', + ), + migrations.RemoveField( + model_name='throughsubjectsversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='throughsubjectsversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='throughsubjectsversion', + name='same_as', + ), + migrations.RemoveField( + model_name='throughsubjectsversion', + name='same_as_version', + ), + migrations.RemoveField( + model_name='throughsubjectsversion', + name='subject', + ), + migrations.RemoveField( + model_name='throughsubjectsversion', + name='subject_version', + ), + migrations.AlterUniqueTogether( + name='throughtags', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='throughtags', + name='change', + ), + migrations.RemoveField( + model_name='throughtags', + name='creative_work', + ), + migrations.RemoveField( + model_name='throughtags', + name='creative_work_version', + ), + migrations.RemoveField( + model_name='throughtags', + name='extra', + ), + migrations.RemoveField( + model_name='throughtags', + name='extra_version', + ), + migrations.RemoveField( + model_name='throughtags', + name='same_as', + ), + migrations.RemoveField( + model_name='throughtags', + name='same_as_version', + ), + migrations.RemoveField( + model_name='throughtags', + name='sources', + ), + migrations.RemoveField( + model_name='throughtags', + name='tag', + ), + migrations.RemoveField( + model_name='throughtags', + name='tag_version', + ), + migrations.RemoveField( + model_name='throughtags', + name='version', + ), + migrations.RemoveField( + model_name='throughtagsversion', + name='change', + ), + migrations.RemoveField( + model_name='throughtagsversion', + name='creative_work', + ), + migrations.RemoveField( + model_name='throughtagsversion', + name='creative_work_version', + ), + migrations.RemoveField( + model_name='throughtagsversion', + name='extra', + ), + migrations.RemoveField( + model_name='throughtagsversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='throughtagsversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='throughtagsversion', + name='same_as', + ), + migrations.RemoveField( + model_name='throughtagsversion', + name='same_as_version', + ), + migrations.RemoveField( + model_name='throughtagsversion', + name='tag', + ), + migrations.RemoveField( + model_name='throughtagsversion', + name='tag_version', + ), + migrations.RemoveField( + model_name='workidentifier', + name='change', + ), + migrations.RemoveField( + model_name='workidentifier', + name='creative_work', + ), + migrations.RemoveField( + model_name='workidentifier', + name='creative_work_version', + ), + migrations.RemoveField( + model_name='workidentifier', + name='extra', + ), + migrations.RemoveField( + model_name='workidentifier', + name='extra_version', + ), + migrations.RemoveField( + model_name='workidentifier', + name='same_as', + ), + migrations.RemoveField( + model_name='workidentifier', + name='same_as_version', + ), + migrations.RemoveField( + model_name='workidentifier', + name='sources', + ), + migrations.RemoveField( + model_name='workidentifier', + name='version', + ), + migrations.RemoveField( + model_name='workidentifierversion', + name='change', + ), + migrations.RemoveField( + model_name='workidentifierversion', + name='creative_work', + ), + migrations.RemoveField( + model_name='workidentifierversion', + name='creative_work_version', + ), + migrations.RemoveField( + model_name='workidentifierversion', + name='extra', + ), + migrations.RemoveField( + model_name='workidentifierversion', + name='extra_version', + ), + migrations.RemoveField( + model_name='workidentifierversion', + name='persistent_id', + ), + migrations.RemoveField( + model_name='workidentifierversion', + name='same_as', + ), + migrations.RemoveField( + model_name='workidentifierversion', + name='same_as_version', + ), + migrations.DeleteModel( + name='Agent', + ), + migrations.DeleteModel( + name='AgentRelation', + ), + migrations.DeleteModel( + name='AgentRelationVersion', + ), + migrations.DeleteModel( + name='AgentVersion', + ), + migrations.DeleteModel( + name='AgentWorkRelation', + ), + migrations.DeleteModel( + name='AgentWorkRelationVersion', + ), + migrations.DeleteModel( + name='Article', + ), + migrations.DeleteModel( + name='ArticleVersion', + ), + migrations.DeleteModel( + name='Book', + ), + migrations.DeleteModel( + name='BookVersion', + ), + migrations.DeleteModel( + name='Cites', + ), + migrations.DeleteModel( + name='CitesVersion', + ), + migrations.DeleteModel( + name='Compiles', + ), + migrations.DeleteModel( + name='CompilesVersion', + ), + migrations.DeleteModel( + name='ConferencePaper', + ), + migrations.DeleteModel( + name='ConferencePaperVersion', + ), + migrations.DeleteModel( + name='Consortium', + ), + migrations.DeleteModel( + name='ConsortiumVersion', + ), + migrations.DeleteModel( + name='Contributor', + ), + migrations.DeleteModel( + name='ContributorVersion', + ), + migrations.DeleteModel( + name='Corrects', + ), + migrations.DeleteModel( + name='CorrectsVersion', + ), + migrations.DeleteModel( + name='CreativeWork', + ), + migrations.DeleteModel( + name='CreativeWorkVersion', + ), + migrations.DeleteModel( + name='Creator', + ), + migrations.DeleteModel( + name='CreatorVersion', + ), + migrations.DeleteModel( + name='DataSet', + ), + migrations.DeleteModel( + name='DataSetVersion', + ), + migrations.DeleteModel( + name='Department', + ), + migrations.DeleteModel( + name='DepartmentVersion', + ), + migrations.DeleteModel( + name='Discusses', + ), + migrations.DeleteModel( + name='DiscussesVersion', + ), + migrations.DeleteModel( + name='Disputes', + ), + migrations.DeleteModel( + name='DisputesVersion', + ), + migrations.DeleteModel( + name='Dissertation', + ), + migrations.DeleteModel( + name='DissertationVersion', + ), + migrations.DeleteModel( + name='Documents', + ), + migrations.DeleteModel( + name='DocumentsVersion', + ), + migrations.DeleteModel( + name='Extends', + ), + migrations.DeleteModel( + name='ExtendsVersion', + ), + migrations.DeleteModel( + name='Funder', + ), + migrations.DeleteModel( + name='FunderVersion', + ), + migrations.DeleteModel( + name='Host', + ), + migrations.DeleteModel( + name='HostVersion', + ), + migrations.DeleteModel( + name='Institution', + ), + migrations.DeleteModel( + name='InstitutionVersion', + ), + migrations.DeleteModel( + name='IsAffiliatedWith', + ), + migrations.DeleteModel( + name='IsAffiliatedWithVersion', + ), + migrations.DeleteModel( + name='IsDerivedFrom', + ), + migrations.DeleteModel( + name='IsDerivedFromVersion', + ), + migrations.DeleteModel( + name='IsEmployedBy', + ), + migrations.DeleteModel( + name='IsEmployedByVersion', + ), + migrations.DeleteModel( + name='IsMemberOf', + ), + migrations.DeleteModel( + name='IsMemberOfVersion', + ), + migrations.DeleteModel( + name='IsPartOf', + ), + migrations.DeleteModel( + name='IsPartOfVersion', + ), + migrations.DeleteModel( + name='IsSupplementTo', + ), + migrations.DeleteModel( + name='IsSupplementToVersion', + ), + migrations.DeleteModel( + name='Organization', + ), + migrations.DeleteModel( + name='OrganizationVersion', + ), + migrations.DeleteModel( + name='Patent', + ), + migrations.DeleteModel( + name='PatentVersion', + ), + migrations.DeleteModel( + name='Person', + ), + migrations.DeleteModel( + name='PersonVersion', + ), + migrations.DeleteModel( + name='Poster', + ), + migrations.DeleteModel( + name='PosterVersion', + ), + migrations.DeleteModel( + name='Preprint', + ), + migrations.DeleteModel( + name='PreprintVersion', + ), + migrations.DeleteModel( + name='Presentation', + ), + migrations.DeleteModel( + name='PresentationVersion', + ), + migrations.DeleteModel( + name='PrincipalInvestigator', + ), + migrations.DeleteModel( + name='PrincipalInvestigatorContact', + ), + migrations.DeleteModel( + name='PrincipalInvestigatorContactVersion', + ), + migrations.DeleteModel( + name='PrincipalInvestigatorVersion', + ), + migrations.DeleteModel( + name='Project', + ), + migrations.DeleteModel( + name='ProjectVersion', + ), + migrations.DeleteModel( + name='Publication', + ), + migrations.DeleteModel( + name='PublicationVersion', + ), + migrations.DeleteModel( + name='Publisher', + ), + migrations.DeleteModel( + name='PublisherVersion', + ), + migrations.DeleteModel( + name='References', + ), + migrations.DeleteModel( + name='ReferencesVersion', + ), + migrations.DeleteModel( + name='Registration', + ), + migrations.DeleteModel( + name='RegistrationVersion', + ), + migrations.DeleteModel( + name='RepliesTo', + ), + migrations.DeleteModel( + name='RepliesToVersion', + ), + migrations.DeleteModel( + name='Report', + ), + migrations.DeleteModel( + name='ReportVersion', + ), + migrations.DeleteModel( + name='Repository', + ), + migrations.DeleteModel( + name='RepositoryVersion', + ), + migrations.DeleteModel( + name='Retraction', + ), + migrations.DeleteModel( + name='RetractionVersion', + ), + migrations.DeleteModel( + name='Retracts', + ), + migrations.DeleteModel( + name='RetractsVersion', + ), + migrations.DeleteModel( + name='Reviews', + ), + migrations.DeleteModel( + name='ReviewsVersion', + ), + migrations.DeleteModel( + name='Software', + ), + migrations.DeleteModel( + name='SoftwareVersion', + ), + migrations.DeleteModel( + name='Thesis', + ), + migrations.DeleteModel( + name='ThesisVersion', + ), + migrations.DeleteModel( + name='UsesDataFrom', + ), + migrations.DeleteModel( + name='UsesDataFromVersion', + ), + migrations.DeleteModel( + name='WorkingPaper', + ), + migrations.DeleteModel( + name='WorkingPaperVersion', + ), + migrations.DeleteModel( + name='WorkRelation', + ), + migrations.DeleteModel( + name='WorkRelationVersion', + ), + migrations.DeleteModel( + name='AbstractAgent', + ), + migrations.DeleteModel( + name='AbstractAgentRelation', + ), + migrations.DeleteModel( + name='AbstractAgentRelationVersion', + ), + migrations.DeleteModel( + name='AbstractAgentVersion', + ), + migrations.DeleteModel( + name='AbstractAgentWorkRelation', + ), + migrations.DeleteModel( + name='AbstractAgentWorkRelationVersion', + ), + migrations.DeleteModel( + name='AbstractCreativeWork', + ), + migrations.DeleteModel( + name='AbstractCreativeWorkVersion', + ), + migrations.DeleteModel( + name='AbstractWorkRelation', + ), + migrations.DeleteModel( + name='AbstractWorkRelationVersion', + ), + migrations.DeleteModel( + name='AgentIdentifier', + ), + migrations.DeleteModel( + name='AgentIdentifierVersion', + ), + migrations.DeleteModel( + name='Award', + ), + migrations.DeleteModel( + name='AwardVersion', + ), + migrations.DeleteModel( + name='Change', + ), + migrations.DeleteModel( + name='ChangeSet', + ), + migrations.DeleteModel( + name='ExtraData', + ), + migrations.DeleteModel( + name='ExtraDataVersion', + ), + migrations.DeleteModel( + name='Subject', + ), + migrations.DeleteModel( + name='SubjectTaxonomy', + ), + migrations.DeleteModel( + name='SubjectVersion', + ), + migrations.DeleteModel( + name='Tag', + ), + migrations.DeleteModel( + name='TagVersion', + ), + migrations.DeleteModel( + name='ThroughAwards', + ), + migrations.DeleteModel( + name='ThroughAwardsVersion', + ), + migrations.DeleteModel( + name='ThroughContributor', + ), + migrations.DeleteModel( + name='ThroughContributorVersion', + ), + migrations.DeleteModel( + name='ThroughSubjects', + ), + migrations.DeleteModel( + name='ThroughSubjectsVersion', + ), + migrations.DeleteModel( + name='ThroughTags', + ), + migrations.DeleteModel( + name='ThroughTagsVersion', + ), + migrations.DeleteModel( + name='WorkIdentifier', + ), + migrations.DeleteModel( + name='WorkIdentifierVersion', + ), + migrations.AlterIndexTogether( + name='unusedcelerytask', + index_together=set([]), + ), + migrations.RemoveField( + model_name='unusedcelerytask', + name='provider', + ), + migrations.RemoveField( + model_name='unusedcelerytask', + name='started_by', + ), + migrations.DeleteModel( + name='UnusedCeleryProviderTask', + ), + migrations.DeleteModel( + name='UnusedCeleryTask', + ), + ] diff --git a/share/models/celery.py b/share/models/celery.py index 2f17e22ed..071932d6d 100644 --- a/share/models/celery.py +++ b/share/models/celery.py @@ -1,15 +1,8 @@ from celery import states -from django.conf import settings from django.db import models -from django.utils.translation import ugettext as _ - -from model_utils import Choices - -from typedmodels.models import TypedModel from share.models.fields import DateTimeAwareJSONField -from share.models.fuzzycount import FuzzyCountManager from share.models.jobs import get_share_version @@ -51,35 +44,3 @@ def as_dict(self): 'traceback': self.traceback, 'meta': self.meta, } - - -class UnusedCeleryTask(TypedModel): - """Keeping this model around so we have the data to refer back to, if need be. - """ - STATUS = Choices( - (0, 'started', _('started')), - (1, 'retried', _('retried')), - (2, 'failed', _('failed')), - (3, 'succeeded', _('succeeded')), - ) - - uuid = models.UUIDField(db_index=True, unique=True) - name = models.TextField(blank=True, db_index=True) - args = models.TextField(blank=True) - kwargs = models.TextField(blank=True) - timestamp = models.DateTimeField(auto_now_add=True, db_index=True) - started_by = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='started_by', null=True, on_delete=models.CASCADE) - # TODO rename to 'source' - provider = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='provider', null=True, on_delete=models.CASCADE) - status = models.IntegerField(choices=STATUS) - - objects = FuzzyCountManager() - - class Meta: - ordering = ('-timestamp',) - index_together = ('type', 'name', 'app_label', 'timestamp') - - -class UnusedCeleryProviderTask(UnusedCeleryTask): - app_label = models.TextField(db_index=True, blank=True) - app_version = models.TextField(db_index=True, blank=True) diff --git a/share/models/ingest.py b/share/models/ingest.py index 1ac11fa6e..f761c4e4a 100644 --- a/share/models/ingest.py +++ b/share/models/ingest.py @@ -18,7 +18,6 @@ from share.models.fields import EncryptedJSONField from share.models.fuzzycount import FuzzyCountManager -from share.models.indexes import ConcurrentIndex from share.util import chunked, placeholders, BaseJSONAPIMeta from share.util.extensions import Extensions @@ -480,9 +479,9 @@ def created(self): class Meta: unique_together = ('suid', 'sha256') verbose_name_plural = 'Raw Data' - indexes = ( - ConcurrentIndex(fields=['no_output']), - ) + indexes = [ + models.Index(fields=['no_output'], name='share_rawda_no_outp_f0330f_idx'), + ] class JSONAPIMeta(BaseJSONAPIMeta): resource_name = 'RawData' From b8c8f21c60dea48b35e93636b3e34016d0e66f33 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Wed, 17 Feb 2021 15:18:41 -0500 Subject: [PATCH 04/14] unravel some tests --- share/models/core.py | 8 - tests/api/test_feeds.py | 8 +- tests/api/test_generated_endpoints.py | 193 +----- tests/api/test_graphql.py | 113 ---- tests/api/test_sources_endpoint.py | 23 +- tests/conftest.py | 103 +-- tests/factories/__init__.py | 63 +- tests/factories/changes.py | 38 -- tests/factories/core.py | 52 -- tests/factories/share_objects.py | 157 ----- tests/features/steps/subjects.py | 138 ---- tests/features/subjects/taxonomy.feature | 124 ---- tests/share/changes/test_change.py | 618 ------------------ .../share/commands/test_enforce_set_lists.py | 90 --- tests/share/disambiguation/test_agent.py | 112 ---- .../disambiguation/test_agent_relation.py | 209 ------ .../share/disambiguation/test_creativework.py | 262 -------- tests/share/disambiguation/test_full.py | 94 --- tests/share/disambiguation/test_person.py | 91 --- tests/share/disambiguation/test_subject.py | 69 -- tests/share/metadata_formats/base.py | 46 +- .../metadata_formats/test_oai_dc_formatter.py | 7 +- .../test_sharev2_elastic_formatter.py | 15 +- tests/share/models/factories.py | 23 - tests/share/models/test_cascade.py | 301 --------- tests/share/models/test_change.py | 185 ------ tests/share/models/test_contribution.py | 76 --- tests/share/models/test_shareobject.py | 231 ------- tests/share/normalize/factories.py | 83 +-- tests/share/normalize/test_harness.py | 16 +- tests/share/normalize/test_models.py | 6 + tests/share/search/test_elastic_manager.py | 27 +- tests/share/search/test_indexer.py | 64 -- tests/share/search/test_indexing.py | 249 ------- tests/share/test_celery.py | 21 +- tests/share/test_oaipmh.py | 188 ------ tests/share/util/test_model_generator.py | 21 - 37 files changed, 200 insertions(+), 3924 deletions(-) delete mode 100644 tests/api/test_graphql.py delete mode 100644 tests/factories/changes.py delete mode 100644 tests/factories/core.py delete mode 100644 tests/factories/share_objects.py delete mode 100644 tests/features/steps/subjects.py delete mode 100644 tests/features/subjects/taxonomy.feature delete mode 100644 tests/share/changes/test_change.py delete mode 100644 tests/share/commands/test_enforce_set_lists.py delete mode 100644 tests/share/disambiguation/test_agent.py delete mode 100644 tests/share/disambiguation/test_agent_relation.py delete mode 100644 tests/share/disambiguation/test_creativework.py delete mode 100644 tests/share/disambiguation/test_full.py delete mode 100644 tests/share/disambiguation/test_person.py delete mode 100644 tests/share/disambiguation/test_subject.py delete mode 100644 tests/share/models/factories.py delete mode 100644 tests/share/models/test_cascade.py delete mode 100644 tests/share/models/test_change.py delete mode 100644 tests/share/models/test_contribution.py delete mode 100644 tests/share/models/test_shareobject.py delete mode 100644 tests/share/search/test_indexer.py delete mode 100644 tests/share/search/test_indexing.py delete mode 100644 tests/share/test_oaipmh.py delete mode 100644 tests/share/util/test_model_generator.py diff --git a/share/models/core.py b/share/models/core.py index 61deb23d4..d41373690 100644 --- a/share/models/core.py +++ b/share/models/core.py @@ -235,14 +235,6 @@ def _format_and_save(self, suid, normalized_datum, record_format): record = None return record - def get_or_create_record(self, suid, record_format): - try: - formatted_record = self.get(suid=suid, record_format=record_format) - except self.model.DoesNotExist: - records = self.save_formatted_metadata(suid, record_formats=[record_format]) - formatted_record = records[0] if records else None - return formatted_record - class FormattedMetadataRecord(models.Model): RECORD_FORMAT = Choices(*Extensions.get_names('share.metadata_formats')) diff --git a/tests/api/test_feeds.py b/tests/api/test_feeds.py index 6cb3517b4..33de18fc6 100644 --- a/tests/api/test_feeds.py +++ b/tests/api/test_feeds.py @@ -5,7 +5,6 @@ from lxml import etree -from share.models import AbstractCreativeWork, AbstractAgentWorkRelation from share.util import IDObfuscator from tests import factories @@ -18,6 +17,7 @@ # TODO add tests for RSS +@pytest.mark.skip @pytest.mark.django_db class TestFeed: @@ -71,14 +71,14 @@ def test_order(self, client, order, fake_items, feed_url): assert resp.status_code == 200 feed = etree.fromstring(resp.content) - works = AbstractCreativeWork.objects.order_by('-' + order).exclude(**{order: None}) + works = None # AbstractCreativeWork.objects.order_by('-' + order).exclude(**{order: None}) assert len(feed.xpath('//atom:entry', namespaces=NAMESPACES)) == 11 for creative_work, entry in zip(works, feed.xpath('//atom:entry', namespaces={'atom': 'http://www.w3.org/2005/Atom'})): try: - contributors = list(AbstractAgentWorkRelation.objects.filter(creative_work_id=creative_work.id)) - first_contributor = AbstractAgentWorkRelation.objects.get(creative_work_id=creative_work.id, order_cited=0) + contributors = None # list(AbstractAgentWorkRelation.objects.filter(creative_work_id=creative_work.id)) + first_contributor = None # AbstractAgentWorkRelation.objects.get(creative_work_id=creative_work.id, order_cited=0) except Exception: contributors = None diff --git a/tests/api/test_generated_endpoints.py b/tests/api/test_generated_endpoints.py index 8d3a3b7bf..d57dcd910 100644 --- a/tests/api/test_generated_endpoints.py +++ b/tests/api/test_generated_endpoints.py @@ -1,126 +1,11 @@ import pytest -import json -import re - -from share.disambiguation.matcher import Matcher -from share.disambiguation.strategies import DatabaseStrategy -from share.regulate import Regulator -from share.util import IDObfuscator from tests import factories -from tests.share.normalize.factories import * - - -def camelCase_to_underscore(name): - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) - return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() - - -initial = [ - Preprint( - id=1, - is_deleted=False, - identifiers=[WorkIdentifier(1, id=1)], - agent_relations=[ - Contributor(agent=Organization(id=1, name='American Heart Association')), - Creator(agent=Organization(2, id=2)), - Creator(agent=Organization(id=3)), - - ] - ), - CreativeWork( - id=2, - identifiers=[WorkIdentifier(2, id=2)], - agent_relations=[ - Creator(agent=Person(1, identifiers=[AgentIdentifier(14)])), - Funder(agent=Institution(id=5, name='NIH')), - Publisher(agent=Institution(id=6)), - ], - related_works=[ - Publication( - 11, - id=11, - agent_relations=[ - Contributor(id=12, agent=Institution(id=7, name="Test University")) - ], - identifiers=[WorkIdentifier(3, id=3)] - ) - ] - ) -] - - -@pytest.mark.django_db -class TestGeneratedEndpoints: - - @pytest.mark.parametrize('generator, model, route, controlled_values', [ - ([Institution(id=5, name='NIH')], 'institution', 'institutions', ['name']), - ([Organization(2, id=2)], 'organization', 'organizations', ['name']), - ([CreativeWork( - id=2, - identifiers=[WorkIdentifier(2, id=2)], - agent_relations=[Funder(agent=Institution(id=5, name='NIH'))] - )], 'funder', 'funders', ['citedAs']), - ([CreativeWork( - id=2, - identifiers=[WorkIdentifier(2, id=2)], - related_works=[ - Publication(11, id=11, identifiers=[WorkIdentifier(3, id=3)]) - ] - )], 'publication', 'publications', ['title', 'description']), - ([CreativeWork( - id=2, - identifiers=[WorkIdentifier(2, id=2)], - agent_relations=[Creator(agent=Person(1, identifiers=[AgentIdentifier(14)]))] - )], 'person', 'people', ['name']), - ]) - def test_get_data(self, generator, model, route, controlled_values, client, Graph, ingest): - ingest(Graph(initial)) - - graph = Graph(*generator) - Regulator().regulate(graph) - matches = Matcher(DatabaseStrategy()).find_all_matches(graph) - - for node in graph: - if node.type == model: - expected = node - expected_id = IDObfuscator.encode(matches[node]) - break - response = client.get('/api/v2/{}/{}/'.format(route, expected_id)) - - actual = json.loads(response.content.decode(encoding='UTF-8')) - - assert response.status_code == 200 - assert actual['data']['id'] == expected_id - assert actual['data']['attributes']['type'] == expected.type - for value in controlled_values: - assert actual['data']['attributes'][value] == expected[camelCase_to_underscore(value)] - - def test_can_delete_work(self, client, normalized_data_id): - preprint = factories.AbstractCreativeWorkFactory(is_deleted=False) - preprint.administrative_change(type='share.dataset') - assert preprint.is_deleted is False - - encoded_id = IDObfuscator.encode(preprint) - response = client.get('/api/v2/datasets/{}/'.format(encoded_id)) - assert response.status_code == 200 - - preprint.administrative_change(is_deleted=True) - assert preprint.is_deleted is True - - response = client.get('/api/v2/datasets/{}/'.format(encoded_id)) - assert response.status_code == 403 - assert response.json() == {"errors": [{"source": {"pointer": "/data"}, "detail": "This data set has been removed.", "status": "403"}]} - - response = client.get('/api/v2/datasets/') - assert response.status_code == 200 - assert response.json() == {'data': [], 'links': {'next': None, 'prev': None}} +# TODO these tests belong somewhere else @pytest.mark.django_db @pytest.mark.parametrize('endpoint, factory', [ - ('agents', factories.AbstractAgentFactory), - ('creativeworks', factories.AbstractCreativeWorkFactory), ('normalizeddata', factories.NormalizedDataFactory), ('rawdata', factories.RawDatumFactory), ]) @@ -172,84 +57,10 @@ def test_next_page(self, client, endpoint, factory): assert set(x['id'] for x in resp.json()['data']) & set(x['id'] for x in resp2.json()['data']) == set() def test_bad_cursor(self, client, endpoint, factory): - resp = client.get('/api/v2/creativeworks/', {'page[cursor]': 1}) + resp = client.get(f'/api/v2/{endpoint}/', {'page[cursor]': 1}) assert resp.status_code == 404 assert resp.json() == {'errors': [{ 'status': '404', 'detail': 'Invalid cursor', 'source': {'pointer': '/data'}, }]} - - -@pytest.mark.django_db -class TestDeprecation: - @pytest.mark.parametrize('endpoint', [ - 'agents', - 'creativeworks', - 'tags', - 'subjects', - 'throughtags', - 'throughsubjects', - 'agents', - 'organizations', - 'consortiums', - 'departments', - 'institutions', - 'people', - 'creativeworks', - 'datasets', - 'patents', - 'posters', - 'presentations', - 'publications', - 'articles', - 'books', - 'conferencepapers', - 'dissertations', - 'preprints', - 'projects', - 'registrations', - 'reports', - 'theses', - 'workingpapers', - 'repositories', - 'retractions', - 'software', - 'workidentifiers', - 'agentidentifiers', - 'agentworkrelations', - 'contributors', - 'creators', - 'principalinvestigators', - 'principalinvestigatorcontacts', - 'funders', - 'hosts', - 'publishers', - 'throughcontributors', - 'awards', - 'throughawards', - 'workrelations', - 'cites', - 'compiles', - 'corrects', - 'discusses', - 'disputes', - 'documents', - 'extends', - 'isderivedfrom', - 'ispartof', - 'issupplementto', - 'references', - 'repliesto', - 'retracts', - 'reviews', - 'usesdatafroms', - 'agentrelations', - 'isaffiliatedwith', - 'isemployedby', - 'ismemberof', - ]) - def test_responds_410(self, client, settings, endpoint): - settings.HIDE_DEPRECATED_VIEWS = True - resp = client.get(f'/api/v2/{endpoint}/') - assert resp.status_code == 410 diff --git a/tests/api/test_graphql.py b/tests/api/test_graphql.py deleted file mode 100644 index 07a83f686..000000000 --- a/tests/api/test_graphql.py +++ /dev/null @@ -1,113 +0,0 @@ -import pytest -from collections import OrderedDict - -from share.util import IDObfuscator - -from tests import factories - - -@pytest.fixture -def schema(): - from share.graphql.schema import schema - return schema - - -@pytest.mark.django_db -class TestCreativeWorks: - - query = ''' - query { - } - ''' - - def test_basic_case(self, schema): - x = factories.AbstractCreativeWorkFactory() - source = factories.SourceFactory() - x.sources.add(source.user) - - # Have to use % formats because of {}s everywhere - result = schema.execute(''' - query { - creativeWork(id: "%s") { - id, - title, - description, - sources { - title - } - } - } - ''' % (IDObfuscator.encode(x), )) - - assert not result.errors - - assert result.data == OrderedDict([ - ('creativeWork', OrderedDict([ - ('id', IDObfuscator.encode(x)), - ('title', x.title), - ('description', x.description), - ('sources', [ - OrderedDict([('title', source.long_title)]) - ]) - ])) - ]) - - def test_deleted_sources(self, schema): - x = factories.AbstractCreativeWorkFactory() - source = factories.SourceFactory(is_deleted=True) - x.sources.add(source.user) - - # Have to use % formats because of {}s everywhere - result = schema.execute(''' - query { - creativeWork(id: "%s") { - id, - title, - description, - sources { - title - } - } - } - ''' % (IDObfuscator.encode(x), )) - - assert not result.errors - - assert result.data == OrderedDict([ - ('creativeWork', OrderedDict([ - ('id', IDObfuscator.encode(x)), - ('title', x.title), - ('description', x.description), - ('sources', []) - ])) - ]) - - def test_no_icon(self, schema): - x = factories.AbstractCreativeWorkFactory() - source = factories.SourceFactory(icon='') - x.sources.add(source.user) - - # Have to use % formats because of {}s everywhere - result = schema.execute(''' - query { - creativeWork(id: "%s") { - id, - title, - description, - sources { - title - } - } - } - ''' % (IDObfuscator.encode(x), )) - - assert not result.errors - - assert result.data == OrderedDict([ - ('creativeWork', OrderedDict([ - ('id', IDObfuscator.encode(x)), - ('title', x.title), - ('description', x.description), - ('sources', []) - ])) - ]) diff --git a/tests/api/test_sources_endpoint.py b/tests/api/test_sources_endpoint.py index 6695190c1..7de639ef1 100644 --- a/tests/api/test_sources_endpoint.py +++ b/tests/api/test_sources_endpoint.py @@ -117,38 +117,41 @@ class TestSourcesGet: endpoint = '/api/v2/sources/' def test_count(self, client): - total = Source.objects.exclude(icon='').exclude(is_deleted=True).count() + sources_qs = Source.objects.exclude(icon='').exclude(is_deleted=True) + source_count = sources_qs.count() resp = client.get(self.endpoint) - assert total > 0 + assert source_count > 0 assert resp.status_code == 200 - assert resp.json()['meta']['pagination']['count'] == total + assert resp.json()['meta']['pagination']['count'] == source_count def test_is_deleted(self, client): - total = Source.objects.exclude(icon='').exclude(is_deleted=True).count() + sources_qs = Source.objects.exclude(icon='').exclude(is_deleted=True) + source_count = sources_qs.count() - s = Source.objects.first() + s = sources_qs.first() s.is_deleted = True s.save() resp = client.get(self.endpoint) assert resp.status_code == 200 - assert resp.json()['meta']['pagination']['count'] == total - 1 + assert resp.json()['meta']['pagination']['count'] == source_count - 1 def test_no_icon(self, client): - total = Source.objects.exclude(icon='').exclude(is_deleted=True).count() + sources_qs = Source.objects.exclude(icon='').exclude(is_deleted=True) + source_count = sources_qs.count() - s = Source.objects.first() + s = sources_qs.first() s.icon = None s.save() resp = client.get(self.endpoint) assert resp.status_code == 200 - assert resp.json()['meta']['pagination']['count'] == total - 1 + assert resp.json()['meta']['pagination']['count'] == source_count - 1 def test_by_id(self, client): - source = Source.objects.first() + source = Source.objects.last() resp = client.get('{}{}/'.format(self.endpoint, IDObfuscator.encode(source))) assert resp.status_code == 200 diff --git a/tests/conftest.py b/tests/conftest.py index 97188b3aa..30e24ea86 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,29 +11,16 @@ from urllib3.connection import ConnectionError from elasticsearch.exceptions import ConnectionError as ElasticConnectionError -from share.ingest.change_builder import ChangeBuilder, ChangeSetBuilder -from share.models import Person, NormalizedData, ChangeSet, RawDatum -from share.models import Article, Institution +from share.models import NormalizedData, RawDatum from share.models import ShareUser from share.models import SourceUniqueIdentifier -from share.regulate import Regulator from share.search import MessageType, SearchIndexer from share.search.elastic_manager import ElasticManager -from share.util.graph import MutableGraph from tests import factories from tests.share.normalize.factories import GraphBuilder -def pytest_configure(config): - # The hackiest of all hacks - # Looks like pytest's recursion detection doesn't like typedmodels - # and will sometimes cause all tests to fail - # If we create a queryset here, all of typedmodels cached properties - # will be filled in while recursion detection isn't active - Article.objects.all() - - @pytest.fixture def client(): from django.test.client import Client @@ -137,21 +124,6 @@ def normalized_data_id(normalized_data): return normalized_data.id -@pytest.fixture -def change_set(normalized_data_id): - return ChangeSet.objects.create(normalized_data_id=normalized_data_id) - - -@pytest.fixture -def change_node(): - return next(n for n in MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'person', - 'given_name': 'No', - 'family_name': 'Matter', - }])) - - @pytest.fixture def Graph(): return GraphBuilder() @@ -164,68 +136,6 @@ def expected_graph(*args, **kwargs): return expected_graph -@pytest.fixture -def ingest(normalized_data): - def _ingest(graph, disambiguate=True, regulate=True, user=None, save=True): - if regulate: - Regulator().regulate(graph) - - nd = factories.NormalizedDataFactory(source=user) if user else normalized_data - cs = ChangeSetBuilder(graph, nd, disambiguate=disambiguate).build_change_set() - if save and cs is not None: - cs.accept() - return cs - return _ingest - - -@pytest.fixture -def change_factory(share_user, source, change_set, change_node): - class ChangeFactory: - def from_graph(self, jsonld, disambiguate=False): - nd = NormalizedData.objects.create(data=jsonld, source=share_user) - graph = MutableGraph.from_jsonld(jsonld) - return ChangeSetBuilder(graph, nd, disambiguate=disambiguate).build_change_set() - - def get(self): - return ChangeBuilder(change_node, source).build_change(change_set) - - return ChangeFactory() - - -@pytest.fixture -def change_ids(change_factory): - class ChangeIdFactory: - def get(self): - return change_factory.get().id - return ChangeIdFactory() - - -@pytest.fixture -def john_doe(change_ids): - john = Person.objects.create(given_name='John', family_name='Doe', change_id=change_ids.get()) - john.refresh_from_db() - return john - - -@pytest.fixture -def jane_doe(change_ids): - jane = Person.objects.create(given_name='Jane', family_name='Doe', change_id=change_ids.get()) - jane.refresh_from_db() - return jane - - -@pytest.fixture -def all_about_anteaters(change_ids, share_user): - article = Article.objects.create(title='All about Anteaters', change_id=change_ids.get()) - article.sources.add(share_user) - return article - - -@pytest.fixture -def university_of_whales(change_ids): - return Institution.objects.create(name='University of Whales', change_id=change_ids.get()) - - @pytest.fixture def elastic_test_index_name(): return 'test_share' @@ -246,7 +156,7 @@ def elastic_test_manager(settings, elastic_test_index_name): elastic_test_index_name: { 'DEFAULT_QUEUE': f'{elastic_test_index_name}_queue', 'URGENT_QUEUE': f'{elastic_test_index_name}_queue.urgent', - 'INDEX_SETUP': 'share_classic', + 'INDEX_SETUP': 'postrend_backcompat', }, }, } @@ -264,10 +174,11 @@ def elastic_test_manager(settings, elastic_test_index_name): @pytest.fixture -def index_creativeworks(elastic_test_manager): +def index_records(elastic_test_manager): - def _index_creativeworks(ids): + def _index_records(formatted_records): + suid_ids = [fmr.suid_id for fmr in formatted_records] indexer = SearchIndexer(elastic_manager=elastic_test_manager) - indexer.handle_messages_sync(MessageType.INDEX_CREATIVEWORK, ids) + indexer.handle_messages_sync(MessageType.INDEX_SUID, suid_ids) - return _index_creativeworks + return _index_records diff --git a/tests/factories/__init__.py b/tests/factories/__init__.py index 758874072..a99b84d5a 100644 --- a/tests/factories/__init__.py +++ b/tests/factories/__init__.py @@ -7,11 +7,10 @@ import stevedore -import faker - import factory from factory import fuzzy from factory.django import DjangoModelFactory +import faker from project import celery_app @@ -21,15 +20,41 @@ from share.transform import BaseTransformer from share.util.extensions import Extensions -from tests.factories.core import * # noqa -from tests.factories.changes import * # noqa -from tests.factories.share_objects import * # noqa -from tests.factories.share_objects import ShareObjectFactory + +fake = faker.Faker() + + +class ShareUserFactory(DjangoModelFactory): + username = factory.Sequence(lambda x: '{}{}'.format(fake.name(), x)) + source = factory.RelatedFactory('tests.factories.SourceFactory', 'user') + + class Meta: + model = models.ShareUser + + +class NormalizedDataFactory(DjangoModelFactory): + data = {} + source = factory.SubFactory(ShareUserFactory) + + class Meta: + model = models.NormalizedData + + @classmethod + def _generate(cls, create, attrs): + normalized_datum = super()._generate(create, attrs) + + # HACK: allow overriding auto_now_add on created_at + created_at = attrs.pop('created_at', None) + if created_at is not None: + normalized_datum.created_at = created_at + normalized_datum.save() + + return normalized_datum class SourceFactory(DjangoModelFactory): - name = factory.Faker('sentence') - long_title = factory.Faker('sentence') + name = factory.Sequence(lambda x: '{}{}'.format(fake.name(), x)) + long_title = factory.Sequence(lambda x: '{}{}'.format(fake.sentence(), x)) icon = factory.SelfAttribute('name') user = factory.SubFactory(ShareUserFactory, source=None) @@ -194,28 +219,6 @@ class Meta: model = models.CeleryTaskResult -class SubjectTaxonomyFactory(DjangoModelFactory): - source = factory.SubFactory(SourceFactory) - - class Meta: - model = models.SubjectTaxonomy - - -class SubjectFactory(ShareObjectFactory): - name = factory.Sequence(lambda x: '{}?{}'.format(faker.bs(), x)) - uri = factory.Sequence(lambda x: str(x)) - taxonomy = factory.SubFactory(SubjectTaxonomyFactory) - - class Meta: - model = models.Subject - - -class ThroughSubjectsFactory(ShareObjectFactory): - - class Meta: - model = models.ThroughSubjects - - class FormattedMetadataRecordFactory(DjangoModelFactory): suid = factory.SubFactory(SourceUniqueIdentifierFactory) diff --git a/tests/factories/changes.py b/tests/factories/changes.py deleted file mode 100644 index e6cb40ed2..000000000 --- a/tests/factories/changes.py +++ /dev/null @@ -1,38 +0,0 @@ -import factory -from factory import fuzzy -from factory.django import DjangoModelFactory - -from django.contrib.contenttypes.models import ContentType - -from share import models - -from tests.factories.core import NormalizedDataFactory - - -__all__ = ( - 'ChangeFactory', - 'ChangeSetFactory', -) - - -class ChangeSetFactory(DjangoModelFactory): - normalized_data = factory.SubFactory(NormalizedDataFactory) - - class Meta: - model = models.ChangeSet - - -class ChangeFactory(DjangoModelFactory): - type = fuzzy.FuzzyChoice(models.Change.TYPE._db_values) - change = {} - node_id = factory.Sequence(lambda x: x) - change_set = factory.SubFactory(ChangeSetFactory) - target_type = factory.Iterator(ContentType.objects.all()) - target_version_type = factory.Iterator(ContentType.objects.all()) - - @factory.lazy_attribute - def model_type(self, *args, **kwargs): - return self.target_type - - class Meta: - model = models.Change diff --git a/tests/factories/core.py b/tests/factories/core.py deleted file mode 100644 index ce098c889..000000000 --- a/tests/factories/core.py +++ /dev/null @@ -1,52 +0,0 @@ -import factory -import faker -from factory.django import DjangoModelFactory - -from share import models - -__all__ = ( - 'SourceFactory', - 'ShareUserFactory', - 'NormalizedDataFactory', -) - -faker = faker.Faker() - - -class ShareUserFactory(DjangoModelFactory): - username = factory.Sequence(lambda x: '{}{}'.format(faker.name(), x)) - source = factory.RelatedFactory('tests.factories.core.SourceFactory', 'user') - - class Meta: - model = models.ShareUser - - -class NormalizedDataFactory(DjangoModelFactory): - data = {} - source = factory.SubFactory(ShareUserFactory) - - class Meta: - model = models.NormalizedData - - @classmethod - def _generate(cls, create, attrs): - normalized_datum = super()._generate(create, attrs) - - # HACK: allow overriding auto_now_add on created_at - created_at = attrs.pop('created_at', None) - if created_at is not None: - normalized_datum.created_at = created_at - normalized_datum.save() - - return normalized_datum - - -class SourceFactory(DjangoModelFactory): - name = factory.Sequence(lambda x: '{}{}'.format(faker.name(), x)) - long_title = factory.Sequence(lambda x: '{}{}'.format(faker.sentence(), x)) - icon = factory.SelfAttribute('name') - - user = factory.SubFactory(ShareUserFactory, source=None) - - class Meta: - model = models.Source diff --git a/tests/factories/share_objects.py b/tests/factories/share_objects.py deleted file mode 100644 index f6c64786e..000000000 --- a/tests/factories/share_objects.py +++ /dev/null @@ -1,157 +0,0 @@ -import random -from datetime import timezone - -import faker -import factory -from factory.django import DjangoModelFactory - -from share import models - -from tests.factories.changes import ChangeFactory - -__all__ = ( - 'AbstractAgentFactory', - 'AbstractCreativeWorkFactory', - 'AgentIdentifierFactory', - 'AgentWorkRelationFactory', - 'ExtraDataFactory', - 'TagFactory', - 'ThroughTagsFactory', - 'WorkIdentifierFactory', - 'AbstractWorkRelationFactory', - 'CreatorWorkRelationFactory', -) - - -faker = faker.Faker() - - -class ShareObjectFactory(DjangoModelFactory): - change = factory.SubFactory(ChangeFactory) - - class Meta: - abstract = True - - @classmethod - def _after_postgeneration(cls, obj, create, results=None): - return - - @classmethod - def _create(cls, obj, **attrs): - for key, value in tuple(attrs.items()): - if hasattr(value, 'VersionModel'): - attrs[key + '_version'] = value.version - return super()._create(obj, **attrs) - - @factory.post_generation - def setup_change(self, create, extracted, **kwargs): - self.refresh_from_db() - self.change.target = self - self.change.target_version = self.version - self.change.save() - - -class TypedShareObjectFactory(ShareObjectFactory): - class Meta: - abstract = True - - @factory.lazy_attribute - def type(stub): - model = random.choice(stub._LazyStub__model_class._meta.model._meta.concrete_model.get_type_classes()) - return model._meta.label.lower() - - -class AbstractAgentFactory(TypedShareObjectFactory): - name = factory.Faker('company') - given_name = factory.Faker('first_name') - family_name = factory.Faker('last_name') - - class Meta: - model = models.AbstractAgent - - -class AgentIdentifierFactory(ShareObjectFactory): - scheme = 'http' - host = 'testvalue' - uri = factory.Sequence(lambda x: '{}?{}'.format(faker.uri(), x)) - agent = factory.SubFactory(AbstractAgentFactory) - - class Meta: - model = models.AgentIdentifier - - -class AbstractCreativeWorkFactory(TypedShareObjectFactory): - title = factory.Faker('sentence') - description = factory.Faker('paragraph') - date_updated = factory.Faker('date_time_this_decade', tzinfo=timezone.utc) - date_published = factory.Faker('date_time_this_decade', tzinfo=timezone.utc) - - class Meta: - model = models.AbstractCreativeWork - - @factory.post_generation - def contributors(self, create, extracted, **kwargs): - if not create: - # Simple build, do nothing. - return - - if isinstance(extracted, int): - for _ in range(0, extracted): - AgentWorkRelationFactory(creative_work=self) - - -class WorkIdentifierFactory(ShareObjectFactory): - scheme = 'http' - host = 'testvalue' - uri = factory.Sequence(lambda x: '{}?{}'.format(faker.uri(), x)) - creative_work = factory.SubFactory(AbstractCreativeWorkFactory) - - class Meta: - model = models.WorkIdentifier - - -class AgentWorkRelationFactory(TypedShareObjectFactory): - cited_as = factory.Faker('name') - agent = factory.SubFactory(AbstractAgentFactory) - creative_work = factory.SubFactory(AbstractCreativeWorkFactory) - - class Meta: - model = models.AgentWorkRelation - - -class CreatorWorkRelationFactory(AgentWorkRelationFactory): - order_cited = 0 - type = 'share.creator' - - class Meta: - model = models.AbstractAgentWorkRelation - - -class TagFactory(ShareObjectFactory): - name = factory.Sequence(lambda x: '{}{}'.format(faker.word(), x)) - - class Meta: - model = models.Tag - - -class ThroughTagsFactory(ShareObjectFactory): - tag = factory.SubFactory(TagFactory) - creative_work = factory.SubFactory(AbstractCreativeWorkFactory) - - class Meta: - model = models.ThroughTags - - -class ExtraDataFactory(ShareObjectFactory): - data = {} - - class Meta: - model = models.ExtraData - - -class AbstractWorkRelationFactory(TypedShareObjectFactory): - subject = factory.SubFactory(AbstractCreativeWorkFactory) - related = factory.SubFactory(AbstractCreativeWorkFactory) - - class Meta: - model = models.AbstractWorkRelation diff --git a/tests/features/steps/subjects.py b/tests/features/steps/subjects.py deleted file mode 100644 index ad5eb79c6..000000000 --- a/tests/features/steps/subjects.py +++ /dev/null @@ -1,138 +0,0 @@ -import behave -from django.conf import settings - -from share import models -from share.ingest.change_builder import ChangeSetBuilder -from share.regulate import Regulator -from share.util.graph import MutableGraph - -from tests.factories import NormalizedDataFactory, ShareUserFactory - - -def accept_changes(context, nodes, username): - user = models.ShareUser.objects.get(username=username) - graph = MutableGraph.from_jsonld(nodes) - Regulator().regulate(graph) - nd = NormalizedDataFactory(source=user) - change_set = ChangeSetBuilder(graph, nd, disambiguate=True).build_change_set() - return change_set.accept() if change_set else None - - -def make_subjects(table, work_id=None): - subjects = {} - throughs = [] - - def ref(node): - if not node: - return None - return {'@id': node['@id'], '@type': node['@type']} - - for row in table: - parent, synonym = None, None - - if row.get('PARENT'): - parent = subjects.get(row['PARENT']) - if not parent: - parent = {'@id': '_:{}'.format(row['PARENT']), '@type': 'subject', 'name': row['PARENT']} - subjects[row['PARENT']] = parent - if row.get('SYNONYM'): - synonym = subjects.get(row['SYNONYM']) - if not synonym: - synonym = {'@id': '_:{}'.format(row['SYNONYM']), '@type': 'subject', 'name': row['SYNONYM']} - subjects[row['SYNONYM']] = synonym - - subject = { - '@id': '_:{}'.format(row['NAME']), - '@type': 'subject', - 'name': row['NAME'], - 'uri': row.get('URI'), - 'parent': ref(parent), - 'central_synonym': ref(synonym), - } - subjects[row['NAME']] = subject - if work_id: - throughs.append({ - '@id': '_:through_{}'.format(row['NAME']), - '@type': 'throughsubjects', - 'creative_work': {'@id': work_id, '@type': 'creativework'}, - 'subject': ref(subject), - }) - return [*subjects.values(), *throughs] - - -@behave.given('a central taxonomy') -def add_central_taxonomy(context): - accept_changes(context, make_subjects(context.table), settings.APPLICATION_USERNAME) - - -@behave.given('{username}\'s custom taxonomy') -def add_custom_taxonomy(context, username): - accept_changes(context, make_subjects(context.table), username) - - -@behave.given('a user {username} with a source') -def add_user(context, username): - ShareUserFactory(username=username) - - -@behave.when('{username} adds a work with subjects') -def add_work_with_subjects(context, username): - work = { - '@id': '_:worky', - '@type': 'creativework', - 'title': 'title title', - } - nodes = accept_changes(context, [work, *make_subjects(context.table, work['@id'])], username) - context.work = next(n for n in nodes if isinstance(n, models.AbstractCreativeWork)) - - -@behave.then('central taxonomy exists') -@behave.then('{username}\'s custom taxonomy exists') -def taxonomy_exists(context, username=None): - if not username: - username = settings.APPLICATION_USERNAME - assert models.SubjectTaxonomy.objects.filter(source__user__username=username).exists() - - -@behave.then('{count:d}{root}subjects exist in central taxonomy') -def count_central_subjects(context, count, root): - count_subjects(context, count, root, settings.APPLICATION_USERNAME) - - -@behave.then('{count:d}{root}subjects exist') -@behave.then('{count:d}{root}subjects exist in {username}\'s custom taxonomy') -def count_subjects(context, count, root, username=None): - qs = models.Subject.objects.all() - if username is not None: - qs = qs.filter(taxonomy__source__user__username=username) - - if root == ' root ': - qs = qs.filter(parent__isnull=True) - elif root != ' ': - raise ValueError('Invalid root part: {}'.format(root)) - - assert qs.count() == count - - -@behave.then('{custom_subject} is a synonym of {central_subject}') -def is_synonym(context, custom_subject, central_subject): - custom = models.Subject.objects.get(name=custom_subject) - assert custom.central_synonym.central_synonym_id is None - assert custom.central_synonym.name == central_subject - - -@behave.then('{child_name} is a root') -@behave.then('{child_name} is a child of {parent_name}') -def is_child(context, child_name, parent_name=None): - child = models.Subject.objects.get(name=child_name) - if parent_name: - assert child.taxonomy == child.parent.taxonomy - assert child.parent.name == parent_name - else: - assert child.parent is None - - -@behave.then('{name} has depth {depth:d}') -def subject_depth(context, name, depth): - lineage = models.Subject.objects.get(name=name).lineage() - assert len(lineage) == depth diff --git a/tests/features/subjects/taxonomy.feature b/tests/features/subjects/taxonomy.feature deleted file mode 100644 index c3162af7c..000000000 --- a/tests/features/subjects/taxonomy.feature +++ /dev/null @@ -1,124 +0,0 @@ -Feature: Custom Subject Taxonomy - - Background: - Given a user User1 with a source - And a user User2 with a source - And a central taxonomy - | NAME | PARENT | - | A | | - | A1 | A | - | A2 | A | - | A2a | A2 | - | A2b | A2 | - | A3 | A | - | B | | - | B1 | B | - | B2 | B | - And User1's custom taxonomy - | NAME | PARENT | SYNONYM | URI | - | cA | | A | http://example.com/cA | - | cA1 | cA | A1 | http://example.com/cA1 | - | cA2 | cA | A2 | http://example.com/cA2 | - | cAB | cA | B | http://example.com/cAB | - - Scenario: - Verify background worked. - - Then central taxonomy exists - And User1's custom taxonomy exists - And 13 subjects exist - And 9 subjects exist in central taxonomy - And 4 subjects exist in User1's custom taxonomy - And 3 root subjects exist - And 2 root subjects exist in central taxonomy - And 1 root subjects exist in User1's custom taxonomy - And A is a root - And A1 is a child of A - And cA is a root - And cA1 is a child of cA - And cA1 is a synonym of A1 - And A has depth 1 - And A1 has depth 2 - And A2a has depth 3 - And cA has depth 1 - And cA1 has depth 2 - And cA2 has depth 2 - - Scenario: - Add work with no subject changes. - - When User1 adds a work with subjects - | NAME | PARENT | SYNONYM | - | B1 | | | - | cA | | A | - | cA1 | cA | A1 | - | cA2 | cA | A2 | - Then 13 subjects exist - And 4 subjects exist in User1's custom taxonomy - And 1 root subjects exist in User1's custom taxonomy - - Scenario: - Changing a subject's parent. - - When User1 adds a work with subjects - | NAME | PARENT | SYNONYM | - | cA | | A | - | cA1 | cA | A1 | - | cA2 | cA1 | A2 | - Then cA2 is a child of cA1 - And cA1 is a child of cA - And cA has depth 1 - And cA1 has depth 2 - And cA2 has depth 3 - - Scenario: - Changing a custom subject's synonym. - - When User1 adds a work with subjects - | NAME | PARENT | SYNONYM | - | cA | | A | - | cA1 | cA | B1 | - | cA2 | cA | B2 | - Then cA1 is a synonym of B1 - And cA2 is a synonym of B2 - And cA is a synonym of A - - Scenario: - Add a custom subject. - - When User1 adds a work with subjects - | NAME | PARENT | SYNONYM | - | cA | | A | - | cA1 | cA | A1 | - | cA3 | cA1 | A3 | - Then 14 subjects exist - And 5 subjects exist in User1's custom taxonomy - And cA3 is a child of cA1 - And cA1 is a child of cA - And cA has depth 1 - And cA1 has depth 2 - And cA3 has depth 3 - And cA3 is a synonym of A3 - - Scenario: - Custom taxonomies don't share a namespace. - - When User1 adds a work with subjects - | NAME | PARENT | SYNONYM | URI | - | cA | | A | http://example.com/cA | - | cA1 | cA | A1 | http://example.com/cA1 | - | cA2 | cA | A2 | http://example.com/cA2 | - | cAB | cA | B | http://example.com/cAB | - When User2 adds a work with subjects - | NAME | PARENT | SYNONYM | URI | - | cA | | A | http://example.com/cA | - | cA1 | cA | A1 | http://example.com/cA1 | - | cA2 | cA | A2 | http://example.com/cA2 | - | cAB | cA | B | http://example.com/cAB | - Then User2's custom taxonomy exists - And 17 subjects exist - And 4 subjects exist in User1's custom taxonomy - And 4 subjects exist in User2's custom taxonomy - And 4 root subjects exist - And 1 root subjects exist in User1's custom taxonomy - And 1 root subjects exist in User2's custom taxonomy diff --git a/tests/share/changes/test_change.py b/tests/share/changes/test_change.py deleted file mode 100644 index e8d6d6b16..000000000 --- a/tests/share/changes/test_change.py +++ /dev/null @@ -1,618 +0,0 @@ -import pytest - -from django.contrib.contenttypes.models import ContentType - -from share import models -from share.ingest.change_builder import ChangeBuilder, ChangeSetBuilder -from share.util import IDObfuscator -from share.util.graph import MutableGraph - - -@pytest.fixture -def create_graph(): - return MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'person', - 'given_name': 'Jane', - 'family_name': 'Doe', - }]) - - -@pytest.fixture -def create_graph_node(create_graph): - return next(n for n in create_graph) - - -@pytest.fixture -def create_graph_dependencies(): - return MutableGraph.from_jsonld([{ - '@id': '_:123', - '@type': 'person', - 'given_name': 'Jane', - 'family_name': 'Doe', - }, { - '@id': '_:456', - '@type': 'Creator', - 'agent': {'@id': '_:123', '@type': 'person'}, - 'creative_work': {'@id': '_:789', '@type': 'preprint'}, - }, { - '@id': '_:789', - '@type': 'preprint', - 'title': 'All About Cats', - 'related_agents': [{'@id': '_:456', '@type': 'Creator'}] - }]) - - -@pytest.fixture -def update_graph(jane_doe): - return MutableGraph.from_jsonld([{ - '@id': IDObfuscator.encode(jane_doe), - '@type': 'person', - 'family_name': 'Dough', - }]) - - -@pytest.fixture -def update_graph_node(update_graph): - return next(n for n in update_graph) - - -# @pytest.fixture -# def merge_graph(jane_doe, john_doe): -# return MutableGraph.from_jsonld.from_jsonld({ -# '@graph': [{ -# '@id': '_:1234', -# '@type': 'MergeAction', -# 'into': {'@id': IDObfuscator.encode(jane_doe), '@type': 'person'}, -# 'from': [{'@id': IDObfuscator.encode(john_doe), '@type': 'person'}] -# }] -# }) - - -class TestChange: - - @pytest.mark.django_db - def test_create(self, create_graph_node, change_set): - change = ChangeBuilder(create_graph_node).build_change(change_set) - - assert change.type == models.Change.TYPE.create - - assert change.target is None - assert change.target_type == ContentType.objects.get(app_label='share', model='abstractagent') - assert change.target_id is None - - assert change.target_version is None - assert change.target_version_type == ContentType.objects.get(app_label='share', model='abstractagentversion') - assert change.target_version_id is None - - assert change.change == {'given_name': 'Jane', 'family_name': 'Doe'} - - @pytest.mark.django_db - def test_create_accept(self, create_graph_node, change_set): - change = ChangeBuilder(create_graph_node).build_change(change_set) - person = change.accept() - - assert person.pk is not None - assert isinstance(person, models.Person) - assert person.versions.first() is not None - assert person.change == change - assert person.given_name == 'Jane' - assert person.family_name == 'Doe' - assert change.affected_abstractagent == person - - @pytest.mark.django_db - def test_create_accept_no_save(self, create_graph_node, change_set): - change = ChangeBuilder(create_graph_node).build_change(change_set) - person = change.accept(save=False) - - assert person.pk is None - - @pytest.mark.django_db - def test_update_accept(self, jane_doe, update_graph_node, change_set): - change = ChangeBuilder( - update_graph_node, - matches={update_graph_node: jane_doe}, - ).build_change(change_set) - - assert jane_doe.family_name == 'Doe' - - person = change.accept() - jane_doe.refresh_from_db() - - assert jane_doe == person - assert jane_doe.family_name == 'Dough' - assert len(jane_doe.versions.all()) == 2 - - @pytest.mark.django_db - def test_update_accept_no_save(self, jane_doe, update_graph_node, change_set): - change = ChangeBuilder( - update_graph_node, - matches={update_graph_node: jane_doe}, - ).build_change(change_set) - - person = change.accept(save=False) - - assert person.family_name == 'Dough' - - person.refresh_from_db() - - assert person.family_name == 'Doe' - assert len(jane_doe.versions.all()) == 1 - assert jane_doe.version == jane_doe.versions.first() - - -class TestChangeSet: - - @pytest.mark.django_db - def test_create_dependencies_accept(self, normalized_data, create_graph_dependencies): - change_set = ChangeSetBuilder(create_graph_dependencies, normalized_data).build_change_set() - - assert change_set.changes.count() == 3 - assert change_set.changes.all()[0].node_id == '_:123' - assert change_set.changes.all()[1].node_id == '_:789' - assert change_set.changes.all()[2].node_id == '_:456' - - assert change_set.changes.last().change == { - 'agent': {'@id': '_:123', '@type': 'person'}, - 'creative_work': {'@id': '_:789', '@type': 'preprint'}, - } - - changed = change_set.accept() - - assert len(changed) == 3 - - assert isinstance(changed[0], models.Person) - assert isinstance(changed[1], models.Preprint) - assert isinstance(changed[2], models.Creator) - - assert None not in [c.pk for c in changed] - - @pytest.mark.django_db - def test_update_dependencies_accept(self, john_doe, normalized_data): - john_doe_id = IDObfuscator.encode(john_doe) - graph = MutableGraph.from_jsonld([{ - '@id': john_doe_id, - '@type': 'person', - 'given_name': 'Jane', - }, { - '@id': '_:456', - '@type': 'Creator', - 'agent': {'@id': john_doe_id, '@type': 'person'}, - 'creative_work': {'@id': '_:789', '@type': 'preprint'}, - }, { - '@id': '_:789', - '@type': 'preprint', - 'title': 'All About Cats', - }]) - - change_set = ChangeSetBuilder(graph, normalized_data, matches={ - john_doe_id: john_doe, - }).build_change_set() - - change_set.accept() - - john_doe.refresh_from_db() - - assert john_doe.given_name == 'Jane' - assert models.Preprint.objects.filter(agent_relations__agent=john_doe).count() == 1 - assert models.Preprint.objects.filter(agent_relations__agent=john_doe).first().title == 'All About Cats' - - @pytest.mark.django_db - def test_can_delete_work(self, john_doe, normalized_data): - graph = MutableGraph.from_jsonld([{ - '@id': '_:abc', - '@type': 'workidentifier', - 'uri': 'http://osf.io/faq', - 'creative_work': {'@id': '_:789', '@type': 'preprint'} - }, { - '@id': '_:789', - '@type': 'preprint', - 'title': 'All About Cats', - }]) - - change_set = ChangeSetBuilder(graph, normalized_data, disambiguate=True).build_change_set() - - preprint, identifier = change_set.accept() - - assert preprint.is_deleted is False - - graph = MutableGraph.from_jsonld([{ - '@id': '_:abc', - '@type': 'workidentifier', - 'uri': 'http://osf.io/faq', - 'creative_work': {'@id': '_:789', '@type': 'preprint'} - }, { - '@id': '_:789', - 'is_deleted': True, - '@type': 'preprint', - }]) - - ChangeSetBuilder(graph, normalized_data, disambiguate=True).build_change_set().accept() - - preprint.refresh_from_db() - assert preprint.is_deleted is True - - # @pytest.mark.django_db - # def test_merge_accept(self, normalized_data, merge_graph, john_doe, jane_doe): - # change_set = ChangeSet.objects.from_graph(merge_graph, normalized_data) - # ChangeSet.objects.from_graph(MutableGraph.from_jsonld({ - # '@graph': [{ - # '@id': '_:123', - # '@type': 'preprint', - # 'title': 'All About Cats' - # }, { - # '@id': '_:456', - # '@type': 'contributor', - # 'person': {'@id': john_doe.pk, '@type': 'person'}, - # 'creative_work': {'@id': '_:123', '@type': 'preprint'}, - # }] - # }), normalized_data).accept() - - # assert Preprint.objects.filter(contributor__person=john_doe).count() == 1 - # assert Preprint.objects.filter(contributor__person=john_doe).count() == 1 - # assert Preprint.objects.filter(contributor__person=jane_doe).count() == 0 - - # change_set.accept() - - # john_doe.refresh_from_db() - # jane_doe.refresh_from_db() - - # # Jane should not have been modified - # assert jane_doe.same_as is None - # assert jane_doe.versions.count() == 1 - - # # John should have been updated - # assert john_doe.versions.count() == 2 - - # # John's same_as field and same_as_version should have been updated - # assert john_doe.same_as == jane_doe - # assert john_doe.version.same_as == jane_doe - # assert john_doe.same_as_version == jane_doe.version - # assert john_doe.version.same_as_version == jane_doe.version - - # # John's latest change should be set to the merge change - # assert john_doe.change.change_set == change_set - # assert john_doe.version.change.change_set == change_set - - # # Ensure that date modifieds have been update - # assert john_doe.versions.first().date_modified > john_doe.versions.last().date_modified - - # # John is no longer a contributor on anything - # assert Preprint.objects.filter(contributor__person=john_doe).count() == 0 - # assert Preprint.objects.filter(contributor__person_version=john_doe.version).count() == 0 - - # # Jane is now a contributor - # assert Preprint.objects.filter(contributor__person=jane_doe).count() == 1 - # assert Preprint.objects.filter(contributor__person_version=jane_doe.version).count() == 1 - - # # The affected contributor should have been updated - # assert Contributor.objects.get(person=jane_doe).versions.count() == 2 - # assert Contributor.objects.get(person=jane_doe).change.change_set == change_set - - @pytest.mark.django_db - def test_change_work_type(self, normalized_data): - ''' - A CreativeWork with an Identifier exists. Accept a new changeset - with a Preprint with the same Identifier. The preprint should - disambiguate to the existing work, and the work's type should be - updated to Preprint - ''' - title = 'Ambiguous Earthquakes' - uri = 'http://osf.io/special-snowflake' - - cg = MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'project', - 'title': title, - 'identifiers': [{'@id': '_:2345', '@type': 'workidentifier'}] - }, { - '@id': '_:2345', - '@type': 'workidentifier', - 'uri': uri, - 'creative_work': {'@id': '_:1234', '@type': 'project'} - }]) - - original_change_set = ChangeSetBuilder(cg, normalized_data, disambiguate=True).build_change_set() - - work, identifier = original_change_set.accept() - id = work.id - - assert identifier.uri == uri - assert models.Project.objects.count() == 1 - assert models.Preprint.objects.count() == 0 - assert models.CreativeWork.objects.count() == 1 - assert models.Project.objects.all()[0].changes.count() == 1 - - cg = MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'preprint', - 'identifiers': [{'@id': '_:2345', '@type': 'workidentifier'}] - }, { - '@id': '_:2345', - '@type': 'workidentifier', - 'uri': uri, - 'creative_work': {'@id': '_:1234', '@type': 'preprint'} - }]) - - change_set = ChangeSetBuilder(cg, normalized_data, disambiguate=True).build_change_set() - - change_set.accept() - - assert models.Project.objects.count() == 0 - assert models.Preprint.objects.count() == 1 - assert models.CreativeWork.objects.count() == 1 - assert models.Preprint.objects.get(id=id).title == title - assert models.Preprint.objects.all()[0].changes.count() == 2 - - @pytest.mark.django_db - def test_generic_creative_work(self, normalized_data): - ''' - A Preprint with an Identifier exists. Accept a changeset with a - CreativeWork with the same Identifier and a different title. - The Preprint's title should be updated to the new value, but its type - should remain the same. - ''' - old_title = 'Ambiguous Earthquakes' - uri = 'http://osf.io/special-snowflake' - - original_change_set = ChangeSetBuilder(MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'preprint', - 'title': old_title, - 'identifiers': [{'@id': '_:2345', '@type': 'workidentifier'}] - }, { - '@id': '_:2345', - '@type': 'workidentifier', - 'uri': uri, - 'creative_work': {'@id': '_:1234', '@type': 'preprint'} - }]), normalized_data).build_change_set() - - preprint, identifier = original_change_set.accept() - id = preprint.id - - assert identifier.uri == uri - assert models.Preprint.objects.count() == 1 - assert models.CreativeWork.objects.filter(type='share.creativework').count() == 0 - assert models.Preprint.objects.get(id=id).title == old_title - - new_title = 'Ambidextrous Earthquakes' - - graph = MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'creativework', - 'title': new_title, - 'identifiers': [{'@id': '_:2345', '@type': 'workidentifier'}] - }, { - '@id': '_:2345', - '@type': 'workidentifier', - 'uri': uri, - 'creative_work': {'@id': '_:1234', '@type': 'creativework'} - }]) - - change_set = ChangeSetBuilder(graph, normalized_data, disambiguate=True).build_change_set() - change_set.accept() - - assert models.Preprint.objects.count() == 1 - assert models.CreativeWork.objects.filter(type='share.creativework').count() == 0 - assert models.Preprint.objects.get(id=id).title == new_title - - @pytest.mark.django_db - def test_related_works(self, normalized_data): - ''' - Create two works with a relation between them. - ''' - uri = 'http://osf.io/special-snowflake' - - change_set = ChangeSetBuilder(MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'preprint', - 'title': 'Dogs are okay too', - 'related_works': [{'@id': '_:foo', '@type': 'cites'}] - }, { - '@id': '_:2345', - '@type': 'creativework', - 'title': 'Cats, tho', - 'identifiers': [{'@id': '_:4567', '@type': 'workidentifier'}] - }, { - '@id': '_:foo', - '@type': 'cites', - 'subject': {'@id': '_:1234', '@type': 'preprint'}, - 'related': {'@id': '_:2345', '@type': 'creativework'}, - }, { - '@id': '_:4567', - '@type': 'workidentifier', - 'uri': uri, - 'creative_work': {'@id': '_:2345', '@type': 'creativework'} - }]), normalized_data).build_change_set() - change_set.accept() - - assert models.Preprint.objects.count() == 1 - assert models.CreativeWork.objects.filter(type='share.creativework').count() == 1 - - p = models.Preprint.objects.first() - c = models.AbstractCreativeWork.objects.get(title='Cats, tho') - - assert p.related_works.count() == 1 - assert p.related_works.first() == c - assert p.outgoing_creative_work_relations.count() == 1 - assert p.outgoing_creative_work_relations.first()._meta.model_name == 'cites' - assert p.outgoing_creative_work_relations.first().related == c - assert c.incoming_creative_work_relations.count() == 1 - assert c.incoming_creative_work_relations.first()._meta.model_name == 'cites' - assert c.incoming_creative_work_relations.first().subject == p - - @pytest.mark.django_db - def test_add_relation_related(self, normalized_data): - ''' - A work exists. Add a second work with a relation to the first work. - The first work should have the appropriate inverse relation to the - second work. - ''' - - uri = 'http://osf.io/special-snowflake' - ChangeSetBuilder(MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'article', - 'title': 'All About Cats', - 'identifiers': [{'@id': '_:2345', '@type': 'workidentifier'}] - }, { - '@id': '_:2345', - '@type': 'workidentifier', - 'uri': uri, - 'creative_work': {'@id': '_:1234', '@type': 'article'} - }]), normalized_data).build_change_set().accept() - - assert models.Article.objects.count() == 1 - - graph = MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'preprint', - 'title': 'Dogs are okay too', - 'related_works': [{'@id': '_:foo', '@type': 'cites'}] - }, { - '@id': '_:foo', - '@type': 'cites', - 'subject': {'@id': '_:1234', '@type': 'preprint'}, - 'related': {'@id': '_:2345', '@type': 'creativework'}, - }, { - '@id': '_:2345', - '@type': 'creativework', - 'identifiers': [{'@id': '_:4567', '@type': 'workidentifier'}] - }, { - '@id': '_:4567', - '@type': 'workidentifier', - 'uri': uri, - 'creative_work': {'@id': '_:2345', '@type': 'creativework'} - }]) - change_set = ChangeSetBuilder(graph, normalized_data, disambiguate=True).build_change_set() - change_set.accept() - - assert models.Article.objects.count() == 1 - assert models.Preprint.objects.count() == 1 - assert models.CreativeWork.objects.filter(type='share.creativework').count() == 0 - - cat = models.Article.objects.first() - dog = models.Preprint.objects.first() - - assert dog.outgoing_creative_work_relations.count() == 1 - assert dog.outgoing_creative_work_relations.first()._meta.model_name == 'cites' - assert dog.outgoing_creative_work_relations.first().related == cat - assert cat.incoming_creative_work_relations.count() == 1 - assert cat.incoming_creative_work_relations.first()._meta.model_name == 'cites' - assert cat.incoming_creative_work_relations.first().subject == dog - - @pytest.mark.django_db - def test_add_work_with_existing_relation(self, normalized_data): - ''' - Harvest a work that has a relation to some work identified by a DOI. - The related work should be a CreativeWork with no information except - the one Identifier. - Then harvest a work with the same DOI. It should update the - CreativeWork's type and attributes instead of creating a new work. - ''' - - uri = 'http://osf.io/special-snowflake' - - ChangeSetBuilder(MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'preprint', - 'title': 'Dogs are okay', - 'related_works': [{'@id': '_:foo', '@type': 'cites'}] - }, { - '@id': '_:foo', - '@type': 'cites', - 'subject': {'@id': '_:1234', '@type': 'preprint'}, - 'related': {'@id': '_:2345', '@type': 'creativework'}, - }, { - '@id': '_:2345', - '@type': 'creativework', - 'identifiers': [{'@id': '_:4567', '@type': 'workidentifier'}] - }, { - '@id': '_:4567', - '@type': 'workidentifier', - 'uri': uri, - 'creative_work': {'@id': '_:2345', '@type': 'creativework'} - }]), normalized_data).build_change_set().accept() - - assert models.CreativeWork.objects.filter(type='share.creativework').count() == 1 - assert models.Preprint.objects.count() == 1 - assert models.Article.objects.count() == 0 - - change = MutableGraph.from_jsonld([{ - '@id': '_:1234', - '@type': 'article', - 'title': 'All About Cats', - 'identifiers': [{'@id': '_:2345', '@type': 'workidentifier'}] - }, { - '@id': '_:2345', - '@type': 'workidentifier', - 'uri': uri, - 'creative_work': {'@id': '_:1234', '@type': 'article'} - }]) - - ChangeSetBuilder(change, normalized_data, disambiguate=True).build_change_set().accept() - - assert models.CreativeWork.objects.filter(type='share.creativework').count() == 0 - assert models.Article.objects.count() == 1 - assert models.Preprint.objects.count() == 1 - - cat = models.Article.objects.first() - dog = models.Preprint.objects.first() - - assert dog.outgoing_creative_work_relations.count() == 1 - assert dog.outgoing_creative_work_relations.first()._meta.model_name == 'cites' - assert dog.outgoing_creative_work_relations.first().related == cat - assert cat.incoming_creative_work_relations.count() == 1 - assert cat.incoming_creative_work_relations.first()._meta.model_name == 'cites' - assert cat.incoming_creative_work_relations.first().subject == dog - - @pytest.mark.django_db - def test_ignore_generic_work_type(self, change_factory, all_about_anteaters): - cs = change_factory.from_graph({ - '@graph': [{ - '@id': IDObfuscator.encode(all_about_anteaters), - '@type': 'creativework' - }] - }, disambiguate=True) - - assert cs is None - - @pytest.mark.django_db - def test_work_type_stays_nongeneric(self, change_factory, all_about_anteaters): - new_title = 'Some about Anteaters' - cs = change_factory.from_graph({ - '@graph': [{ - '@id': IDObfuscator.encode(all_about_anteaters), - '@type': 'creativework', - 'title': new_title - }] - }, disambiguate=True) - - assert all_about_anteaters.type == 'share.article' - assert models.Publication.objects.count() == 1 - - cs.accept() - all_about_anteaters.refresh_from_db() - - assert all_about_anteaters.type == 'share.article' - assert all_about_anteaters.title == new_title - - @pytest.mark.django_db - def test_change_agent_type(self, change_factory, university_of_whales): - cs = change_factory.from_graph({ - '@graph': [{ - '@id': IDObfuscator.encode(university_of_whales), - '@type': 'consortium' - }] - }, disambiguate=True) - - assert models.Institution.objects.count() == 1 - assert models.Consortium.objects.count() == 0 - - (org,) = cs.accept() - - assert org.type == 'share.consortium' - assert org.id == university_of_whales.id - assert org.name == university_of_whales.name - assert models.Institution.objects.count() == 0 - assert models.Consortium.objects.count() == 1 diff --git a/tests/share/commands/test_enforce_set_lists.py b/tests/share/commands/test_enforce_set_lists.py deleted file mode 100644 index 2e838a700..000000000 --- a/tests/share/commands/test_enforce_set_lists.py +++ /dev/null @@ -1,90 +0,0 @@ -import pytest - -from django.core.management import call_command - -from tests import factories - - -@pytest.mark.django_db -class TestEnforceSetListsCommand: - - # --dry makes no changes - # without --commit rolls back - # without --whitelist only enforces blacklist - # one source config - # multiple source configs - - @pytest.fixture - def source_config(self): - return factories.SourceConfigFactory() - - @pytest.fixture - def first_tags(self): - return [ - factories.TagFactory(name='first1'), - factories.TagFactory(name='first2'), - ] - - @pytest.fixture - def second_tags(self): - return [ - factories.TagFactory(name='second1'), - factories.TagFactory(name='second2'), - ] - - @pytest.fixture - def both_tags(self): - return [ - factories.TagFactory(name='both1'), - factories.TagFactory(name='both2'), - ] - - @pytest.fixture - def first_work(self, first_tags, both_tags, source_config): - return self._create_work(first_tags + both_tags, source_config.source.user) - - @pytest.fixture - def second_work(self, second_tags, both_tags, source_config): - return self._create_work(second_tags + both_tags, source_config.source.user) - - def _create_work(self, tags, source_user): - work = factories.AbstractCreativeWorkFactory() - for tag in tags: - through_tag = factories.ThroughTagsFactory(creative_work=work, tag=tag) - through_tag.sources.add(source_user) - work.sources.add(source_user) - return work - - @pytest.mark.parametrize('whitelist,blacklist,enforce_whitelist,first_deleted,second_deleted', [ - ([], [], True, False, False), - ([], [], False, False, False), - (['first1'], [], True, False, True), - (['first1'], [], False, False, False), - (['something'], [], True, True, True), - (['something'], [], False, False, False), - ([], ['first1'], True, True, False), - ([], ['second2'], False, False, True), - ([], ['something'], True, False, False), - ([], ['something'], False, False, False), - (['both1', 'first1'], ['first2'], True, True, False), - (['first1', 'second1'], ['both2'], False, True, True), - (['first1', 'second1'], ['both2'], True, True, True), - ([], ['first2', 'second2'], False, True, True), - ]) - def test_enforce_set_lists(self, source_config, first_work, second_work, whitelist, blacklist, enforce_whitelist, first_deleted, second_deleted): - source_config.transformer_kwargs = { - 'approved_sets': whitelist, - 'blocked_sets': blacklist - } - source_config.save() - - args = [source_config.label, '--commit'] - if enforce_whitelist: - args.append('--whitelist') - call_command('enforce_set_lists', *args) - - first_work.refresh_from_db() - assert first_work.is_deleted == first_deleted - - second_work.refresh_from_db() - assert second_work.is_deleted == second_deleted diff --git a/tests/share/disambiguation/test_agent.py b/tests/share/disambiguation/test_agent.py deleted file mode 100644 index 4ca5e0b2e..000000000 --- a/tests/share/disambiguation/test_agent.py +++ /dev/null @@ -1,112 +0,0 @@ -import pytest - -from share import models - -from tests.share.normalize.factories import * - - -initial = [ - Preprint( - identifiers=[WorkIdentifier(1)], - agent_relations=[ - Contributor(agent=Organization(1, name='American Heart Association')), - Creator(agent=Organization(2)), - Creator(agent=Organization(3)), - ] - ), - CreativeWork( - identifiers=[WorkIdentifier(2)], - agent_relations=[ - Host(agent=Institution(4)), - Funder(agent=Institution(name='NIH')), - Publisher(agent=Institution(6)), - ] - ), - CreativeWork( - identifiers=[WorkIdentifier(2)], - related_agents=[ - Institution(), - Consortium(name='COS'), - Organization(), - ] - ), - Publication( - identifiers=[WorkIdentifier(3)], - agent_relations=[ - Creator(agent=Organization(7)) - ], - related_works=[ - Patent( - agent_relations=[ - Contributor(agent=Institution(8)) - ], - identifiers=[WorkIdentifier(4)] - ) - ] - ), - Report( - identifiers=[WorkIdentifier(4)], - agent_relations=[ - Creator(agent=Person(name='Berkeley')), - Publisher(agent=Institution(name='Berkeley')) - ] - ) -] - - -@pytest.mark.django_db -class TestAgentDisambiguation: - - @pytest.fixture - def ingest_initial(self, ingest, Graph): - ingest(Graph(initial)) - - @pytest.mark.parametrize('input, model, delta', [ - # institution with same name already exists - ([Institution(name='NIH')], models.Institution, 0), - # same organization already exists - ([Organization(2)], models.Organization, 0), - # same institution already exists - ([Publication(related_agents=[Institution(4)])], models.Institution, 0), - # consortium with same name already exists - ([Publication(related_agents=[Consortium(name='COS')])], models.Consortium, 0), - # institution already exists on a related work - ([Preprint(related_agents=[Institution(8)])], models.Institution, 0), - # organization where the name does not exist - ([CreativeWork(related_agents=[Organization(name='Bill Gates')])], models.Organization, 1), - # organization and person exist with the same name - ([Organization(name='Berkeley')], models.Organization, 0), - # institution and person exist with the same name - ([Institution(name='Berkeley')], models.Institution, 0), - # person doesn't disambiguate on name - ([Person(name='Berkeley')], models.Person, 1), - ]) - def test_disambiguate(self, ingest_initial, ingest, input, model, delta, Graph): - Graph.reseed() - # Nasty hack to avoid progres' fuzzy counting - before = model.objects.exclude(change=None).count() - - ingest(Graph(input)) - - assert (model.objects.exclude(change=None).count() - before) == delta - - @pytest.mark.parametrize('input', [ - [Institution()], - [Institution(name='Money Money')], - [Organization(name='Money Makers'), Consortium()], - [Institution(identifiers=[AgentIdentifier()])], - [Publication(identifiers=[WorkIdentifier()], agent_relations=[Funder(agent=Organization()), Publisher(agent=Institution())])], - [Preprint(identifiers=[WorkIdentifier()], related_agents=[Institution(), Organization()], agent_relations=[Funder(agent=Institution()), Publisher(agent=Organization())])] - ]) - def test_reaccept(self, ingest_initial, ingest, input, Graph): - Graph.reseed() # Force new values to be generated - - first_cs = ingest(Graph(input)) - assert first_cs is not None - - second_cs = ingest(Graph(input)) - assert second_cs is None - - def test_no_changes(self, ingest, ingest_initial, normalized_data, Graph): - cs = ingest(Graph(initial)) - assert cs is None diff --git a/tests/share/disambiguation/test_agent_relation.py b/tests/share/disambiguation/test_agent_relation.py deleted file mode 100644 index 4f6564a2f..000000000 --- a/tests/share/disambiguation/test_agent_relation.py +++ /dev/null @@ -1,209 +0,0 @@ -import pytest - -from share import models - -from tests.share.normalize.factories import * - - -initial = [ - Preprint( - identifiers=[WorkIdentifier(1)], - agent_relations=[ - Contributor(cited_as='B. Dylan', agent=Person(1, identifiers=[AgentIdentifier(1)], name='Bob Dylan')), - Creator(cited_as='Cat Stevens', agent=Person(1, identifiers=[AgentIdentifier(1)], name='Cat Stevens')), - Contributor(cited_as='AHA', agent=Organization(1, identifiers=[AgentIdentifier(2)], name='American Heart Association')), - Creator(cited_as='COS', agent=Organization(2, identifiers=[AgentIdentifier(3)], name='COS')), - AgentWorkRelation(cited_as='Science Guy', agent=Organization(4, identifiers=[AgentIdentifier(4)], name='Science Guy')), - ] - ), - CreativeWork( - identifiers=[WorkIdentifier(2)], - agent_relations=[ - Funder(cited_as='Bill Gates', agent=Person(name='Bill R. Gates', identifiers=[AgentIdentifier(5)])), - AgentWorkRelation(cited_as='Bill Nye', agent=Person(4, identifiers=[AgentIdentifier(6)], name='Bill Nye')), - Funder(cited_as='NIH', agent=Institution(5, name='National Institute of Health', identifiers=[AgentIdentifier(7)])) - ] - ), - Publication( - identifiers=[WorkIdentifier(3)], - agent_relations=[ - Creator(cited_as='G. Washington', agent=Person(5, identifiers=[AgentIdentifier(8)], name='George Washington')), - Creator(agent=Organization(7)) - ], - related_works=[ - Patent( - agent_relations=[ - Contributor(cited_as='T.J.', agent=Person(8, identifiers=[AgentIdentifier(9)], name='Thomas Jefferson')), - Contributor(agent=Institution(8)) - ], - identifiers=[WorkIdentifier(4)] - ) - ] - ), - Retraction( - identifiers=[WorkIdentifier(5)], - agent_relations=[ - Creator(cited_as='Same Name', agent=Person(9)), - Creator(cited_as='Same Name', agent=Person(10)), - ], - ), -] - - -@pytest.mark.django_db -class TestAgentRelationDisambiguation: - - @pytest.fixture - def ingest_initial(self, Graph, ingest): - ingest(Graph(initial)) - - @pytest.mark.parametrize('input, model_delta', [ - # different name, same cited as, no work, person doesn't match - ( - [Person(name='B. Dylan')], - {models.Person: 1} - ), - # different name, same cited as, no work, agent doesn't match - ( - [Institution(name='NIH')], - {models.Institution: 1} - ), - # person with same cited as, different relation, on a different work doesn't match - ( - [CreativeWork(agent_relations=[Host(cited_as='Bill Gates', agent=Person(name='Bill Gates'))])], - {models.Person: 1} - ), - # agent with same cited as, different relation, on a different work doesn't match - ( - [CreativeWork(agent_relations=[Host(cited_as='AHA', agent=Organization(name='AHA'))])], - {models.Organization: 1} - ), - # person with same cited as, same relation, on a different work doesn't match - ( - [CreativeWork(agent_relations=[Funder(cited_as='Bill Gates', agent=Person(name='Bill Gates'))])], - {models.Person: 1} - ), - # agent with same cited as, same relation, on a different work doesn't match - ( - [CreativeWork(agent_relations=[Funder(cited_as='NIH', agent=Institution(name='NIH'))])], - {models.Institution: 1} - ), - # same work, same person, same relation, same first initial and last name in various formats - ( - [Publication(identifiers=[WorkIdentifier(3)], agent_relations=[Creator(cited_as='Georgy Washington', agent=Person(5, name='George Washington'))])], - {models.Person: 0, models.Creator: 0} - ), - ( - [Publication(identifiers=[WorkIdentifier(3)], agent_relations=[Creator(cited_as='Washington, G', agent=Person(5, name='Washington, G'))])], - {models.Person: 0, models.Creator: 0} - ), - ( - [Preprint(identifiers=[WorkIdentifier(1)], agent_relations=[Creator(cited_as='C. Stevens', agent=Person(5, name='C. Stevens'))])], - {models.Person: 0, models.Creator: 0} - ), - ( - [Preprint(identifiers=[WorkIdentifier(1)], agent_relations=[Creator(cited_as='Stevens, Cat', agent=Person(5, name='Stevens, Cat'))])], - {models.Person: 0, models.Creator: 0} - ), - ( - [Preprint(identifiers=[WorkIdentifier(1)], agent_relations=[Creator(cited_as='Stevens, C. X.', agent=Person(5, name='Stevens, C. X.'))])], - {models.Person: 0, models.Creator: 0} - ), - # same work, same person, different relation (still creator), no identifiers, same first initial and last name - # longer cited as or alphabetical, same work, same person, same relation, different cited_as - ( - [Publication(identifiers=[WorkIdentifier(3)], agent_relations=[Creator(cited_as='George Washington', agent=Person(5, identifiers=[AgentIdentifier(8)], name='George Washington'))])], - {models.Person: 0, models.Creator: 0} - ), - # longer cited as or alphabetical, same work, same agent, same relation, different cited_as - ( - [CreativeWork(identifiers=[WorkIdentifier(2)], agent_relations=[Funder(cited_as='National Institute of Health', agent=Institution(5, name='National Institute of Health', identifiers=[AgentIdentifier(7)]))])], - {models.Institution: 0, models.Funder: 0} - ), - - # Specified class should win (Contributor/Creator) person (Unique to Contributors) - ( - [Publication(identifiers=[WorkIdentifier(3)], agent_relations=[Contributor(cited_as='George Washington', agent=Person(5, identifiers=[AgentIdentifier(8)], name='George Washington'))])], - {models.Person: 0, models.Contributor: 1, models.Creator: -1} - ), - ( - [Patent(identifiers=[WorkIdentifier(4)], agent_relations=[Creator(cited_as='Thomas Jefferson', agent=Person(8, identifiers=[AgentIdentifier(9)], name='Thomas Jefferson'))])], - {models.Person: 0, models.Creator: 1, models.Contributor: -1} - ), - - # Specified class should win (Contributor/Creator) person (Unique to Contributors) - ( - [Preprint(identifiers=[WorkIdentifier(1)], agent_relations=[Contributor(cited_as='COS', agent=Organization(2, identifiers=[AgentIdentifier(3)], name='COS'))])], - {models.Organization: 0, models.Contributor: 1, models.Creator: -1} - ), - ( - [Preprint(identifiers=[WorkIdentifier(1)], agent_relations=[Creator(cited_as='AHA', agent=Organization(1, identifiers=[AgentIdentifier(2)], name='American Heart Association'))])], - {models.Organization: 0, models.Creator: 1, models.Contributor: -1} - ), - - # sub class should win (AgentWorkRelation/Funder) person - ( - [CreativeWork(identifiers=[WorkIdentifier(2)], agent_relations=[AgentWorkRelation(cited_as='Bill Gates', agent=Person(name='Bill R. Gates', identifiers=[AgentIdentifier(5)]))])], - {models.Person: 0, models.AgentWorkRelation: 0} - ), - ( - [CreativeWork(identifiers=[WorkIdentifier(2)], agent_relations=[Funder(cited_as='Bill Nye', agent=Person(4, identifiers=[AgentIdentifier(6)], name='Bill Nye'))])], - {models.Person: 0, models.Funder: 1, models.AgentWorkRelation: -1} - ), - - # sub class should win (AgentWorkRelation/Funder) agent - ( - [CreativeWork(identifiers=[WorkIdentifier(2)], agent_relations=[AgentWorkRelation(cited_as='NIH', agent=Institution(5, name='National Institute of Health', identifiers=[AgentIdentifier(7)]))])], - {models.Institution: 0, models.AgentWorkRelation: 0} - ), - ( - [Preprint(identifiers=[WorkIdentifier(1)], agent_relations=[Funder(cited_as='Science Guy', agent=Organization(4, identifiers=[AgentIdentifier(4)], name='Science Guy'))])], - {models.Institution: 0, models.Funder: 1, models.AgentWorkRelation: -1} - ), - - # sub class should win (AgentWorkRelation/Creator) person - ( - [Publication(identifiers=[WorkIdentifier(3)], agent_relations=[AgentWorkRelation(cited_as='G. Washington', agent=Person(5, identifiers=[AgentIdentifier(8)], name='George Washington'))])], - {models.Person: 0, models.AgentWorkRelation: 0} - ), - ( - [CreativeWork(identifiers=[WorkIdentifier(2)], agent_relations=[Creator(cited_as='Bill Nye', agent=Person(4, identifiers=[AgentIdentifier(6)], name='Bill Nye'))])], - {models.Person: 0, models.Creator: 1, models.AgentWorkRelation: -1} - ), - - # sub class should win (AgentWorkRelation/Creator) agent - ( - [Preprint(identifiers=[WorkIdentifier(1)], agent_relations=[AgentWorkRelation(cited_as='COS', agent=Organization(2, identifiers=[AgentIdentifier(3)], name='COS'))])], - {models.Organization: 0, models.AgentWorkRelation: 0} - ), - ( - [Preprint(identifiers=[WorkIdentifier(1)], agent_relations=[Creator(cited_as='Science Guy', agent=Organization(4, identifiers=[AgentIdentifier(4)], name='Science Guy'))])], - {models.Organization: 0, models.Creator: 1, models.AgentWorkRelation: -1} - ), - - # Same cited as, no changes - ( - Retraction( - identifiers=[WorkIdentifier(5)], - agent_relations=[ - Creator(cited_as='Same Name', agent=Person(9)), - Creator(cited_as='Same Name', agent=Person(10)), - ], - ), - {models.Retraction: 0, models.Creator: 0, models.Person: 0}, - ) - ]) - def test_disambiguate(self, input, ingest_initial, ingest, model_delta, Graph): - Graph.reseed() - before_count = {} - for model in model_delta.keys(): - before_count[model] = model.objects.filter(type=model._meta.label_lower).count() - - ingest(Graph(input)) - - for model in model_delta.keys(): - assert model.objects.filter(type=model._meta.label_lower).count() - before_count[model] == model_delta[model] - - def test_no_changes(self, ingest_initial, ingest, Graph): - cs = ingest(Graph(initial)) - assert cs is None diff --git a/tests/share/disambiguation/test_creativework.py b/tests/share/disambiguation/test_creativework.py deleted file mode 100644 index c4cfdfb63..000000000 --- a/tests/share/disambiguation/test_creativework.py +++ /dev/null @@ -1,262 +0,0 @@ -import pytest - -from share import models -from share.exceptions import MergeRequired - -from tests import factories -from tests.share.normalize.factories import * - - -initial = [ - Preprint( - tags=[Tag(name=' Science')], - identifiers=[WorkIdentifier(1)], - related_agents=[ - Person(1), - Person(2), - Person(3), - Institution(4), - ], - related_works=[ - Article(tags=[Tag(name='Science\n; Stuff')], identifiers=[WorkIdentifier(2)]) - ] - ), - CreativeWork( - tags=[Tag(name='Ghosts N Stuff')], - identifiers=[WorkIdentifier(3)], - related_agents=[ - Person(5), - Person(6), - Person(7), - Organization(8, name='Aperture Science'), - Institution(9), - ], - related_works=[ - DataSet(identifiers=[WorkIdentifier(4)], related_agents=[Consortium(10)]) - ] - ), - Publication( - tags=[Tag(name=' Science')], - identifiers=[WorkIdentifier(5)], - related_agents=[Organization(name='Umbrella Corporation')], - related_works=[ - Patent( - tags=[Tag(name='Science\n; Stuff')], - identifiers=[WorkIdentifier(6)] - ) - ] - ), -] - - -@pytest.mark.django_db -class TestWorkDisambiguation: - @pytest.fixture - def ingest_initial(self, ingest, Graph): - ingest(Graph(initial)) - - @pytest.mark.parametrize('input, model, delta', [ - # creativework with different identifier as creativework - ([CreativeWork(identifiers=[WorkIdentifier(7)])], models.CreativeWork, 1), - # creativework with same identifier as creativework - ([CreativeWork(identifiers=[WorkIdentifier(3)])], models.CreativeWork, 0), - # creativework with same identifier as creativework and other identifiers - ([CreativeWork(identifiers=[WorkIdentifier(3), WorkIdentifier(), WorkIdentifier()])], models.CreativeWork, 0), - # creativework with same identifier as publication - ([CreativeWork(identifiers=[WorkIdentifier(5)])], models.CreativeWork, 0), - ([CreativeWork(identifiers=[WorkIdentifier(5)])], models.Publication, 0), - # creativework with an additional identifier - ([CreativeWork(identifiers=[WorkIdentifier(), WorkIdentifier(5)])], models.Publication, 0), - ([CreativeWork(identifiers=[WorkIdentifier(), WorkIdentifier(5)])], models.WorkIdentifier, 1), - ([CreativeWork(identifiers=[WorkIdentifier(20), WorkIdentifier(5)])], models.Publication, 0), - ([CreativeWork(identifiers=[WorkIdentifier(21), WorkIdentifier(5)])], models.WorkIdentifier, 1), - ([CreativeWork(identifiers=[WorkIdentifier(5), WorkIdentifier(22)])], models.Publication, 0), - ([CreativeWork(identifiers=[WorkIdentifier(5), WorkIdentifier(23)])], models.WorkIdentifier, 1), - # article with same identifier as publication - ([Article(identifiers=[WorkIdentifier(5)])], models.Article, 1), - ([Article(identifiers=[WorkIdentifier(5)])], models.Publication, 0), - # software with same identifier as dataset (same level) - # fails - ([Software(identifiers=[WorkIdentifier(4)])], models.Software, 1), - ([Software(identifiers=[WorkIdentifier(4)])], models.DataSet, -1), - ]) - def test_disambiguate(self, input, model, delta, Graph, ingest, ingest_initial): - Graph.reseed() - # Nasty hack to avoid progres' fuzzy counting - before = model.objects.exclude(change=None).count() - - ingest(Graph(input)) - - assert (model.objects.exclude(change=None).count() - before) == delta - - @pytest.mark.parametrize('input', [ - [Publication(identifiers=[WorkIdentifier()])], - [CreativeWork(identifiers=[WorkIdentifier()])], - [Software(identifiers=[WorkIdentifier()])], - [Article(identifiers=[WorkIdentifier()])], - [Thesis(identifiers=[WorkIdentifier()])], - [Preprint(identifiers=[WorkIdentifier()], related_agents=[Person(), Consortium()], agent_relations=[Funder(), Publisher()])] - ]) - def test_reaccept(self, input, Graph, ingest_initial, ingest): - # Graph.reseed() # Force new values to be generated - - first_cs = ingest(Graph(input)) - assert first_cs is not None - - second_cs = ingest(Graph(input)) - assert second_cs is None - - def test_no_changes(self, Graph, ingest_initial, ingest): - cs = ingest(Graph(initial)) - assert cs is None - - def test_split_brain(self, Graph, ingest_initial, ingest): - # Multiple matches found for a thing should break - graph = Graph(Preprint(identifiers=[WorkIdentifier(1), WorkIdentifier(2)])) - with pytest.raises(MergeRequired) as e: - ingest(graph) - assert e.value.args[0].startswith('Multiple matches for node') - - def test_no_merge_on_blank_value(self, Graph, ingest): - blank_cited_as = [ - Publication( - identifiers=[WorkIdentifier(1)], - agent_relations=[ - Publisher(cited_as='', agent=Organization(1)), - ] - ) - ] - ingest(Graph(blank_cited_as)) - - assert models.Publication.objects.count() == 1 - assert models.Publisher.objects.count() == 1 - assert models.Organization.objects.count() == 1 - - additional_pub = [ - Publication( - identifiers=[WorkIdentifier(1)], - agent_relations=[ - Publisher(cited_as='', agent=Organization(1)), - Publisher(cited_as='', agent=Organization(2)), - ] - ) - ] - ingest(Graph(additional_pub)) - - assert models.Publication.objects.count() == 1 - assert models.Publisher.objects.count() == 2 - assert models.Organization.objects.count() == 2 - - def test_no_timetraveling(self, Graph, ingest): - newer_graph = Graph( - Publication( - id=1, - sparse=True, - identifiers=[WorkIdentifier(1)], - date_updated='2017-02-03T18:07:53.385000Z', - is_deleted=False, - ) - ) - ingest(newer_graph) - - older_graph = Graph( - Publication( - id=1, - sparse=True, - identifiers=[WorkIdentifier(1)], - date_updated='2017-02-03T18:07:50.000000Z', - is_deleted=True, - title='Not Previously Changed' - ) - ) - cs = ingest(older_graph) - c = cs.changes.first() - assert c.change == {'title': 'Not Previously Changed'} - - def test_no_timetraveling_many(self, Graph, ingest): - oldest_graph = Graph( - Publication( - id=1, - sparse=True, - is_deleted=True, - title='The first title', - description='The first description', - identifiers=[WorkIdentifier(1)], - date_updated='2016-02-03T18:07:50.000000Z', - ) - ) - ingest(oldest_graph) - - newer_graph = Graph( - Publication( - id=1, - sparse=True, - is_deleted=False, - identifiers=[WorkIdentifier(1)], - date_updated='2017-02-03T18:07:50.000000Z', - ) - ) - ingest(newer_graph) - - newest_graph = Graph( - Publication( - id=1, - sparse=True, - title='The final title', - identifiers=[WorkIdentifier(1)], - date_updated='2017-02-03T18:07:53.385000Z', - ) - ) - ingest(newest_graph) - - older_graph = Graph( - Publication( - id=1, - sparse=True, - is_deleted=True, - title='The second title', - description='The final description', - identifiers=[WorkIdentifier(1)], - date_updated='2017-01-01T18:00:00.000000Z', - ) - ) - cs = ingest(older_graph) - c = cs.changes.first() - assert c.change == {'description': 'The final description'} - - @pytest.mark.parametrize('first_canonical, second_canonical, change', [ - (True, False, {}), - (True, True, {'type': 'share.article', 'title': 'The Second Title'}), - (False, True, {'type': 'share.article', 'title': 'The Second Title'}), - (False, False, {'type': 'share.article', 'title': 'The Second Title'}), - ]) - def test_canonical(self, Graph, ingest, first_canonical, second_canonical, change): - first_source = factories.SourceFactory(canonical=first_canonical) - second_source = factories.SourceFactory(canonical=second_canonical) - - first_graph = Graph( - Preprint( - id=1, - title='The first title', - identifiers=[WorkIdentifier(1)], - ) - ) - cs = ingest(first_graph, user=first_source.user) - cw = cs.changes.first().target - - assert cw.type == 'share.preprint' - assert cw.title == 'The first title' - - second_graph = Graph( - Article( - id=1, - title='The Second Title', - identifiers=[WorkIdentifier(1)], - ) - ) - ingest(second_graph, user=second_source.user) - - cw = models.AbstractCreativeWork.objects.get(id=cw.id) - - assert cw.type == change.get('type', 'share.preprint') - assert cw.title == change.get('title', 'The first title') diff --git a/tests/share/disambiguation/test_full.py b/tests/share/disambiguation/test_full.py deleted file mode 100644 index dca6bf38b..000000000 --- a/tests/share/disambiguation/test_full.py +++ /dev/null @@ -1,94 +0,0 @@ -import pytest - -from share import models - -from tests.share.normalize.factories import * - - -initial = [ - Preprint( - tags=[Tag(name=' Science')], - identifiers=[WorkIdentifier(1)], - related_agents=[ - Person(), - Person(), - Person(), - Institution(), - ], - related_works=[ - Article(tags=[Tag(name='Science\n; Stuff')], identifiers=[WorkIdentifier(2)]) - ] - ), - CreativeWork( - tags=[Tag(name='Ghosts N Stuff')], - identifiers=[WorkIdentifier(3)], - related_agents=[ - Person(), - Person(), - Person(), - Organization(name='Aperture Science'), - Institution(), - ], - related_works=[ - DataSet(identifiers=[WorkIdentifier(4)], related_agents=[Consortium()]) - ] - ), - Publication( - tags=[Tag(name=' Science')], - identifiers=[WorkIdentifier(5)], - related_agents=[Organization(name='Umbrella Corporation')], - related_works=[ - Patent( - tags=[Tag(name='Science\n; Stuff')], - identifiers=[WorkIdentifier(6)] - ) - ] - ), -] - - -@pytest.mark.django_db -class TestDisambiguation: - - @pytest.fixture - def ingest_initial(self, Graph, ingest): - ingest(Graph(initial)) - - @pytest.mark.parametrize('input, model, delta', [ - ([Tag(name='Science')], models.Tag, 0), - ([Tag(name='Science; Things')], models.Tag, 1), - ([Tag(name='SCIENCE'), Tag(name='Ghosts')], models.Tag, 1), - ([Publication(identifiers=[WorkIdentifier(5)])], models.Publication, 0), - ([Publication(identifiers=[WorkIdentifier(3)])], models.Publication, 1), - ([Organization(name='Aperture Science')], models.Organization, 0), - ([Organization(name='Aperture science')], models.Organization, 1), - ]) - def test_disambiguate(self, input, model, delta, Graph, ingest_initial, ingest): - Graph.reseed() - # Nasty hack to avoid progres' fuzzy counting - before = model.objects.exclude(change=None).count() - - ingest(Graph(input)) - - assert (model.objects.exclude(change=None).count() - before) == delta - - @pytest.mark.parametrize('input', [ - [Tag(name='Not Ipsum')], - [Tag(name='No Step'), Tag(name='On'), Tag(name='Snek')], - [AgentIdentifier(), Organization(name='Team Snag \'em')], - [Person(identifiers=[AgentIdentifier()])], - [Publication(identifiers=[WorkIdentifier()])], - [Preprint(identifiers=[WorkIdentifier()], related_agents=[Person(), Consortium()], agent_relations=[Funder(), Publisher()])] - ]) - def test_reaccept(self, input, Graph, ingest_initial, ingest): - Graph.reseed() # Force new values to be generated - - first_cs = ingest(Graph(input)) - assert first_cs is not None - - second_cs = ingest(Graph(input)) - assert second_cs is None - - def test_no_changes(self, Graph, ingest_initial, ingest): - cs = ingest(Graph(initial)) - assert cs is None diff --git a/tests/share/disambiguation/test_person.py b/tests/share/disambiguation/test_person.py deleted file mode 100644 index 35cb8320f..000000000 --- a/tests/share/disambiguation/test_person.py +++ /dev/null @@ -1,91 +0,0 @@ -import pytest - -from share import models - -from tests.share.normalize.factories import * - - -initial = [ - Preprint( - identifiers=[WorkIdentifier(1)], - agent_relations=[ - Contributor(agent=Person(1, name='Bob Dylan')), - Creator(agent=Person(2)), - Creator(agent=Person(3)), - ] - ), - CreativeWork( - identifiers=[WorkIdentifier(2)], - agent_relations=[ - Host(agent=Person(4, name='Bill Lumbergh'), cited_as='Bill Lumbergh'), - Funder(agent=Person(name='Bill Gates')), - Publisher(agent=Person(6)), - ] - ), - CreativeWork( - identifiers=[WorkIdentifier(2)], - related_agents=[ - Person(), - Person(name='Laura Gates'), - Person(), - ] - ), - Publication( - identifiers=[WorkIdentifier(3)], - agent_relations=[ - Creator(agent=Person(7, identifiers=[AgentIdentifier(1)])) - ], - related_works=[ - Patent( - agent_relations=[ - Contributor(agent=Person(8, identifiers=[AgentIdentifier(2)])) - ], - identifiers=[WorkIdentifier(4)] - ) - ] - ), -] - - -@pytest.mark.django_db -class TestPersonDisambiguation: - - @pytest.fixture - def ingest_initial(self, Graph, ingest): - ingest(Graph(initial)) - - @pytest.mark.parametrize('input, model, delta', [ - ([Person(name='Bob Dylan')], models.Person, 1), - ([Person(7, identifiers=[AgentIdentifier(1)])], models.Person, 0), - ([Publication(identifiers=[WorkIdentifier(2)], agent_relations=[AgentWorkRelation(agent=Person(4, name='Bill Lumbergh'), cited_as='Bill Lumbergh')])], models.Person, 0), - ([Publication(identifiers=[WorkIdentifier(2)], related_agents=[Person(9)])], models.Person, 1), - ([CreativeWork(related_agents=[Person(name='Bill Gates')])], models.Person, 1), - ([Preprint(related_agents=[Person(8, identifiers=[AgentIdentifier(2)])])], models.Person, 0), - ]) - def test_disambiguate(self, input, model, delta, Graph, ingest_initial, ingest): - Graph.reseed() - # Nasty hack to avoid progres' fuzzy counting - before = model.objects.exclude(change=None).count() - - ingest(Graph(input)) - - assert (model.objects.exclude(change=None).count() - before) == delta - - @pytest.mark.parametrize('input', [ - [Person(identifiers=[AgentIdentifier()])], - [Person(identifiers=[AgentIdentifier()]), Person(identifiers=[AgentIdentifier()])], - [Publication(identifiers=[WorkIdentifier()], agent_relations=[Funder(agent=Person()), Publisher(agent=Person())])], - [Preprint(identifiers=[WorkIdentifier()], related_agents=[Person(), Person()], agent_relations=[Funder(agent=Person()), Publisher(agent=Person())])] - ]) - def test_reaccept(self, input, Graph, ingest_initial, ingest): - Graph.reseed() # Force new values to be generated - - first_cs = ingest(Graph(input)) - assert first_cs is not None - - second_cs = ingest(Graph(input)) - assert second_cs is None - - def test_no_changes(self, Graph, ingest_initial, ingest): - cs = ingest(Graph(initial)) - assert cs is None diff --git a/tests/share/disambiguation/test_subject.py b/tests/share/disambiguation/test_subject.py deleted file mode 100644 index efca37448..000000000 --- a/tests/share/disambiguation/test_subject.py +++ /dev/null @@ -1,69 +0,0 @@ -import pytest - -from share import models - -import tests.share.normalize.factories as f - - -initial = [ - f.Subject(name='Science!', uri='http://science.io'), - f.Subject(name='Art!', uri='http://art.io'), -] - - -@pytest.mark.django_db -class TestSubject: - - @pytest.fixture - def ingest_initial(self, Graph, ingest, system_user): - ingest(Graph(initial), user=system_user) - - @pytest.mark.parametrize('input, model_deltas', [ - ([f.Subject(name='Science!')], { - models.Subject: 0, - models.SubjectTaxonomy: 0, - }), - ([f.Publication(subjects=[f.Subject(name='Science!')])], { - models.Publication: 1, - models.Subject: 0, - models.ThroughSubjects: 1, - models.SubjectTaxonomy: 0, - }), - ([f.Publication(subjects=[f.Subject( - name='Science synonym!', - central_synonym=f.Subject(name='Science!') - )])], { - models.Publication: 1, - models.Subject: 1, - models.ThroughSubjects: 1, - models.SubjectTaxonomy: 1, - }), - ]) - def test_disambiguate(self, input, model_deltas, Graph, ingest_initial, ingest): - # Nasty hack to avoid progres' fuzzy counting - before = { - m: m.objects.exclude(id=None).count() - for m in model_deltas.keys() - } - - ingest(Graph(input)) - - after = { - m: m.objects.exclude(id=None).count() - for m in model_deltas.keys() - } - - for model, delta in model_deltas.items(): - assert after[model] - before[model] == delta - - @pytest.mark.parametrize('input', [ - (f.Subject(name='Not Science!', uri='http://science.io')), - (f.Subject(name='Science!')), - ]) - def test_protect_central_taxonomy(self, input, Graph, ingest_initial, ingest): - cs = ingest(Graph(input)) - assert cs is None - - def test_no_changes(self, Graph, ingest_initial, ingest): - cs = ingest(Graph(initial)) - assert cs is None diff --git a/tests/share/metadata_formats/base.py b/tests/share/metadata_formats/base.py index 375744189..1938b8420 100644 --- a/tests/share/metadata_formats/base.py +++ b/tests/share/metadata_formats/base.py @@ -1,8 +1,10 @@ import dateutil import pytest -from tests.factories import RawDatumFactory -from tests.factories.core import NormalizedDataFactory +from share.models.core import FormattedMetadataRecord +from share.util.extensions import Extensions + +from tests.factories import RawDatumFactory, NormalizedDataFactory FORMATTER_TEST_INPUTS = { @@ -326,13 +328,26 @@ @pytest.mark.django_db class BaseMetadataFormatterTest: - # override in child class + + ####### override these things ####### + + # formatter key, as registered in setup.py + formatter_key = None + + # dictionary with the same keys as `FORMATTER_TEST_INPUTS`, mapping to values + # that `assert_formatter_outputs_equal` will understand expected_outputs = {} - # override in child class - def test_formatter(self, normalized_datum, expected_output): + def assert_formatter_outputs_equal(self, actual_output, expected_output): + """raise AssertionError if the two outputs aren't equal + + @param actual_output (str): return value of the formatter's `.format()` method + @param expected_output: corresponding value from this class's `expected_outputs` dictionary + """ raise NotImplementedError + ####### don't override anything else ####### + @pytest.fixture(scope='class', autouse=True) def _sanity_check(self): assert FORMATTER_TEST_INPUTS.keys() == self.expected_outputs.keys(), f'check the test class\'s `expected_outputs` matches {__name__}.FORMATTER_TEST_INPUTS' @@ -341,6 +356,10 @@ def _sanity_check(self): def _test_key(self, request): return request.param + @pytest.fixture + def formatter(self): + return Extensions.get('share.metadata_formats', self.formatter_key)() + @pytest.fixture def formatter_test_input(self, _test_key): return FORMATTER_TEST_INPUTS[_test_key] @@ -359,3 +378,20 @@ def normalized_datum(self, formatter_test_input): ), **formatter_test_input['normalized_datum_kwargs'], ) + + def test_formatter(self, formatter, normalized_datum, expected_output): + actual_output = formatter.format(normalized_datum) + self.assert_formatter_outputs_equal(actual_output, expected_output) + + def test_save_formatted_records(self, normalized_datum, expected_output): + saved_records = FormattedMetadataRecord.objects.save_formatted_records( + suid=normalized_datum.raw.suid, + record_formats=[self.formatter_key], + normalized_datum=normalized_datum, + ) + if expected_output is None: + assert len(saved_records) == 0 + else: + assert len(saved_records) == 1 + actual_output = saved_records[0].formatted_metadata + self.assert_formatter_outputs_equal(actual_output, expected_output) diff --git a/tests/share/metadata_formats/test_oai_dc_formatter.py b/tests/share/metadata_formats/test_oai_dc_formatter.py index c3a7e819e..5f1bc9b5a 100644 --- a/tests/share/metadata_formats/test_oai_dc_formatter.py +++ b/tests/share/metadata_formats/test_oai_dc_formatter.py @@ -1,7 +1,5 @@ from lxml import etree -from share.metadata_formats.oai_dc import OaiDcFormatter - from tests.share.metadata_formats.base import BaseMetadataFormatterTest @@ -20,8 +18,9 @@ def xml_elements_equal(element_1, element_2): class TestOaiDcFormatter(BaseMetadataFormatterTest): - def test_formatter(self, normalized_datum, expected_output): - actual_output = OaiDcFormatter().format(normalized_datum) + formatter_key = 'oai_dc' + + def assert_formatter_outputs_equal(self, actual_output, expected_output): if expected_output is None: assert actual_output is None else: diff --git a/tests/share/metadata_formats/test_sharev2_elastic_formatter.py b/tests/share/metadata_formats/test_sharev2_elastic_formatter.py index 8daeb2d6a..91ae75722 100644 --- a/tests/share/metadata_formats/test_sharev2_elastic_formatter.py +++ b/tests/share/metadata_formats/test_sharev2_elastic_formatter.py @@ -2,7 +2,7 @@ import pytest from unittest.mock import patch -from share.metadata_formats.sharev2_elastic import ShareV2ElasticFormatter, format_type +from share.metadata_formats.sharev2_elastic import format_type from tests.share.metadata_formats.base import BaseMetadataFormatterTest @@ -21,10 +21,15 @@ def fake_id_encode(obj): class TestSharev2ElasticFormatter(BaseMetadataFormatterTest): - @patch('share.util.IDObfuscator.encode', wraps=fake_id_encode) - def test_formatter(self, encode_mock, normalized_datum, expected_output): - actual_output = json.loads(ShareV2ElasticFormatter().format(normalized_datum)) - assert actual_output == expected_output + @pytest.fixture(scope='class', autouse=True) + def patch_encode(self): + with patch('share.util.IDObfuscator.encode', wraps=fake_id_encode): + yield + + def assert_formatter_outputs_equal(self, actual_output, expected_output): + assert json.loads(actual_output) == expected_output + + formatter_key = 'sharev2_elastic' expected_outputs = { 'mycorrhizas': { diff --git a/tests/share/models/factories.py b/tests/share/models/factories.py deleted file mode 100644 index 883f5f863..000000000 --- a/tests/share/models/factories.py +++ /dev/null @@ -1,23 +0,0 @@ -import factory - -from share import models - -from tests.factories.core import * # noqa -from tests.factories.changes import * # noqa -from tests.factories.share_objects import * # noqa -from tests.factories.share_objects import ShareObjectFactory - - -AgentFactory = AbstractAgentFactory - - -class PreprintFactory(AbstractCreativeWorkFactory): - type = 'share.preprint' - - -class ThroughAgentWorkRelationFactory(ShareObjectFactory): - subject = factory.SubFactory(AgentWorkRelationFactory) - related = factory.SubFactory(AgentWorkRelationFactory) - - class Meta: - model = models.ThroughContributor diff --git a/tests/share/models/test_cascade.py b/tests/share/models/test_cascade.py deleted file mode 100644 index 99cf53d09..000000000 --- a/tests/share/models/test_cascade.py +++ /dev/null @@ -1,301 +0,0 @@ -import pytest - -from django.apps import apps - -from share import models -from share.harvest.base import FetchResult - -from tests.share.models import factories -from tests.share.normalize.factories import * - - -@pytest.mark.django_db -class TestDeleteCascadeShareObjects: - - def test_one_version(self): - cw = factories.AbstractCreativeWorkFactory(title='All About Cats') - - assert models.CreativeWork.objects.all().count() == 1 - assert models.AbstractCreativeWorkVersion.objects.count() == 1 - - cw.delete() - - assert models.CreativeWork.objects.all().count() == 0 - assert models.AbstractCreativeWorkVersion.objects.count() == 0 - - def test_many_versions(self): - cw = factories.AbstractCreativeWorkFactory(title='All about cats') - cw.administrative_change(title='All About Kets') - - assert models.CreativeWork.objects.all().count() == 1 - assert models.AbstractCreativeWorkVersion.objects.count() == 2 - - assert cw.versions.count() == 2 - - cw.delete() - - assert models.CreativeWork.objects.all().count() == 0 - assert models.AbstractCreativeWorkVersion.objects.count() == 0 - - def test_multiple_sources(self): - pass - - def test_no_truncate(self): - cws = [factories.AbstractCreativeWorkFactory() for _ in range(10)] - - assert models.CreativeWork.objects.all().count() == 10 - assert models.AbstractCreativeWorkVersion.objects.count() == 10 - - for i, cw in enumerate(cws): - cw.delete() - assert models.CreativeWork.objects.all().count() == 9 - i - assert models.AbstractCreativeWorkVersion.objects.count() == 9 - i - - -@pytest.mark.django_db -class TestDeleteCascadeRelations: - - def test_foreign_key(self): - identifier = factories.WorkIdentifierFactory() - - assert models.WorkIdentifier.objects.count() == 1 - assert models.AbstractCreativeWork.objects.count() == 1 - assert models.WorkIdentifierVersion.objects.count() == 1 - assert models.AbstractCreativeWorkVersion.objects.count() == 1 - - identifier.delete() - - assert models.WorkIdentifier.objects.count() == 0 - assert models.AbstractCreativeWork.objects.count() == 1 - assert models.WorkIdentifierVersion.objects.count() == 0 - assert models.AbstractCreativeWorkVersion.objects.count() == 1 - - def test_foreign_key_inverse(self): - identifier = factories.WorkIdentifierFactory() - - assert models.WorkIdentifier.objects.count() == 1 - assert models.AbstractCreativeWork.objects.count() == 1 - assert models.WorkIdentifierVersion.objects.count() == 1 - assert models.AbstractCreativeWorkVersion.objects.count() == 1 - - identifier.creative_work.delete() - - assert models.WorkIdentifier.objects.count() == 0 - assert models.AbstractCreativeWork.objects.count() == 0 - assert models.WorkIdentifierVersion.objects.count() == 0 - assert models.AbstractCreativeWorkVersion.objects.count() == 0 - - def test_many_to_many(self): - work = factories.AbstractCreativeWorkFactory() - for i in range(10): - factories.ThroughTagsFactory(creative_work=work, tag=factories.TagFactory(name=str(i))) - - for model, (count, version_count) in { - models.AbstractCreativeWork: (1, 1), - models.ThroughTags: (10, 10), - models.Tag: (10, 10), - }.items(): - assert model.objects.count() == count - assert model.VersionModel.objects.count() == version_count - - work.delete() - - for model, (count, version_count) in { - models.AbstractCreativeWork: (0, 0), - models.ThroughTags: (0, 0), - models.Tag: (10, 10), - }.items(): - assert model.objects.count() == count - assert model.VersionModel.objects.count() == version_count - - def test_many_to_many_inverse(self): - for i in range(10): - factories.ThroughTagsFactory(tag=factories.TagFactory(name=str(i))) - - for model, (count, version_count) in { - models.AbstractCreativeWork: (10, 10), - models.ThroughTags: (10, 10), - models.Tag: (10, 10), - }.items(): - assert model.objects.count() == count - assert model.VersionModel.objects.count() == version_count - - models.ThroughTags.objects.all().delete() - - for model, (count, version_count) in { - models.AbstractCreativeWork: (10, 10), - models.ThroughTags: (0, 0), - models.Tag: (10, 10), - }.items(): - assert model.objects.count() == count - assert model.VersionModel.objects.count() == version_count - - def test_one_to_one(self): - work = factories.AbstractCreativeWorkFactory(extra=factories.ExtraDataFactory()) - - assert work.extra is not None - assert work.extra_version is not None - - for model, (count, version_count) in { - models.AbstractCreativeWork: (1, 1), - models.ExtraData: (1, 1), - }.items(): - assert model.objects.count() == count - assert model.VersionModel.objects.count() == version_count - - work.delete() - - for model, (count, version_count) in { - models.AbstractCreativeWork: (0, 0), - models.ExtraData: (1, 1), - }.items(): - assert model.objects.count() == count - assert model.VersionModel.objects.count() == version_count - - def test_one_to_one_inverse(self): - work = factories.AbstractCreativeWorkFactory(extra=factories.ExtraDataFactory()) - - assert work.extra is not None - assert work.extra_version is not None - - for model, (count, version_count) in { - models.AbstractCreativeWork: (1, 1), - models.ExtraData: (1, 1), - }.items(): - assert model.objects.count() == count - assert model.VersionModel.objects.count() == version_count - - work.extra.delete() - - for model, (count, version_count) in { - models.AbstractCreativeWork: (0, 0), - models.ExtraData: (0, 0), - }.items(): - assert model.objects.count() == count - assert model.VersionModel.objects.count() == version_count - - -@pytest.mark.django_db -class TestDeleteCascadeNonShareObjects: - - def test_change(self): - work = factories.AbstractCreativeWorkFactory() - work.change.delete() - - assert models.AbstractCreativeWork.objects.count() == 0 - - def test_changeset(self): - work = factories.AbstractCreativeWorkFactory() - work.change.change_set.delete() - - assert models.Change.objects.count() == 0 - assert models.AbstractCreativeWork.objects.count() == 0 - - def test_normalizeddata(self): - work = factories.AbstractCreativeWorkFactory() - work.change.change_set.normalized_data.delete() - - assert models.Change.objects.count() == 0 - assert models.ChangeSet.objects.count() == 0 - assert models.AbstractCreativeWork.objects.count() == 0 - - def test_rawdata(self, source_config): - work = factories.AbstractCreativeWorkFactory(change__change_set__normalized_data__raw=models.RawDatum.objects.store_data(source_config, FetchResult('unique', 'data'))) - work.change.change_set.normalized_data.delete() - - assert models.Change.objects.count() == 0 - assert models.ChangeSet.objects.count() == 0 - assert models.NormalizedData.objects.count() == 0 - assert models.AbstractCreativeWork.objects.count() == 0 - - -@pytest.mark.django_db -class TestDeleteCascade: - initial = [ - Preprint( - tags=[Tag(name=' Science')], - identifiers=[WorkIdentifier(1)], - related_agents=[ - Person(), - Person(), - Person(), - Institution(), - ], - related_works=[ - Article(tags=[Tag(name='Science\n; Stuff')], identifiers=[WorkIdentifier(2)]) - ] - ), - CreativeWork( - tags=[Tag(name='Ghosts N Stuff')], - identifiers=[WorkIdentifier(3)], - related_agents=[ - Person(), - Person(), - Person(), - Organization(name='Aperture Science'), - Institution(), - ], - related_works=[ - DataSet(identifiers=[WorkIdentifier(4)], related_agents=[Consortium()]) - ] - ), - Publication( - tags=[Tag(name=' Science')], - identifiers=[WorkIdentifier(5)], - related_agents=[Organization(name='Umbrella Corporation')], - related_works=[ - Patent( - tags=[Tag(name='Science\n; Stuff')], - identifiers=[WorkIdentifier(6)] - ) - ] - ), - ] - - @pytest.mark.parametrize('queryset, deltas', [ - ( - models.NormalizedData.objects.all(), { - models.AbstractCreativeWork: -6, - models.AbstractAgent: -11, - models.ChangeSet: -1, - models.Change: -47, - } - ), ( - models.DataSet.objects.all(), { - models.DataSet: -1, - models.AbstractCreativeWork: -1, - models.Preprint: 0, - models.Change: 0, - } - ), ( - models.ChangeSet.objects.all(), { - models.Change: -47, - models.DataSet: -1, - models.AbstractAgent: -11, - models.AbstractCreativeWork: -6, - models.NormalizedData: 0, - } - ), ( - models.Tag.objects.filter(name='science'), { - models.Tag: -1, - apps.get_model('share.Tag_sources'): -1, - models.Tag: -1, - models.Tag.VersionModel: -1, - models.ThroughTags: -4, - models.ThroughTags.VersionModel: -4, - models.ChangeSet: 0, - models.Change: 0, - models.AbstractCreativeWork: 0, - models.AbstractCreativeWork.VersionModel: 0, - } - ) - ]) - def test_delete_cascade(self, queryset, deltas, Graph, ingest): - ingest(Graph(self.initial)) - - before = {model: model.objects.count() for model in deltas.keys()} - - queryset.delete() - - for model, delta in deltas.items(): - assert model.objects.count() - before[model] == delta diff --git a/tests/share/models/test_change.py b/tests/share/models/test_change.py deleted file mode 100644 index d9e64a431..000000000 --- a/tests/share/models/test_change.py +++ /dev/null @@ -1,185 +0,0 @@ -import pytest -import pendulum - -from share.exceptions import IngestConflict -from share.models import AbstractCreativeWork -from share.models import AgentWorkRelation -from share.models import Person -from share.models import Tag -from share.models.change import Change -from share.models.change import ChangeSet -from share.util import IDObfuscator -from share.ingest.ingester import Ingester - -from tests import factories - - -@pytest.fixture -def ld_graph(): - return { - '@context': {}, - '@graph': [ - {'@id': '_:d486fd737bea4fbe9566b7a2842651ef', '@type': 'Organization', 'name': 'Department of Physics'}, - - {'@id': '_:c4f10e02785a4b4d878f48d08ffc7fce', 'related': {'@type': 'Organization', '@id': '_:d486fd737bea4fbe9566b7a2842651ef'}, '@type': 'IsAffiliatedWith', 'subject': {'@type': 'Person', '@id': '_:7e742fa3377e4f119e36f8629144a0bc'}}, - {'@id': '_:7e742fa3377e4f119e36f8629144a0bc', 'related_agents': [{'@type': 'IsAffiliatedWith', '@id': '_:c4f10e02785a4b4d878f48d08ffc7fce'}], '@type': 'Person', 'family_name': 'Prendergast', 'given_name': 'David'}, - {'@id': '_:687a4ba2cbd54ab7a2f2c3cd1777ea8a', '@type': 'Creator', 'creative_work': {'@type': 'Article', '@id': '_:6203fec461bb4b3fa956772acbd9c50d'}, 'agent': {'@type': 'Person', '@id': '_:7e742fa3377e4f119e36f8629144a0bc'}}, - - {'@id': '_:69e859cefed140bd9b717c5b610d300c', '@type': 'Organization', 'name': 'NMRC, University College, Cork, Ireland'}, - - {'@id': '_:2fd829eeda214adca2d4d34d02b10328', 'related': {'@type': 'Organization', '@id': '_:69e859cefed140bd9b717c5b610d300c'}, '@type': 'IsAffiliatedWith', 'subject': {'@type': 'Person', '@id': '_:ed3cc2a50f6d499db933a28d16bca5d6'}}, - {'@id': '_:ed3cc2a50f6d499db933a28d16bca5d6', 'related_agents': [{'@type': 'IsAffiliatedWith', '@id': '_:2fd829eeda214adca2d4d34d02b10328'}], '@type': 'Person', 'family_name': 'Nolan', 'given_name': 'M.'}, - {'@id': '_:27961f3c7c644101a500772477aff304', '@type': 'Creator', 'creative_work': {'@type': 'Article', '@id': '_:6203fec461bb4b3fa956772acbd9c50d'}, 'agent': {'@type': 'Person', '@id': '_:ed3cc2a50f6d499db933a28d16bca5d6'}}, - - {'@id': '_:d4f10e02785a4b4d878f48d08ffc7fce', 'related': {'@type': 'Organization', '@id': '_:d486fd737bea4fbe9566b7a2842651ef'}, '@type': 'IsAffiliatedWith', 'subject': {'@type': 'Person', '@id': '_:9a1386475d314b9bb524931e24361aaa'}}, - {'@id': '_:9a1386475d314b9bb524931e24361aaa', 'related_agents': [{'@type': 'IsAffiliatedWith', '@id': '_:d4f10e02785a4b4d878f48d08ffc7fce'}], '@type': 'Person', 'family_name': 'Filippi', 'given_name': 'Claudia'}, - {'@id': '_:bf7726af4542405888463c796e5b7686', '@type': 'Creator', 'creative_work': {'@type': 'Article', '@id': '_:6203fec461bb4b3fa956772acbd9c50d'}, 'agent': {'@type': 'Person', '@id': '_:9a1386475d314b9bb524931e24361aaa'}}, - - {'@id': '_:e4f10e02785a4b4d878f48d08ffc7fce', 'related': {'@type': 'Organization', '@id': '_:d486fd737bea4fbe9566b7a2842651ef'}, '@type': 'IsAffiliatedWith', 'subject': {'@type': 'Person', '@id': '_:78639db07e2e4ee88b422a8920d8a095'}}, - {'@id': '_:78639db07e2e4ee88b422a8920d8a095', 'related_agents': [{'@type': 'IsAffiliatedWith', '@id': '_:e4f10e02785a4b4d878f48d08ffc7fce'}], '@type': 'Person', 'family_name': 'Fahy', 'given_name': 'Stephen'}, - {'@id': '_:18d151204d7c431388a7e516defab1bc', '@type': 'Creator', 'creative_work': {'@type': 'Article', '@id': '_:6203fec461bb4b3fa956772acbd9c50d'}, 'agent': {'@type': 'Person', '@id': '_:78639db07e2e4ee88b422a8920d8a095'}}, - - {'@id': '_:5fd829eeda214adca2d4d34d02b10328', 'related': {'@type': 'Organization', '@id': '_:69e859cefed140bd9b717c5b610d300c'}, '@type': 'IsAffiliatedWith', 'subject': {'@type': 'Person', '@id': '_:f4cec0271c7d4085bac26dbb2b32a002'}}, - {'@id': '_:f4cec0271c7d4085bac26dbb2b32a002', 'related_agents': [{'@type': 'IsAffiliatedWith', '@id': '_:5fd829eeda214adca2d4d34d02b10328'}], '@type': 'Person', 'family_name': 'Greer', 'given_name': 'J.'}, - {'@id': '_:a17f28109536459ca02d99bf777400ae', '@type': 'Creator', 'creative_work': {'@type': 'Article', '@id': '_:6203fec461bb4b3fa956772acbd9c50d'}, 'agent': {'@type': 'Person', '@id': '_:f4cec0271c7d4085bac26dbb2b32a002'}}, - - {'@id': '_:6203fec461bb4b3fa956772acbd9c50d', 'date_updated': '2016-10-20T00:00:00+00:00', 'related_agents': [{'@type': 'Creator', '@id': '_:687a4ba2cbd54ab7a2f2c3cd1777ea8a'}, {'@type': 'Creator', '@id': '_:27961f3c7c644101a500772477aff304'}, {'@type': 'Creator', '@id': '_:bf7726af4542405888463c796e5b7686'}, {'@type': 'Creator', '@id': '_:18d151204d7c431388a7e516defab1bc'}, {'@type': 'Creator', '@id': '_:a17f28109536459ca02d99bf777400ae'}], 'title': 'Impact of Electron-Electron Cusp on Configuration Interaction Energies', '@type': 'Article', 'description': ' The effect of the electron-electron cusp on the convergence of configuration\ninteraction (CI) wave functions is examined. By analogy with the\npseudopotential approach for electron-ion interactions, an effective\nelectron-electron interaction is developed which closely reproduces the\nscattering of the Coulomb interaction but is smooth and finite at zero\nelectron-electron separation. The exact many-electron wave function for this\nsmooth effective interaction has no cusp at zero electron-electron separation.\nWe perform CI and quantum Monte Carlo calculations for He and Be atoms, both\nwith the Coulomb electron-electron interaction and with the smooth effective\nelectron-electron interaction. We find that convergence of the CI expansion of\nthe wave function for the smooth electron-electron interaction is not\nsignificantly improved compared with that for the divergent Coulomb interaction\nfor energy differences on the order of 1 mHartree. This shows that, contrary to\npopular belief, description of the electron-electron cusp is not a limiting\nfactor, to within chemical accuracy, for CI calculations.\n'} # noqa - ] - } - - -@pytest.mark.django_db -class TestChange: - - def test_create_person(self, change_factory): - change_set = change_factory.from_graph({ - '@graph': [{ - '@id': '_:1234', - '@type': 'Person', - 'given_name': 'John', - 'family_name': 'Doe' - }] - }) - - assert change_set.status == ChangeSet.STATUS.pending - assert change_set.changes.count() == 1 - assert change_set.changes.first().type == Change.TYPE.create - - (person, ) = change_set.accept() - - assert person.change == change_set.changes.first() - assert change_set.status == ChangeSet.STATUS.accepted - - def test_update_creative_work(self, change_factory): - preprint, identifier = change_factory.from_graph({ - '@graph': [{ - '@id': '_:5678', - '@type': 'workidentifier', - 'uri': 'http://share.osf.io', - 'creative_work': {'@id': '_:890', '@type': 'preprint'} - }, { - '@id': '_:890', - '@type': 'preprint', - 'title': 'All about Cats and more', - 'identifiers': [{'@id': '_:5678', '@type': 'workidentifier'}] - }] - }).accept() - - change_set = change_factory.from_graph({ - '@graph': [{ - '@id': '_:1234', - '@type': 'workidentifier', - 'uri': 'http://share.osf.io', - 'creative_work': {'@id': '_:890', '@type': 'preprint'} - }, { - '@id': '_:890', - '@type': 'preprint', - 'title': 'JUST ABOUT CATS', - 'identifiers': [{'@id': '_:1234', '@type': 'workidentifier'}], - }] - }, disambiguate=True) - - assert change_set.changes.count() == 1 - assert change_set.changes.first().target == preprint - - -@pytest.mark.django_db -class TestChangeGraph: - - def test_changes(self, change_factory, ld_graph): - change_factory.from_graph(ld_graph).accept() - - assert Person.objects.filter(pk__isnull=False).count() == 5 - assert AgentWorkRelation.objects.filter(pk__isnull=False).count() == 5 - assert AbstractCreativeWork.objects.filter(pk__isnull=False).count() == 1 - - def test_change_existing(self, change_factory, jane_doe): - change_set = change_factory.from_graph({ - '@graph': [{ - '@id': IDObfuscator.encode(jane_doe), - '@type': 'Person', - 'given_name': 'John' - }] - }, disambiguate=True) - - assert change_set.changes.first().target == jane_doe - - assert jane_doe.given_name == 'Jane' - - change_set.accept() - jane_doe.refresh_from_db() - - assert jane_doe.given_name == 'John' - - def test_unique_violation_raises(self, change_factory, change_ids): - Tag.objects.create(name='mycooltag', change_id=change_ids.get()) - - change_set = change_factory.from_graph({ - '@graph': [{ - '@type': 'Tag', - '@id': '_:1234', - 'name': 'mycooltag' - }] - }, disambiguate=False) - - with pytest.raises(IngestConflict): - change_set.accept() - - def test_date_updated_update(self, change_ids, change_factory, all_about_anteaters): - """ - Submitting an identical date as a string should be recognized as no change. - """ - all_about_anteaters.date_updated = pendulum.now() - all_about_anteaters.change_id = change_ids.get() - all_about_anteaters.save() - - change_set = change_factory.from_graph({ - '@graph': [{ - '@type': 'article', - '@id': IDObfuscator.encode(all_about_anteaters), - 'date_updated': str(all_about_anteaters.date_updated) - }] - }, disambiguate=True) - - assert change_set is None - - def test_add_multiple_sources(self): - source1 = factories.SourceFactory() - source2 = factories.SourceFactory() - - work = factories.AbstractCreativeWorkFactory(title='All about Canada') - data = [{'@id': IDObfuscator.encode(work), '@type': 'creativework', 'title': 'All aboot Canada'}] - - assert work.sources.count() == 0 - - Ingester(data, 'suid1').as_user(source1.user).ingest(index=False) - - work.refresh_from_db() - assert work.title == 'All aboot Canada' - assert work.sources.count() == 1 - - Ingester(data, 'suid2').as_user(source2.user).ingest(index=False) - - work.refresh_from_db() - assert work.title == 'All aboot Canada' - assert work.sources.count() == 2 diff --git a/tests/share/models/test_contribution.py b/tests/share/models/test_contribution.py deleted file mode 100644 index 8d98272f2..000000000 --- a/tests/share/models/test_contribution.py +++ /dev/null @@ -1,76 +0,0 @@ -import pytest - -from django.db.utils import IntegrityError -from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ - -from tests.share.models import factories - - -@pytest.mark.django_db -class TestAbstractAgentWorkRelation: - - def test_exists(self): - x = factories.PreprintFactory(contributors=5) - assert x.related_agents.count() == 5 - for contributor in x.agent_relations.all(): - assert contributor.creative_work == x - assert list(contributor.agent.related_works.all()) == [x] - - def test_unique(self): - pp = factories.PreprintFactory() - person = factories.AgentFactory(type='share.person') - factories.AgentWorkRelationFactory(creative_work=pp, agent=person, type='share.creator') - - with pytest.raises(IntegrityError): - factories.AgentWorkRelationFactory(creative_work=pp, agent=person, type='share.creator') - - def test_many_contribution_types(self): - pp = factories.PreprintFactory(contributors=0) - person = factories.AgentFactory(type='share.person') - funding = factories.AgentWorkRelationFactory(creative_work=pp, agent=person, type='share.funder') - - assert pp.related_agents.count() == 1 - assert pp.agent_relations.count() == 1 - - collaboration = factories.AgentWorkRelationFactory(creative_work=pp, agent=person, type='share.creator') - - assert funding != collaboration - assert pp.related_agents.count() == 2 - assert pp.agent_relations.count() == 2 - - -@pytest.mark.django_db -class TestThroughAgentWorkRelation: - - def test_exists(self): - pp = factories.PreprintFactory(contributors=0) - person = factories.AgentFactory(type='share.person') - consortium = factories.AgentFactory(type='share.consortium') - consortium_collaboration = factories.AgentWorkRelationFactory(creative_work=pp, agent=consortium, type='share.contributor') - collaboration = factories.AgentWorkRelationFactory(creative_work=pp, agent=person, type='share.creator') - factories.ThroughAgentWorkRelationFactory(subject=collaboration, related=consortium_collaboration, change=factories.ChangeFactory()) - - assert list(collaboration.contributed_through.all()) == [consortium_collaboration] - - def test_must_share_creative_work(self): - pp1 = factories.PreprintFactory(contributors=0) - pp2 = factories.PreprintFactory(contributors=0) - person = factories.AgentFactory(type='share.person') - consortium = factories.AgentFactory(type='share.consortium') - - consortium_collaboration = factories.AgentWorkRelationFactory(creative_work=pp1, agent=consortium, type='share.creator') - collaboration = factories.AgentWorkRelationFactory(creative_work=pp2, agent=person, type='share.creator') - - with pytest.raises(ValidationError) as e: - factories.ThroughAgentWorkRelationFactory(subject=collaboration, related=consortium_collaboration) - assert e.value.args == (_('ThroughContributors must contribute to the same AbstractCreativeWork'), None, None) - - def test_cannot_be_self(self): - pp = factories.PreprintFactory(contributors=0) - person = factories.AgentFactory(type='share.person') - collaboration = factories.AgentWorkRelationFactory(creative_work=pp, agent=person, type='share.creator') - - with pytest.raises(ValidationError) as e: - factories.ThroughAgentWorkRelationFactory(subject=collaboration, related=collaboration) - assert e.value.args == (_('A contributor may not contribute through itself'), None, None) diff --git a/tests/share/models/test_shareobject.py b/tests/share/models/test_shareobject.py deleted file mode 100644 index 62e362caf..000000000 --- a/tests/share/models/test_shareobject.py +++ /dev/null @@ -1,231 +0,0 @@ -import pytest - -from share import models -from share.management.commands.maketriggermigrations import Command -from share.models import AgentIdentifier -from share.models import Article -from share.models import Person -from share.models import Preprint -from share.models.base import ShareObject - -from tests import factories - - -@pytest.mark.django_db -def test_build_trigger(): - assert issubclass(Person, ShareObject) - - trigger_build = Command() - procedure, trigger = trigger_build.build_operations(Person) - - assert trigger.reversible is True - assert 'DROP TRIGGER share_agent_change' in trigger.reverse_sql - assert 'DROP TRIGGER IF EXISTS share_agent_change' in trigger.sql - - assert procedure.reversible is True - assert 'DROP FUNCTION before_share_agent_change' in procedure.reverse_sql - assert 'CREATE OR REPLACE FUNCTION before_share_agent_change' in procedure.sql - - -class TestVersioning: - - @pytest.mark.django_db - def test_timestamping(self, change_ids): - p = Person(given_name='John', family_name='Doe', change_id=change_ids.get()) - p.save() - - created, modified = p.date_created, p.date_modified - - assert (p.date_created - p.date_modified).total_seconds() < 1 - - p.given_name = 'Jane' - p.change_id = change_ids.get() - p.save() - - assert modified < p.date_modified - assert created == p.date_created - - @pytest.mark.django_db - def test_creates_version(self, change_ids): - p = Person(given_name='John', family_name='Doe', change_id=change_ids.get()) - p.save() - p.refresh_from_db() - - assert isinstance(p.version, Person.VersionModel) - - @pytest.mark.django_db - def test_simple(self, change_ids): - p = Person(given_name='John', family_name='Doe', change_id=change_ids.get()) - p.save() - - p.given_name = 'Jane' - p.change_id = change_ids.get() - p.save() - - assert p.versions.all()[0].given_name == 'Jane' - assert p.versions.all()[1].given_name == 'John' - - @pytest.mark.django_db - def test_many_versions(self, change_ids): - p = Person(given_name='John', family_name='Doe', change_id=change_ids.get()) - p.save() - - names = ['Jane', 'John', 'Jone', 'Jane', 'James', 'Joe', 'Jim', 'Jack', 'Jacklynn'] - - for name in names: - p.given_name = name - p.change_id = change_ids.get() - p.save() - - p.refresh_from_db() - assert p.versions.first() == p.version - - for i, name in enumerate(reversed(['John'] + names)): - assert p.versions.all()[i].given_name == name - - @pytest.mark.django_db - def test_relations(self, john_doe, change_ids): - ident = AgentIdentifier.objects.create( - uri='http://dinosaurs.sexy/john_doe', - agent=john_doe, - agent_version=john_doe.version, - change_id=change_ids.get() - ) - - ident.refresh_from_db() - john_doe.refresh_from_db() - - assert john_doe.identifiers.count() == 1 - - assert john_doe == ident.agent - assert john_doe.version == ident.agent_version - - @pytest.mark.django_db - def test_relations_related_changed(self, john_doe, change_ids): - ident = AgentIdentifier.objects.create( - uri='http://dinosaurs.sexy/john_doe', - agent=john_doe, - agent_version=john_doe.version, - change_id=change_ids.get() - ) - - ident.refresh_from_db() - john_doe.refresh_from_db() - - john_doe.given_name = 'James' - john_doe.change_id = change_ids.get() - john_doe.save() - john_doe.refresh_from_db() - - assert john_doe.identifiers.count() == 1 - - assert john_doe == ident.agent - assert john_doe.version != ident.agent_version - assert john_doe.versions.last() == ident.agent_version - - assert ident == john_doe.identifiers.first() - - -@pytest.mark.django_db -class TestAdministrativeChange: - - def test_must_change(self, john_doe): - with pytest.raises(ValueError) as e: - john_doe.administrative_change() - assert e.value.args == ('Pass allow_empty=True to allow empty changes', ) - - def test_allow_empty(self, john_doe): - john_doe.administrative_change(allow_empty=True) - - def test_works(self, john_doe): - assert john_doe.version == john_doe.versions.first() - assert john_doe.versions.count() == 1 - assert john_doe.given_name == 'John' - john_doe.administrative_change(given_name='Jane') - assert john_doe.given_name == 'Jane' - assert john_doe.versions.count() == 2 - assert john_doe.change.change_set.normalized_data.source.username == 'system' - - def test_invalid_attribute(self, john_doe): - with pytest.raises(AttributeError) as e: - john_doe.administrative_change(favorite_animal='Anteater') - assert e.value.args == ('favorite_animal', ) - - def test_transition_types(self, all_about_anteaters): - assert all_about_anteaters.type == 'share.article' - all_about_anteaters.administrative_change(type='share.preprint') - assert all_about_anteaters.type == 'share.preprint' - - with pytest.raises(Article.DoesNotExist): - all_about_anteaters.refresh_from_db() - assert Preprint.objects.get(pk=all_about_anteaters.pk) - - def test_related_objects(self): - work = factories.AbstractCreativeWorkFactory() - identifier = factories.WorkIdentifierFactory() - - assert identifier.creative_work != work - - identifier.administrative_change(creative_work=work) - identifier.refresh_from_db() - - assert identifier.creative_work == work - - -@pytest.mark.django_db -class TestDefrankenize: - - def test_must_be_really_sure_about_it(self): - work = factories.AbstractCreativeWorkFactory() - - with pytest.raises(ValueError) as e: - work.defrankenize() - assert e.value.args == ('You have to be really sure about this', ) - - with pytest.raises(ValueError) as e: - work.defrankenize(True) - assert e.value.args == ('You have to be really sure about this', ) - - work.defrankenize(im_really_sure_about_this=True) - - def test_resets_name(self): - work = factories.AbstractCreativeWorkFactory(title='This is a franken', is_deleted=False) - - work.defrankenize(im_really_sure_about_this=True) - - assert work.title == 'Defrankenized work' - assert work.is_deleted is True - - @pytest.mark.parametrize('agents', [0, 5]) - @pytest.mark.parametrize('idents', [0, 5]) - @pytest.mark.parametrize('iworks', [0, 5]) - @pytest.mark.parametrize('oworks', [0, 5]) - @pytest.mark.parametrize('tags', [0, 5]) - def test_removes_relations(self, idents, agents, oworks, iworks, tags): - work = factories.AbstractCreativeWorkFactory(title='This is a franken', is_deleted=False) - - factories.WorkIdentifierFactory.create_batch(idents, creative_work=work) - factories.AgentWorkRelationFactory.create_batch(agents, creative_work=work) - factories.AbstractWorkRelationFactory.create_batch(iworks, subject=work) - factories.AbstractWorkRelationFactory.create_batch(oworks, related=work) - factories.ThroughTagsFactory.create_batch(tags, creative_work=work) - - work.defrankenize(im_really_sure_about_this=True) - - assert work.is_deleted is True - assert work.title == 'Defrankenized work' - - # We want to keep these around for now - assert work.changes.exists() is True - assert work.versions.exists() is True - - # Didn't delete any agents - assert models.AbstractAgent.objects.count() == agents - assert models.Tag.objects.count() == tags - assert models.AbstractCreativeWork.objects.count() == 1 + oworks + iworks - - for field in models.AbstractCreativeWork._meta.get_fields(): - if not field.one_to_many or field.name in ('changes', 'versions'): - continue - - assert getattr(work, field.get_accessor_name()).count() == 0 diff --git a/tests/share/normalize/factories.py b/tests/share/normalize/factories.py index 791b94659..70e9cfb7c 100644 --- a/tests/share/normalize/factories.py +++ b/tests/share/normalize/factories.py @@ -11,11 +11,8 @@ import factory import factory.fuzzy -from typedmodels.models import TypedModel - -from django.apps import apps - -from share import models +from share.schema import ShareV2Schema +from share.schema.shapes import RelationShape from share.transform.chain.links import IRILink from share.util import TopologicalSorter from share.util.graph import MutableGraph, MutableNode @@ -24,13 +21,13 @@ logger = logging.getLogger(__name__) +sharev2_schema = ShareV2Schema() used_ids = set() _Faker = faker.Faker() -def format_id(model, id): - id_namespace = model._meta.concrete_model._meta.model_name.lower().replace('abstract', '') - return '_:{}--{}'.format(id_namespace, id) +def format_id(type_name, id): + return '_:{}--{}'.format(type_name, id) class FactoryGraph(MutableGraph): @@ -170,8 +167,8 @@ def __init__(self, graph, random_states, normalize_fields=False): self.random_states = random_states self.normalize_fields = normalize_fields - def get_factory(self, model): - return globals()[model._meta.concrete_model.__name__ + 'Factory'] + def get_factory(self, schema_type): + return globals()[schema_type.concrete_type + 'Factory'] def build_nodes(self, nodes): for n in nodes: @@ -196,56 +193,61 @@ def build(self, attrs): if self.normalize_fields: attrs['parse'] = True - model = apps.get_model('share', node_type) - relations = {} for key in tuple(attrs.keys()): if isinstance(attrs[key], (dict, list)): relations[key] = attrs.pop(key) + schema_type = sharev2_schema.get_type(node_type.replace('Abstract', '')) + # Extract/generate required relations. # e.g. WorkIdentifier requires a work, Creator requires work and agent - for f in model._meta.fields: - if f.name not in attrs and f.is_relation and not f.null and f.editable and f.concrete: + for field_name in schema_type.explicit_fields: + field = sharev2_schema.get_field(node_type, field_name) + if ( + field_name not in attrs + and field.is_relation + and field.is_required + ): try: - relation = relations.pop(f.name) + relation = relations.pop(field_name) except KeyError: # Value missing for required relation; generate a fake one - relation = {'type': f.remote_field.model._meta.concrete_model._meta.model_name} - attrs[f.name] = self.build(relation) + relation = {'type': field.related_concrete_type} + attrs[field_name] = self.build(relation) if sparse: # Don't generate fake data for missing fields node = FactoryNode(self.graph, type=node_type, **attrs) else: # If it's a specific type, pass it along, otherwise let the factory choose a subtype - concrete_model_name = model._meta.concrete_model._meta.model_name - if node_type != concrete_model_name: + concrete_type_name = schema_type.concrete_type + if node_type != concrete_type_name: attrs['type'] = node_type if seed: - seed_ctx = self.random_states.seed(seed=str(seed) + concrete_model_name) + seed_ctx = self.random_states.seed(seed=str(seed) + concrete_type_name) else: - seed_ctx = self.random_states.seed(name=concrete_model_name) + seed_ctx = self.random_states.seed(name=concrete_type_name) with seed_ctx: - node = self.get_factory(model)(graph=self.graph, **attrs) + node = self.get_factory(schema_type)(graph=self.graph, **attrs) # Build specified *-to-many relations for key, value in sorted(relations.items(), key=lambda x: x[0]): - field = model._meta.get_field(key) + field = sharev2_schema.get_field(node_type, key) if isinstance(value, list): - if field.many_to_many: + if field.relation_shape == RelationShape.MANY_TO_MANY: related = [self.build(v) for v in value] for rel in related: self.build({ - 'type': field.remote_field.through._meta.model_name, - field.m2m_field_name(): node, - field.m2m_reverse_field_name(): rel, + 'type': field.through_concrete_type, + field.incoming_through_relation: node, + field.outgoing_through_relation: rel, }) else: - reverse_name = field.remote_field.name + reverse_name = field.inverse_relation for v in value: v[reverse_name] = node self.build(v) @@ -280,10 +282,12 @@ class TypedGraphNodeFactory(GraphNodeFactory): @factory.lazy_attribute def type(stub): model_name = re.sub('Factory$', '', stub._LazyStub__model_class.__name__) - model = apps.get_model('share', model_name) - if issubclass(model, TypedModel) and model._meta.concrete_model is model: - model = random.choice(model._meta.concrete_model.get_type_classes()) - return model._meta.model_name.lower() + schema_type = sharev2_schema.get_type(model_name.replace('Abstract', '')) + if schema_type.concrete_type == model_name: + model_name = random.choice( + list(sharev2_schema.get_type_names(schema_type.concrete_type)) + ) + return model_name.lower() class AbstractAgentFactory(TypedGraphNodeFactory): @@ -403,19 +407,16 @@ def parse(self, _, parse, **kwargs): self['host'] = parsed['authority'] -def _params(seed=None, id=None, type=None, model=None, **kwargs): - ret = {'type': type, **kwargs} +def _params(seed=None, id=None, schema_type=None, model=None, **kwargs): + ret = {'type': schema_type.name.lower(), **kwargs} if id is not None: - ret['id'] = format_id(model, id) + ret['id'] = format_id(schema_type.concrete_type.lower().replace('abstract', ''), id) if seed is not None: ret['seed'] = seed return ret __all__ = () -for model_name in dir(models): - model = getattr(models, model_name) - if not hasattr(model, 'VersionModel'): - continue - __all__ += (model_name, ) - locals()[model_name] = functools.partial(_params, type=model_name.lower(), model=model) + +for schema_type in sharev2_schema.schema_types.values(): + locals()[schema_type.name] = functools.partial(_params, schema_type=schema_type) diff --git a/tests/share/normalize/test_harness.py b/tests/share/normalize/test_harness.py index 3701a0c5e..53e386da4 100644 --- a/tests/share/normalize/test_harness.py +++ b/tests/share/normalize/test_harness.py @@ -1,4 +1,18 @@ -from tests.share.normalize.factories import * +from tests.share.normalize.factories import ( + Agent, + AgentIdentifier, + Article, + CreativeWork, + Institution, + Organization, + Patent, + Person, + Preprint, + Publication, + Tag, + ThroughTags, + WorkIdentifier, +) from tests.share.normalize.factories import FactoryGraph diff --git a/tests/share/normalize/test_models.py b/tests/share/normalize/test_models.py index ab2e37a0c..2bf1afd4a 100644 --- a/tests/share/normalize/test_models.py +++ b/tests/share/normalize/test_models.py @@ -90,6 +90,7 @@ def test_normalize_tag(self, input, output, Graph, ExpectedGraph): Tag(name='Crash ,Bandicoot '), ], [Tag(name='bandicoot'), Tag(name='crash')]), ]) + @pytest.mark.skip def test_normalize_tags_on_work(self, input, output, Graph, ExpectedGraph): graph = Graph(CreativeWork(tags=input)) Regulator(validate=False).regulate(graph) @@ -177,6 +178,7 @@ def test_normalize_person(self, input, output, Graph, ExpectedGraph): Person(name='B. D. Dylan', identifiers=[AgentIdentifier(1)]) ], [Person(name='B. D. Dylan', identifiers=[AgentIdentifier(1)])]), ]) + @pytest.mark.skip def test_normalize_person_relation(self, input, output, Graph, ExpectedGraph): graph = Graph(*input) Regulator(validate=False).regulate(graph) @@ -259,6 +261,7 @@ def test_normalize_agent(self, input, output, Graph, ExpectedGraph): Institution(name='Cook Institute', identifiers=[AgentIdentifier(1)]) ], [Institution(name='Cooking Institute', identifiers=[AgentIdentifier(1)])]), ]) + @pytest.mark.skip def test_normalize_organization_institution_name(self, input, output, Graph, ExpectedGraph): graph = Graph(*input) Regulator(validate=False).regulate(graph) @@ -328,6 +331,7 @@ def test_normalize_organization_institution_name(self, input, output, Graph, Exp Host(cited_as='Cook Institute', agent=Institution(id=1)) ]), ]) + @pytest.mark.skip def test_normalize_mixed_agent_relation(self, input, output, Graph, ExpectedGraph): graph = Graph(CreativeWork(agent_relations=input)) Regulator(validate=False).regulate(graph) @@ -416,6 +420,7 @@ def test_normalize_mixed_agent_relation(self, input, output, Graph, ExpectedGrap Creator(cited_as='Magpie', agent=Person(id=0, name='Magpie', identifiers=[AgentIdentifier(1, id=1)])), ]), ]) + @pytest.mark.skip def test_normalize_contributor_creator_relation(self, input, output, Graph, ExpectedGraph): graph = Graph(CreativeWork(agent_relations=input)) Regulator(validate=False).regulate(graph) @@ -448,6 +453,7 @@ def test_normalize_contributor_creator_relation(self, input, output, Graph, Expe CreativeWork(1, id=1), ), ]) + @pytest.mark.skip def test_normalize_related_work(self, input, output, Graph, ExpectedGraph): graph = Graph(input) Regulator(validate=False).regulate(graph) diff --git a/tests/share/search/test_elastic_manager.py b/tests/share/search/test_elastic_manager.py index 878f75ddf..e820adda5 100644 --- a/tests/share/search/test_elastic_manager.py +++ b/tests/share/search/test_elastic_manager.py @@ -4,7 +4,7 @@ from elasticsearch.exceptions import NotFoundError from share.search.elastic_manager import ElasticManager -from share.search.index_setup import ShareClassicIndexSetup, PostRendBackcompatIndexSetup +from share.search.index_setup import PostRendBackcompatIndexSetup, TroveV0IndexSetup class TestIsolatedElasticManager: @@ -13,12 +13,12 @@ def isolated_elastic_manager(self, settings): custom_settings = { **settings.ELASTICSEARCH, 'INDEXES': { - 'classic_index': { - 'INDEX_SETUP': 'share_classic', - }, 'postrend_index': { 'INDEX_SETUP': 'postrend_backcompat', }, + 'trove_index': { + 'INDEX_SETUP': 'trove_v0', + }, }, } @@ -27,16 +27,17 @@ def isolated_elastic_manager(self, settings): yield isolated_elastic_manager @pytest.mark.parametrize('index_name, expected_setup_class', [ - ('classic_index', ShareClassicIndexSetup), ('postrend_index', PostRendBackcompatIndexSetup), + ('trove_index', TroveV0IndexSetup), ]) def test_get_index_setup(self, isolated_elastic_manager, index_name, expected_setup_class): index_setup = isolated_elastic_manager.get_index_setup(index_name) assert isinstance(index_setup, expected_setup_class) @pytest.mark.parametrize('index_name', [ - 'classic_index', 'postrend_index', + # TODO when it's not just a stub + # 'trove_index', ]) def test_create_index(self, isolated_elastic_manager, index_name): index_setup = isolated_elastic_manager.get_index_setup(index_name) @@ -60,8 +61,8 @@ def test_create_index(self, isolated_elastic_manager, index_name): ], any_order=True) @pytest.mark.parametrize('index_name', [ - 'classic_index', 'postrend_index', + 'trove_index', ]) def test_create_index_already_exists(self, isolated_elastic_manager, index_name): mock_es_client = isolated_elastic_manager.es_client @@ -73,8 +74,8 @@ def test_create_index_already_exists(self, isolated_elastic_manager, index_name) isolated_elastic_manager.create_index(index_name) @pytest.mark.parametrize('index_name', [ - 'classic_index', 'postrend_index', + 'trove_index', ]) def test_delete_index(self, isolated_elastic_manager, index_name): mock_es_client = isolated_elastic_manager.es_client @@ -120,9 +121,9 @@ def test_send_actions_sync(self, isolated_elastic_manager): bock_mulk.assert_called_once_with(isolated_elastic_manager.es_client, input_actions) @pytest.mark.parametrize('index_names, expected_arg', [ - (['classic_index'], 'classic_index'), + (['trove_index'], 'trove_index'), (['postrend_index'], 'postrend_index'), - (['classic_index', 'postrend_index'], 'classic_index,postrend_index'), + (['trove_index', 'postrend_index'], 'trove_index,postrend_index'), ]) def test_refresh_indexes(self, isolated_elastic_manager, index_names, expected_arg): mock_es_client = isolated_elastic_manager.es_client @@ -132,8 +133,8 @@ def test_refresh_indexes(self, isolated_elastic_manager, index_names, expected_a mock_es_client.indices.refresh.assert_called_once_with(index=expected_arg) @pytest.mark.parametrize('index_name', [ - 'classic_index', 'postrend_index', + 'trove_index', ]) def test_initial_update_primary_alias(self, isolated_elastic_manager, index_name, settings): alias_name = settings.ELASTICSEARCH['PRIMARY_INDEX'] @@ -152,8 +153,8 @@ def test_initial_update_primary_alias(self, isolated_elastic_manager, index_name ) @pytest.mark.parametrize('index_name', [ - 'classic_index', 'postrend_index', + 'trove_index', ]) def test_update_primary_alias(self, isolated_elastic_manager, index_name, settings): alias_name = settings.ELASTICSEARCH['PRIMARY_INDEX'] @@ -175,8 +176,8 @@ def test_update_primary_alias(self, isolated_elastic_manager, index_name, settin ) @pytest.mark.parametrize('index_name', [ - 'classic_index', 'postrend_index', + 'trove_index', ]) def test_unnecessary_update_primary_alias(self, isolated_elastic_manager, index_name, settings): alias_name = settings.ELASTICSEARCH['PRIMARY_INDEX'] diff --git a/tests/share/search/test_indexer.py b/tests/share/search/test_indexer.py deleted file mode 100644 index 0525ae0cc..000000000 --- a/tests/share/search/test_indexer.py +++ /dev/null @@ -1,64 +0,0 @@ -import pytest - -from share import models -from share.search import SearchIndexer - -from tests import factories - - -class TestIdsToReindex: - - @pytest.mark.parametrize('model', [models.AbstractAgent, models.Tag, models.Source]) - @pytest.mark.parametrize('pks', [set(), {1}, {1, 2, 3}]) - def test_noops(self, model, pks): - result = SearchIndexer(None).pks_to_reindex(model, pks) - assert result == pks - - @pytest.mark.django_db - def test_related_works(self): - def part_of(child_work, parent_work): - factories.AbstractWorkRelationFactory( - type='share.ispartof', - subject=child_work, - related=parent_work - ) - - def retracts(retraction, work): - factories.AbstractWorkRelationFactory( - type='share.retracts', - subject=retraction, - related=work - ) - - child = factories.AbstractCreativeWorkFactory() - lost_sibling = factories.AbstractCreativeWorkFactory(is_deleted=True) - parent = factories.AbstractCreativeWorkFactory() - gparent = factories.AbstractCreativeWorkFactory() - ggparent = factories.AbstractCreativeWorkFactory() - gggparent = factories.AbstractCreativeWorkFactory() - - retraction = factories.AbstractCreativeWorkFactory() - - part_of(child, parent) - part_of(lost_sibling, parent) - part_of(parent, gparent) - part_of(gparent, ggparent) - part_of(ggparent, gggparent) - retracts(retraction, child) - - cases = [ - ({child}, {child}), - ({lost_sibling}, {lost_sibling}), - ({parent}, {parent, child}), - ({gparent}, {gparent, parent, child}), - ({ggparent}, {ggparent, gparent, parent, child}), - ({gggparent}, {gggparent, ggparent, gparent, parent}), - ({retraction}, {retraction, child}), - ({retraction, ggparent}, {retraction, ggparent, gparent, parent, child}), - ] - - for input, expected in cases: - input_ids = {w.id for w in input} - expected_ids = {w.id for w in expected} - actual_ids = SearchIndexer(None).pks_to_reindex(models.AbstractCreativeWork, input_ids) - assert expected_ids == actual_ids diff --git a/tests/share/search/test_indexing.py b/tests/share/search/test_indexing.py deleted file mode 100644 index 33022da9b..000000000 --- a/tests/share/search/test_indexing.py +++ /dev/null @@ -1,249 +0,0 @@ -import json -import re - -import pytest - -import kombu - -from django.conf import settings - -from share import models -from share import util -from share.search import fetchers -from share.search.messages import V1Message, V2Message, IndexableMessage - -from tests import factories - - -@pytest.mark.skip -class TestIndexableMessage: - - VERSION_MAP = { - # 0: V0Message, - 1: V1Message, - 2: V2Message, - } - - @pytest.fixture(params=[ - # (0, models.CreativeWork, [], {'version': 0, 'CreativeWork': []}), - # (0, models.CreativeWork, [], {'CreativeWork': []}), - # (0, models.CreativeWork, [1, 2, 3, 4], {'CreativeWork': [1, 2, 3, 4]}), - # (0, models.Agent, [], {'Agent': []}), - # (0, models.Agent, [], {'Person': []}), - # (0, models.Agent, [], {'share.AbstractAgent': []}), - # (0, models.Tag, [1], {'tag': [1]}), - (1, models.Tag, [1, 2], {'model': 'Tag', 'ids': [1, 2], 'version': 1}), - ]) - def message(self, request): - version, model, ids, payload = request.param - - self.EXPECTED_IDS = ids - self.EXPECTED_MODEL = model - self.EXPECTED_VERSION = version - - return kombu.Message( - content_type='application/json', - body=json.dumps(payload), - ) - - def test_wrap(self, message): - message = IndexableMessage.wrap(message) - - assert list(message) == self.EXPECTED_IDS - assert message.model == self.EXPECTED_MODEL - assert message.PROTOCOL_VERSION == self.EXPECTED_VERSION - assert isinstance(message, self.VERSION_MAP[self.EXPECTED_VERSION]) - - @pytest.mark.parametrize('version', [30, 10, None, 'Foo', '1', '0', {}, []]) - def test_invalid_version(self, version): - with pytest.raises(ValueError) as e: - IndexableMessage.wrap(kombu.Message( - content_type='application/json', - body=json.dumps({'version': version}), - )) - assert e.value.args == ('Invalid version "{}"'.format(version), ) - - -class TestFetchers: - - @pytest.mark.parametrize('model, fetcher', [ - (models.Agent, fetchers.AgentFetcher), - (models.Person, fetchers.AgentFetcher), - (models.AbstractAgent, fetchers.AgentFetcher), - (models.Article, fetchers.CreativeWorkFetcher), - (models.Preprint, fetchers.CreativeWorkFetcher), - (models.CreativeWork, fetchers.CreativeWorkFetcher), - (models.AbstractCreativeWork, fetchers.CreativeWorkFetcher), - ]) - def test_fetcher_for(self, model, fetcher): - assert isinstance(fetchers.fetcher_for(model), fetcher) - - def test_fetcher_not_found(self): - with pytest.raises(ValueError) as e: - fetchers.fetcher_for(models.AgentIdentifier) - assert e.value.args == ('No fetcher exists for ', ) - - @pytest.mark.django_db - @pytest.mark.parametrize('id, type, final_type, types', [ - (12, 'share.agent', 'agent', ['agent']), - (1850, 'share.Person', 'person', ['agent', 'person']), - (1850, 'share.Institution', 'institution', ['agent', 'organization', 'institution']), - (85, 'share.preprint', 'preprint', ['creative work', 'publication', 'preprint']), - (85, 'share.Software', 'software', ['creative work', 'software']), - ]) - def test_populate_types(self, id, type, final_type, types): - fetcher = fetchers.Fetcher() - - populated = fetcher.populate_types({'id': id, 'type': type}) - - assert populated['type'] == final_type - assert populated['types'] == types[::-1] - assert util.IDObfuscator.decode_id(populated['id']) == id - - @pytest.mark.django_db - def test_creativework_fetcher(self): - works = [ - factories.AbstractCreativeWorkFactory(), - factories.AbstractCreativeWorkFactory(is_deleted=True), - factories.AbstractCreativeWorkFactory(), - factories.AbstractCreativeWorkFactory(), - ] - - for work in works[:-1]: - factories.WorkIdentifierFactory(creative_work=work) - - factories.WorkIdentifierFactory.create_batch(5, creative_work=works[0]) - - source = factories.SourceFactory() - works[1].sources.add(source.user) - - # Trim trailing zeros - def iso(x): - return re.sub(r'(\.\d+?)0*\+', r'\1+', x.isoformat()) - - fetched = list(fetchers.CreativeWorkFetcher()(work.id for work in works)) - - # TODO add more variance - assert fetched == [{ - 'id': util.IDObfuscator.encode(work), - 'type': work._meta.verbose_name, - 'types': [cls._meta.verbose_name for cls in type(work).mro() if hasattr(cls, '_meta') and cls._meta.proxy], - - 'title': work.title, - 'description': work.description, - - 'date': iso(work.date_published), - 'date_created': iso(work.date_created), - 'date_modified': iso(work.date_modified), - 'date_published': iso(work.date_published), - 'date_updated': iso(work.date_updated), - - 'is_deleted': work.is_deleted or not work.identifiers.exists(), - 'justification': getattr(work, 'justification', None), - 'language': work.language, - 'registration_type': getattr(work, 'registration_type', None), - 'retracted': work.outgoing_creative_work_relations.filter(type='share.retracted').exists(), - 'withdrawn': getattr(work, 'withdrawn', None), - - 'identifiers': list(work.identifiers.values_list('uri', flat=True)), - 'sources': [user.source.long_title for user in work.sources.all()], - 'subjects': [], - 'subject_synonyms': [], - 'tags': [], - - 'lists': {}, - } for work in works] - - @pytest.mark.django_db - @pytest.mark.parametrize('bepresses, customs, expected', [ - ([], [-1], { - 'subjects': ['mergik|Magic|Cool Magic|SUPER COOL MAGIC'], - 'subject_synonyms': ['bepress|Engineering|Computer Engineering|Data Storage Systems'], - }), - ([-1], [], { - 'subjects': ['bepress|Engineering|Computer Engineering|Data Storage Systems'], - 'subject_synonyms': [], - }), - ([-1], [-1], { - 'subjects': ['bepress|Engineering|Computer Engineering|Data Storage Systems', 'mergik|Magic|Cool Magic|SUPER COOL MAGIC'], - 'subject_synonyms': ['bepress|Engineering|Computer Engineering|Data Storage Systems'], - }), - ([0, 1], [], { - 'subjects': ['bepress|Engineering', 'bepress|Engineering|Computer Engineering'], - 'subject_synonyms': [], - }), - ([], [0, 1], { - 'subjects': ['mergik|Magic', 'mergik|Magic|Cool Magic'], - 'subject_synonyms': ['bepress|Engineering', 'bepress|Engineering|Computer Engineering'], - }), - ]) - def test_subject_indexing(self, bepresses, customs, expected): - custom_tax = factories.SubjectTaxonomyFactory(source__long_title='mergik') - system_tax = models.SubjectTaxonomy.objects.get(source__user__username=settings.APPLICATION_USERNAME) - - custom = ['Magic', 'Cool Magic', 'SUPER COOL MAGIC'] - bepress = ['Engineering', 'Computer Engineering', 'Data Storage Systems'] - - for i, name in enumerate(tuple(bepress)): - bepress[i] = factories.SubjectFactory( - name=name, - taxonomy=system_tax, - parent=bepress[i - 1] if i > 0 else None, - ) - - for i, name in enumerate(tuple(custom)): - custom[i] = factories.SubjectFactory( - name=name, - taxonomy=custom_tax, - central_synonym=bepress[i], - parent=custom[i - 1] if i > 0 else None, - ) - - work = factories.AbstractCreativeWorkFactory() - - for i in bepresses: - factories.ThroughSubjectsFactory(subject=bepress[i], creative_work=work) - for i in customs: - factories.ThroughSubjectsFactory(subject=custom[i], creative_work=work) - - fetched = next(fetchers.CreativeWorkFetcher()([work.id])) - assert {k: v for k, v in fetched.items() if k.startswith('subject')} == expected - - @pytest.mark.django_db - def test_lineage_indexing(self): - child = factories.AbstractCreativeWorkFactory() - - fetched = next(fetchers.CreativeWorkFetcher()([child.id])) - assert fetched['lists'].get('lineage') is None - - actual_lineage = [child] - for _ in range(5): - new_parent = factories.AbstractCreativeWorkFactory() - factories.AbstractWorkRelationFactory( - type='share.ispartof', - subject=actual_lineage[0], - related=new_parent - ) - actual_lineage.insert(0, new_parent) - - for i, work in enumerate(actual_lineage): - expected_lineage = actual_lineage[:i][-3:] - fetched = next(fetchers.CreativeWorkFetcher()([work.id])) - fetched_lineage = fetched['lists'].get('lineage', []) - - assert len(fetched_lineage) == len(expected_lineage) - for indexed, ancestor in zip(fetched_lineage, expected_lineage): - assert indexed['id'] == util.IDObfuscator.encode(ancestor) - assert indexed['title'] == ancestor.title - assert set(indexed['identifiers']) == set(ancestor.identifiers.values_list('uri')) - - @pytest.mark.django_db - def test_agent_fetcher(self): - agents = [ - factories.AbstractAgentFactory(), - factories.AbstractAgentFactory(), - factories.AbstractAgentFactory(), - factories.AbstractAgentFactory(), - ] - - list(fetchers.AgentFetcher()(agent.id for agent in agents)) diff --git a/tests/share/test_celery.py b/tests/share/test_celery.py index cd55b95f1..34e480ada 100644 --- a/tests/share/test_celery.py +++ b/tests/share/test_celery.py @@ -7,8 +7,7 @@ from django.utils import timezone from django.core import serializers -from share import models -from share.celery import TaskResultCleaner +from share.celery import TaskResultCleaner, CeleryTaskResult from tests import factories @@ -29,18 +28,18 @@ def task_result_data(self): def test_delete_false(self): trc = TaskResultCleaner(datetime.timedelta(weeks=520), delete=False) - assert trc.delete_queryset(models.CeleryTaskResult.objects.all()) == 0 - assert models.CeleryTaskResult.objects.count() != 0 + assert trc.delete_queryset(CeleryTaskResult.objects.all()) == 0 + assert CeleryTaskResult.objects.count() != 0 def test_delete_queryset(self): trc = TaskResultCleaner(datetime.timedelta(weeks=520)) - assert trc.delete_queryset(models.CeleryTaskResult.objects.all()) == 100 - assert models.CeleryTaskResult.objects.count() == 0 + assert trc.delete_queryset(CeleryTaskResult.objects.all()) == 100 + assert CeleryTaskResult.objects.count() == 0 def test_no_bucket(self): trc = TaskResultCleaner(datetime.timedelta(weeks=520), bucket=None) trc.put_s3 = mock.Mock() - trc.archive_queryset('name', models.CeleryTaskResult.objects.all()) + trc.archive_queryset('name', CeleryTaskResult.objects.all()) assert trc.put_s3.called is False # @pytest.mark.parametrize('access_key, secret_key, folder_name, bucket_name', [ @@ -62,7 +61,7 @@ def test_archive(self, mock_boto): trc = TaskResultCleaner(0, bucket=mock.Mock()) factories.CeleryTaskResultFactory.create_batch(100, status='SUCCESS') trc.archive() - assert models.CeleryTaskResult.objects.count() <= 100 # There's an autouse fixture that makes 100 + assert CeleryTaskResult.objects.count() <= 100 # There's an autouse fixture that makes 100 for call in mock_boto.resource('s3').Object.call_args_list: assert call[0][0] is trc.bucket assert isinstance(call[0][1], str) @@ -71,13 +70,13 @@ def test_archive_chunksize(self, mock_boto): trc = TaskResultCleaner(0, bucket=mock.Mock(), chunk_size=1) factories.CeleryTaskResultFactory.create_batch(100, status='SUCCESS') trc.archive() - assert models.CeleryTaskResult.objects.count() <= 100 # There's an autouse fixture that makes 100 + assert CeleryTaskResult.objects.count() <= 100 # There's an autouse fixture that makes 100 assert len(mock_boto.resource('s3').Object.call_args_list) >= 100 def test_serialization(self): trc = TaskResultCleaner(0) - compressed = trc.compress_and_serialize(models.CeleryTaskResult.objects.all()) + compressed = trc.compress_and_serialize(CeleryTaskResult.objects.all()) reloaded = list(serializers.deserialize('json', bz2.decompress(compressed.getvalue()))) assert len(reloaded) == 100 for task in reloaded: - assert isinstance(task.object, models.CeleryTaskResult) + assert isinstance(task.object, CeleryTaskResult) diff --git a/tests/share/test_oaipmh.py b/tests/share/test_oaipmh.py deleted file mode 100644 index e716d0c1c..000000000 --- a/tests/share/test_oaipmh.py +++ /dev/null @@ -1,188 +0,0 @@ -import math -import pendulum -import pytest -import random -from lxml import etree -from unittest.mock import patch - -from django.test.client import Client - -from share import models -from share.oaipmh.util import format_datetime -from share.oaipmh.views import OAIPMHView -from share.util import IDObfuscator - -from tests.share.models import factories - - -NAMESPACES = { - 'dc': 'http://purl.org/dc/elements/1.1/', - 'ns0': 'http://www.openarchives.org/OAI/2.0/', - 'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/', -} - - -@pytest.fixture(autouse=True) -def ensure_legacy_oaipmh(): - with patch.object(OAIPMHView, '_should_use_legacy_repository', return_value=True): - yield - - -@pytest.fixture -def oai_works(): - return [factories.PreprintFactory() for i in range(17)] - - -def oai_request(data, post, errors=False): - client = Client() - if post: - response = client.post('/oai-pmh/', data) - else: - response = client.get('/oai-pmh/', data) - assert response.status_code == 200 - parsed = etree.fromstring(response.content, parser=etree.XMLParser(recover=True)) - actual_errors = parsed.xpath('//ns0:error', namespaces=NAMESPACES) - if errors: - assert actual_errors - return actual_errors - assert not actual_errors - return parsed - - -@pytest.mark.django_db -@pytest.mark.parametrize('post', [True, False]) -class TestOAIVerbs: - - def test_identify(self, post): - parsed = oai_request({'verb': 'Identify'}, post) - assert parsed.xpath('//ns0:Identify/ns0:repositoryName', namespaces=NAMESPACES)[0].text == 'SHARE' - - def test_list_sets(self, post): - parsed = oai_request({'verb': 'ListSets'}, post) - assert len(parsed.xpath('//ns0:ListSets/ns0:set', namespaces=NAMESPACES)) > 100 - - def test_list_formats(self, post): - parsed = oai_request({'verb': 'ListMetadataFormats'}, post) - prefixes = parsed.xpath('//ns0:ListMetadataFormats/ns0:metadataFormat/ns0:metadataPrefix', namespaces=NAMESPACES) - assert len(prefixes) == 1 - assert prefixes[0].text == 'oai_dc' - - def test_list_identifiers(self, post, all_about_anteaters): - parsed = oai_request({'verb': 'ListIdentifiers', 'metadataPrefix': 'oai_dc'}, post) - identifiers = parsed.xpath('//ns0:ListIdentifiers/ns0:header/ns0:identifier', namespaces=NAMESPACES) - assert len(identifiers) == 1 - assert identifiers[0].text == 'oai:share.osf.io:{}'.format(IDObfuscator.encode(all_about_anteaters)) - - def test_list_records(self, post, all_about_anteaters, django_assert_num_queries): - with django_assert_num_queries(3): - parsed = oai_request({'verb': 'ListRecords', 'metadataPrefix': 'oai_dc'}, post) - records = parsed.xpath('//ns0:ListRecords/ns0:record', namespaces=NAMESPACES) - assert len(records) == 1 - assert all_about_anteaters.title == records[0].xpath('ns0:metadata/oai_dc:dc/dc:title', namespaces=NAMESPACES)[0].text - ant_id = 'oai:share.osf.io:{}'.format(IDObfuscator.encode(all_about_anteaters)) - assert ant_id == records[0].xpath('ns0:header/ns0:identifier', namespaces=NAMESPACES)[0].text - - def test_get_record(self, post, all_about_anteaters): - ant_id = 'oai:share.osf.io:{}'.format(IDObfuscator.encode(all_about_anteaters)) - parsed = oai_request({'verb': 'GetRecord', 'metadataPrefix': 'oai_dc', 'identifier': ant_id}, post) - records = parsed.xpath('//ns0:GetRecord/ns0:record', namespaces=NAMESPACES) - assert len(records) == 1 - assert all_about_anteaters.title == records[0].xpath('ns0:metadata/oai_dc:dc/dc:title', namespaces=NAMESPACES)[0].text - assert ant_id == records[0].xpath('ns0:header/ns0:identifier', namespaces=NAMESPACES)[0].text - - @pytest.mark.parametrize('verb, params, errors', [ - ('GetRecord', {}, ['badArgument']), - ('GetRecord', {'something': '{id}'}, ['badArgument']), - ('GetRecord', {'identifier': '{id}'}, ['badArgument']), - ('GetRecord', {'identifier': 'bad', 'metadataPrefix': 'oai_dc'}, ['idDoesNotExist']), - ('GetRecord', {'identifier': '{id}', 'metadataPrefix': 'bad'}, ['cannotDisseminateFormat']), - ('Identify', {'metadataPrefix': 'oai_dc'}, ['badArgument']), - ('ListIdentifiers', {}, ['badArgument']), - ('ListIdentifiers', {'something': '{id}'}, ['badArgument']), - ('ListIdentifiers', {'metadataPrefix': 'bad'}, ['cannotDisseminateFormat']), - ('ListIdentifiers', {'set': 'not_a_set', 'metadataPrefix': 'oai_dc'}, ['noRecordsMatch']), - ('ListIdentifiers', {'resumptionToken': 'token', 'metadataPrefix': 'oai_dc'}, ['badArgument']), - ('ListIdentifiers', {'resumptionToken': 'token'}, ['badResumptionToken']), - ('ListRecords', {}, ['badArgument']), - ('ListRecords', {'something': '{id}'}, ['badArgument']), - ('ListRecords', {'metadataPrefix': 'bad'}, ['cannotDisseminateFormat']), - ('ListRecords', {'set': 'not_a_set', 'metadataPrefix': 'oai_dc'}, ['noRecordsMatch']), - ('ListRecords', {'resumptionToken': 'token', 'metadataPrefix': 'oai_dc'}, ['badArgument']), - ('ListRecords', {'resumptionToken': 'token'}, ['badResumptionToken']), - ('ListMetadataFormats', {'metadataPrefix': 'oai_dc'}, ['badArgument']), - ('ListMetadataFormats', {'identifier': 'bad_identifier'}, ['idDoesNotExist']), - ('ListSets', {'something': 'oai_dc'}, ['badArgument']), - ('ListSets', {'resumptionToken': 'bad_token'}, ['badResumptionToken']), - ]) - def test_oai_errors(self, verb, params, errors, post, all_about_anteaters): - ant_id = IDObfuscator.encode(all_about_anteaters) - data = {'verb': verb, **{k: v.format(id=ant_id) for k, v in params.items()}} - actual_errors = oai_request(data, post, errors=True) - for error_code in errors: - assert any(e.attrib.get('code') == error_code for e in actual_errors) - - -@pytest.mark.django_db -@pytest.mark.parametrize('post', [True, False]) -@pytest.mark.parametrize('verb', ['ListRecords', 'ListIdentifiers']) -@pytest.mark.parametrize('page_size', [5, 10, 100]) -class TestOAILists: - - def test_full_list(self, oai_works, post, verb, page_size, monkeypatch): - monkeypatch.setattr('share.oaipmh.legacy.repository.OAIRepository.PAGE_SIZE', page_size) - self._assert_full_list(verb, {}, post, len(oai_works), page_size) - - @pytest.mark.parametrize('from_date, to_date, expected_count', [ - (pendulum.now().subtract(days=1), None, 17), - (None, pendulum.now().subtract(days=1), 0), - (pendulum.now().add(days=1), None, 0), - (None, pendulum.now().add(days=1), 17), - (pendulum.now().subtract(days=1), pendulum.now().add(days=1), 17), - ]) - def test_filter_date(self, oai_works, post, verb, page_size, monkeypatch, from_date, to_date, expected_count): - monkeypatch.setattr('share.oaipmh.legacy.repository.OAIRepository.PAGE_SIZE', page_size) - params = {} - if from_date: - params['from'] = format_datetime(from_date) - if to_date: - params['until'] = format_datetime(to_date) - self._assert_full_list(verb, params, post, expected_count, page_size) - - @pytest.mark.parametrize('expected_count', range(0, 17, 6)) - def test_filter_set(self, oai_works, post, verb, page_size, monkeypatch, expected_count): - monkeypatch.setattr('share.oaipmh.legacy.repository.OAIRepository.PAGE_SIZE', page_size) - source = models.Source.objects.select_related('user').first() - for work in random.sample(oai_works, expected_count): - work.sources.add(source.user) - - self._assert_full_list(verb, {'set': source.name}, post, expected_count, page_size) - - def _assert_full_list(self, verb, params, post, expected_count, page_size): - if not expected_count: - errors = oai_request({'verb': verb, 'metadataPrefix': 'oai_dc', **params}, post, errors=True) - assert len(errors) == 1 - assert errors[0].attrib.get('code') == 'noRecordsMatch' - return - - pages = 0 - count = 0 - token = None - while True: - if token: - parsed = oai_request({'verb': verb, 'resumptionToken': token}, post) - else: - parsed = oai_request({'verb': verb, 'metadataPrefix': 'oai_dc', **params}, post) - page = parsed.xpath('//ns0:header/ns0:identifier', namespaces=NAMESPACES) - pages += 1 - count += len(page) - token = parsed.xpath('//ns0:resumptionToken', namespaces=NAMESPACES) - assert len(token) == 1 - token = token[0].text - if token: - assert len(page) == page_size - else: - assert len(page) <= page_size - break - - assert count == expected_count - assert pages == math.ceil(expected_count / page_size) diff --git a/tests/share/util/test_model_generator.py b/tests/share/util/test_model_generator.py deleted file mode 100644 index 5a1f65ef8..000000000 --- a/tests/share/util/test_model_generator.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from django.apps import apps - - -@pytest.mark.parametrize('model_name, field_name', [ - ('Funder', 'awards'), - ('Funder', 'award_versions'), - ('Funder', 'agent'), - ('Funder', 'creative_work'), - ('Contributor', 'contributed_through'), - ('Creator', 'order_cited'), - ('Person', 'given_name'), - ('Person', 'family_name'), - ('Person', 'additional_name'), - ('Person', 'suffix'), -]) -def test_field_exists(model_name, field_name): - model = apps.get_model('share', model_name) - field = model._meta.get_field(field_name) - assert field From 9daa650db75e31c451e9aeca0d6857aac23fe5c9 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Thu, 25 Feb 2021 14:59:37 -0500 Subject: [PATCH 05/14] fix and unskip atom feed tests --- api/views/feeds.py | 52 +++++--------------- tests/api/test_feeds.py | 103 +++++++++++----------------------------- tests/conftest.py | 23 +++++++-- 3 files changed, 61 insertions(+), 117 deletions(-) diff --git a/api/views/feeds.py b/api/views/feeds.py index 70711141e..b64814695 100644 --- a/api/views/feeds.py +++ b/api/views/feeds.py @@ -1,10 +1,9 @@ from xml.sax.saxutils import unescape -import datetime from furl import furl import json import logging -import re +import pendulum import requests from django.contrib.syndication.views import Feed @@ -29,24 +28,7 @@ def prepare_string(s): return s -def parse_date(s): - if not s: - return None - - # strptime can't parse +00:00 - s = re.sub(r'\+(\d\d):(\d\d)', r'+\1\2', s) - - for date_fmt in ('%Y-%m-%dT%H:%M:%S%z', '%Y-%m-%dT%H:%M:%S.%f%z'): - try: - return datetime.datetime.strptime(s, date_fmt) - except ValueError: - pass - logger.error('Could not parse date "%s"', s) - return None - - class MetadataRecordsRSS(Feed): - _share_index_key = 'BACKCOMPAT_INDEX' # TODO remove when we drop legacy feeds link = '{}api/v2/feeds/rss/'.format(settings.SHARE_WEB_URL) description = 'Updates to the SHARE open dataset' author_name = 'SHARE' @@ -80,8 +62,7 @@ def items(self, obj): headers = {'Content-Type': 'application/json'} search_url = '{}{}/creativeworks/_search'.format( settings.ELASTICSEARCH['URL'], - # TODO as soon as we can drop the legacy feeds, use settings.ELASTICSEARCH['PRIMARY_INDEX'] - settings.ELASTICSEARCH[self._share_index_key], + settings.ELASTICSEARCH['PRIMARY_INDEX'], ) elastic_response = requests.post(search_url, data=json.dumps(obj), headers=headers) json_response = elastic_response.json() @@ -122,10 +103,10 @@ def item_author_name(self, item): return prepare_string('{}{}'.format(author_name, ' et al.' if len(authors) > 1 else '')) def item_pubdate(self, item): - return parse_date(item.get('date_published')) + return pendulum.parse(item.get('date_published')) def item_updateddate(self, item): - return parse_date(item.get(self._order)) + return pendulum.parse(item.get(self._order)) def item_categories(self, item): categories = item.get('subjects', []) @@ -134,35 +115,26 @@ def item_categories(self, item): class MetadataRecordsAtom(MetadataRecordsRSS): - _share_index_key = 'BACKCOMPAT_INDEX' # TODO remove when we drop legacy feeds feed_type = Atom1Feed subtitle = MetadataRecordsRSS.description link = '{}api/v2/feeds/atom/'.format(settings.SHARE_WEB_URL) -# TODO remove when we drop legacy feeds class LegacyCreativeWorksRSS(MetadataRecordsRSS): - _share_index_key = 'LEGACY_INDEX' # TODO remove when we drop legacy feeds link = '{}api/v2/rss/'.format(settings.SHARE_WEB_URL) def __call__(self, request, *args, **kwargs): - if not settings.SHARE_LEGACY_PIPELINE: - correct_url = furl(MetadataRecordsRSS.link).set(query_params=request.GET) - return HttpResponseGone( - f'This feed has been removed -- please update to use {correct_url}' - ) - return super().__call__(request, *args, **kwargs) + correct_url = furl(MetadataRecordsRSS.link).set(query_params=request.GET) + return HttpResponseGone( + f'This feed has been removed -- please update to use {correct_url}' + ) -# TODO remove when we drop legacy feeds class LegacyCreativeWorksAtom(MetadataRecordsAtom): - _share_index_key = 'LEGACY_INDEX' # TODO remove when we drop legacy feeds link = '{}api/v2/atom/'.format(settings.SHARE_WEB_URL) def __call__(self, request, *args, **kwargs): - if not settings.SHARE_LEGACY_PIPELINE: - correct_url = furl(MetadataRecordsAtom.link).set(query_params=request.GET) - return HttpResponseGone( - f'This feed has been removed -- please update to use {correct_url}' - ) - return super().__call__(request, *args, **kwargs) + correct_url = furl(MetadataRecordsAtom.link).set(query_params=request.GET) + return HttpResponseGone( + f'This feed has been removed -- please update to use {correct_url}' + ) diff --git a/tests/api/test_feeds.py b/tests/api/test_feeds.py index 33de18fc6..1b619a862 100644 --- a/tests/api/test_feeds.py +++ b/tests/api/test_feeds.py @@ -1,13 +1,12 @@ import pytest -from datetime import timezone import faker from lxml import etree -from share.util import IDObfuscator +from share.util.graph import MutableGraph -from tests import factories +from tests.share.normalize import factories as f fake = faker.Faker() @@ -17,39 +16,25 @@ # TODO add tests for RSS -@pytest.mark.skip @pytest.mark.django_db class TestFeed: @pytest.fixture - def fake_items(self, settings, index_creativeworks): - ids = [] - for i in range(11): - person_0 = factories.AbstractAgentFactory(type='share.person') - person_1 = factories.AbstractAgentFactory(type='share.person') - - work = factories.AbstractCreativeWorkFactory( - date_published=None if i % 3 == 0 else fake.date_time_this_decade(tzinfo=timezone.utc), + def fake_items(self, settings, index_records): + records = [ + f.CreativeWork( + identifiers=[f.WorkIdentifier()], + agent_relations=[ + f.Creator(), + f.Creator(), + ], ) - if i % 3 == 1: - factories.CreatorWorkRelationFactory(creative_work=work, agent=person_0, order_cited=0) - factories.CreatorWorkRelationFactory(creative_work=work, agent=person_1, order_cited=1) - if i % 3 == 2: - factories.CreatorWorkRelationFactory(creative_work=work, agent=person_0) + for i in range(11) + ] + return index_records(records) - # Works without identifiers won't be surfaced in search - factories.WorkIdentifierFactory(creative_work=work) - - ids.append(work.id) - - index_creativeworks(ids) - - @pytest.mark.parametrize('feed_url', [ - '/api/v2/atom/', - '/api/v2/feeds/atom/', - ]) - def test_get_feed(self, client, fake_items, feed_url, settings): - resp = client.get(feed_url) + def test_atom(self, client, fake_items): + resp = client.get('/api/v2/feeds/atom') assert resp.status_code == 200 feed = etree.fromstring(resp.content) @@ -58,50 +43,20 @@ def test_get_feed(self, client, fake_items, feed_url, settings): assert feed.nsmap == {None: 'http://www.w3.org/2005/Atom'} assert feed.getchildren()[0].text.startswith('SHARE: Atom feed for query:') - @pytest.mark.parametrize('order', [ - 'date_updated', - 'date_modified', - ]) - @pytest.mark.parametrize('feed_url', [ - '/api/v2/atom/', - '/api/v2/feeds/atom/', - ]) - def test_order(self, client, order, fake_items, feed_url): - resp = client.get(feed_url, {'order': order}) - assert resp.status_code == 200 - - feed = etree.fromstring(resp.content) - works = None # AbstractCreativeWork.objects.order_by('-' + order).exclude(**{order: None}) - - assert len(feed.xpath('//atom:entry', namespaces=NAMESPACES)) == 11 - - for creative_work, entry in zip(works, feed.xpath('//atom:entry', namespaces={'atom': 'http://www.w3.org/2005/Atom'})): - try: - contributors = None # list(AbstractAgentWorkRelation.objects.filter(creative_work_id=creative_work.id)) - first_contributor = None # AbstractAgentWorkRelation.objects.get(creative_work_id=creative_work.id, order_cited=0) - except Exception: - contributors = None - - assert entry.find('atom:title', namespaces=NAMESPACES).text == creative_work.title - assert entry.find('atom:summary', namespaces=NAMESPACES).text == creative_work.description - assert entry.find('atom:link', namespaces=NAMESPACES).attrib['href'].endswith(IDObfuscator.encode(creative_work)) - - if not contributors: - assert entry.find('atom:author', namespaces=NAMESPACES)[0].text == 'No authors provided.' - elif len(contributors) > 1: - assert entry.find('atom:author', namespaces=NAMESPACES)[0].text == '{} et al.'.format(first_contributor.agent.name) - else: - assert entry.find('atom:author', namespaces=NAMESPACES)[0].text == first_contributor.agent.name - - if getattr(creative_work, order): - assert entry.find('atom:updated', namespaces=NAMESPACES).text == getattr(creative_work, order).replace(microsecond=0).isoformat() - else: - assert entry.find('atom:updated', namespaces=NAMESPACES).text is None - - if creative_work.date_published: - assert entry.find('atom:published', namespaces=NAMESPACES).text == creative_work.date_published.isoformat() - else: - assert entry.find('atom:published', namespaces=NAMESPACES) is None + expected_graphs = [ + MutableGraph.from_jsonld(normd.data) + for normd in fake_items + ] + expected_titles = set( + gr.get_central_node(guess=True)['title'] + for gr in expected_graphs + ) + actual_titles = set( + element.text + for element in feed.xpath('//atom:entry/atom:title', namespaces=NAMESPACES) + ) + assert len(actual_titles) == 11 + assert actual_titles == expected_titles @pytest.mark.parametrize('feed_url, expected_status', [ ('/api/v2/atom/', 410), diff --git a/tests/conftest.py b/tests/conftest.py index 30e24ea86..6d732db9c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,7 @@ from share.models import NormalizedData, RawDatum from share.models import ShareUser from share.models import SourceUniqueIdentifier +from share.models import FormattedMetadataRecord from share.search import MessageType, SearchIndexer from share.search.elastic_manager import ElasticManager @@ -176,9 +177,25 @@ def elastic_test_manager(settings, elastic_test_index_name): @pytest.fixture def index_records(elastic_test_manager): - def _index_records(formatted_records): - suid_ids = [fmr.suid_id for fmr in formatted_records] + def _index_records(normalized_graphs): + normalized_datums = [ + factories.NormalizedDataFactory( + data=GraphBuilder()(ng).to_jsonld(), + raw=factories.RawDatumFactory( + datum='', + ), + ) + for ng in normalized_graphs + ] + suids = [nd.raw.suid for nd in normalized_datums] + for normd, suid in zip(normalized_datums, suids): + FormattedMetadataRecord.objects.save_formatted_records( + suid=suid, + record_formats=['sharev2_elastic'], + normalized_datum=normd, + ) indexer = SearchIndexer(elastic_manager=elastic_test_manager) - indexer.handle_messages_sync(MessageType.INDEX_SUID, suid_ids) + indexer.handle_messages_sync(MessageType.INDEX_SUID, [suid.id for suid in suids]) + return normalized_datums return _index_records From 4ebb7d3bc478cbd015e1007b1221bf9f593dfb92 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Thu, 25 Feb 2021 15:23:44 -0500 Subject: [PATCH 06/14] delete unused code --- api/renderers.py | 59 ------------------------------------------------ 1 file changed, 59 deletions(-) delete mode 100644 api/renderers.py diff --git a/api/renderers.py b/api/renderers.py deleted file mode 100644 index 74cf2954e..000000000 --- a/api/renderers.py +++ /dev/null @@ -1,59 +0,0 @@ -import ujson -import six - -from rest_framework.renderers import JSONRenderer -from rest_framework.settings import api_settings -from rest_framework.utils import encoders - - -class JSONLDRenderer(JSONRenderer): - """ - Renderer which serializes to JSON. - """ - media_type = 'application/vnd.api+json' - format = 'json' - encoder_class = encoders.JSONEncoder - ensure_ascii = not api_settings.UNICODE_JSON - compact = api_settings.COMPACT_JSON - - # We don't set a charset because JSON is a binary encoding, - # that can be encoded as utf-8, utf-16 or utf-32. - # See: http://www.ietf.org/rfc/rfc4627.txt - # Also: http://lucumr.pocoo.org/2013/7/19/application-mimetypes-and-encodings/ - charset = None - - def render(self, data, accepted_media_type=None, renderer_context=None): - """ - Render `data` into JSON, returning a bytestring. - """ - if data is None: - return bytes() - - renderer_context = renderer_context or {} - indent = self.get_indent(accepted_media_type, renderer_context) or 4 - - # if indent is None: - # separators = SHORT_SEPARATORS if self.compact else LONG_SEPARATORS - # else: - # separators = INDENT_SEPARATORS - - ret = ujson.dumps( # UJSON is faster - data, - # , cls=self.encoder_class, - escape_forward_slashes=False, - indent=indent, ensure_ascii=self.ensure_ascii, - # separators=separators - ) - - # On python 2.x json.dumps() returns bytestrings if ensure_ascii=True, - # but if ensure_ascii=False, the return type is underspecified, - # and may (or may not) be unicode. - # On python 3.x json.dumps() returns unicode strings. - if isinstance(ret, six.text_type): - # We always fully escape \u2028 and \u2029 to ensure we output JSON - # that is a strict javascript subset. If bytes were returned - # by json.dumps() then we don't have these characters in any case. - # See: http://timelessrepo.com/json-isnt-a-javascript-subset - ret = ret.replace('\u2028', '\\u2028').replace('\u2029', '\\u2029') - return bytes(ret.encode('utf-8')) - return ret From ce7f9ca2a41259c5c87a701d2624869af05f615e Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Mon, 17 May 2021 12:34:07 -0400 Subject: [PATCH 07/14] fix deduplicating subjects --- share/regulate/steps/deduplicate.py | 1 + tests/share/regulate/steps/test_deduplicate.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/share/regulate/steps/deduplicate.py b/share/regulate/steps/deduplicate.py index 65b4f85a6..80ccd7e65 100644 --- a/share/regulate/steps/deduplicate.py +++ b/share/regulate/steps/deduplicate.py @@ -27,6 +27,7 @@ class Deduplicate(GraphStep): 'throughtags': {'tag', 'creative_work'}, # 'award': {}, 'throughawards': {'funder', 'award'}, + 'throughsubjects': {'subject', 'creative_work'}, } def regulate_graph(self, graph): diff --git a/tests/share/regulate/steps/test_deduplicate.py b/tests/share/regulate/steps/test_deduplicate.py index ba22aa711..f5f002de3 100644 --- a/tests/share/regulate/steps/test_deduplicate.py +++ b/tests/share/regulate/steps/test_deduplicate.py @@ -5,6 +5,8 @@ from tests.share.normalize.factories import ( CreativeWork, Preprint, + Registration, + Subject, WorkIdentifier, ) From 7e20bebf8e96c2c142a4f2568d827924d37e8937 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Mon, 17 May 2021 13:19:04 -0400 Subject: [PATCH 08/14] remove transitional hack --- tests/share/test_oaipmh_trove.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/share/test_oaipmh_trove.py b/tests/share/test_oaipmh_trove.py index cea8f6162..15dfc7478 100644 --- a/tests/share/test_oaipmh_trove.py +++ b/tests/share/test_oaipmh_trove.py @@ -3,13 +3,11 @@ import pytest import random from lxml import etree -from unittest.mock import patch from django.test.client import Client from share.models import SourceUniqueIdentifier, FormattedMetadataRecord from share.oaipmh.util import format_datetime -from share.oaipmh.views import OAIPMHView from share.util import IDObfuscator from tests.factories import FormattedMetadataRecordFactory, SourceFactory @@ -22,12 +20,6 @@ } -@pytest.fixture(autouse=True) -def ensure_non_legacy_oaipmh(): - with patch.object(OAIPMHView, '_should_use_legacy_repository', return_value=False): - yield - - def oai_request(data, pls_post, expect_errors=False): client = Client() if pls_post: From d9aa7ac9aa0b413348d6ec1b7fe33626d7254c7f Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Tue, 18 May 2021 11:22:22 -0400 Subject: [PATCH 09/14] remove unused sharectl commands --- share/bin/__init__.py | 1 - share/bin/services.py | 44 ------------------------------------------- 2 files changed, 45 deletions(-) delete mode 100644 share/bin/services.py diff --git a/share/bin/__init__.py b/share/bin/__init__.py index 7a24ef017..ba1d935b3 100644 --- a/share/bin/__init__.py +++ b/share/bin/__init__.py @@ -11,7 +11,6 @@ 'info', 'ingest', 'search', - 'services', ) diff --git a/share/bin/services.py b/share/bin/services.py deleted file mode 100644 index acc0bde25..000000000 --- a/share/bin/services.py +++ /dev/null @@ -1,44 +0,0 @@ -import logging -# import threading - -# from celery.signals import worker_ready -# from celery.signals import worker_shutdown - -from share.bin.util import command -# from share.search.daemon import SearchIndexerDaemon - - -@command('Launch the SHARE API server', parsed=False) -def server(args, argv): - from django.core.management import execute_from_command_line - execute_from_command_line(['', 'runserver'] + argv[1:]) - - -@command('Launch a Celery worker') -def worker(args, argv): - """ - Usage: {0} worker [options] - - Options: - -B, --beat Also run the celery beat periodic task scheduler. - -l, --loglevel=LOGLEVEL Logging level. [Default: INFO] - - For local development only. Deployments should use the celery binary. - """ - # -I, --indexer Also run the search indexer daemon. - from project.celery import app - - # if args['--indexer']: - # sid = SearchIndexerDaemon(app) - - # @worker_ready.connect - # def start_sid(*args, **kwargs): - # threading.Thread(target=sid.run, daemon=True).start() - - # @worker_shutdown.connect - # def stop_sid(*args, **kwargs): - # sid.stop() - - worker = app.Worker(loglevel=getattr(logging, args['--loglevel'].upper()), beat=args['--beat']) - worker.start() - return worker.exitcode From 8d42644dece03e3e9acaca642114d1967f991b01 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Tue, 18 May 2021 11:22:34 -0400 Subject: [PATCH 10/14] add minimal sharectl smoke test --- .travis.yml | 1 + tests/share/bin/test_sharectl.py | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 tests/share/bin/test_sharectl.py diff --git a/.travis.yml b/.travis.yml index afbb616d7..8ffa3a5f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,7 @@ install: - travis_retry pip install flake8==3.8.3 --force-reinstall --upgrade - travis_retry pip install -r dev-requirements.txt - travis_retry pip install . + - travis_retry python setup.py develop before_script: - wget -q --waitretry=1 --retry-connrefused -T 10 -O - http://127.0.0.1:9200 diff --git a/tests/share/bin/test_sharectl.py b/tests/share/bin/test_sharectl.py new file mode 100644 index 000000000..71f6a42f3 --- /dev/null +++ b/tests/share/bin/test_sharectl.py @@ -0,0 +1,6 @@ +import subprocess + + +def test_sharectl_at_least_runs(): + completed_process = subprocess.run(['sharectl', '-v']) + assert completed_process.returncode == 0 From 1790cb45b40b297c513cc6f8ce32163d457dae09 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Tue, 18 May 2021 15:08:57 -0400 Subject: [PATCH 11/14] trim some dependencies --- project/settings.py | 5 ----- requirements.txt | 13 +------------ 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/project/settings.py b/project/settings.py index e727342e1..6bc2520c0 100644 --- a/project/settings.py +++ b/project/settings.py @@ -75,7 +75,6 @@ def split(string, delim): 'rest_framework', 'corsheaders', 'revproxy', - 'graphene_django', 'prettyjson', 'allauth', @@ -158,10 +157,6 @@ def split(string, delim): ), } -GRAPHENE = { - 'SCHEMA': 'share.graphql.schema.schema' -} - MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', diff --git a/requirements.txt b/requirements.txt index a4f1d6b0c..f0f87523f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,15 +6,14 @@ celery==4.1.0 # BSD 3 Clause colorlog==2.7.0 # MIT dateparser==0.4.0 # BSD django-allauth==0.31.0 # MIT +django-celery-beat==1.1.1 # BSD 3 Clause django-cors-headers==2.0.2 # MIT django-extensions==1.7.8 # MIT django-filter==1.0.2 # BSD -django-include==0.2.1 # MIT django-model-utils==2.6.1 # BSD django-oauth-toolkit==0.12.0 # BSD django-prettyjson==0.3.0 # BSD 3 Clause django-revproxy==0.10.0 # MPL 2.0 -django-typed-models==0.6.0 # BSD 3 Clause django==1.11.16 # BSD 3 Clause django[bcrypt]==1.11.16 # BSD 3 Clause djangorestframework==3.6.2 # BSD @@ -22,9 +21,6 @@ docopt==0.6.2 # MIT elasticsearch==5.4.0 # Apache 2.0 furl==0.4.95 # None gevent==1.2.2 # MIT -graphene==1.4 # MIT -graphql-core==1.1 # MIT -graphql-relay==0.5.0 # MIT jsonschema==2.5.1 # MIT lxml==3.6.0 # BSD kombu==4.1.0 # BSD 3 Clause @@ -52,10 +48,3 @@ xmltodict==0.10.2 # MIT # Allows custom-rendered IDs, hiding null values, and including data in error responses git+git://github.com/cos-forks/django-rest-framework-json-api.git@v2.2.0+cos1 -# TODO: When graphene-django doesn't break on inherited fields with choices (https://github.com/graphql-python/graphene-django/pull/156) -# graphene-django==1.3 # MIT -git+git://github.com/aaxelb/graphene-django.git@be20450a663073add7c7ace437cccaf2832cf4f3 - -# django-celery-beat==1.0.1 # BSD 3 Clause -# Contains a fix for handling disabled tasks that still has not been release -git+git://github.com/celery/django-celery-beat@f014edcb954c707cb7628f4416257b6a58689523 From 5e32afcfde0c609ab2f5e7c7ad20978e33e9d87f Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Tue, 18 May 2021 15:09:08 -0400 Subject: [PATCH 12/14] remove some unused utils --- share/util/__init__.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/share/util/__init__.py b/share/util/__init__.py index 07e04dea7..bd9aa831a 100644 --- a/share/util/__init__.py +++ b/share/util/__init__.py @@ -9,22 +9,6 @@ def strip_whitespace(string): return re.sub(WHITESPACE_RE, ' ', string).strip() -def sort_dict_by_key(hierarchy): - types = OrderedDict() - for key, value in sorted(hierarchy.items()): - if isinstance(value, dict): - types[key] = sort_dict_by_key(value) - else: - types[key] = value - return types - - -def ensure_iterable(maybe_iterable): - if isinstance(maybe_iterable, (list, tuple, set)): - return maybe_iterable - return [maybe_iterable] - - class InvalidID(Exception): def __init__(self, value, message='Invalid ID'): super().__init__(value, message) @@ -216,15 +200,5 @@ def chunked(iterable, size=25, fail_fast=False): raise e -def interweave(*iterables): - iters = [iter(i) for i in iterables] - while iters: - for i in tuple(iters): - try: - yield next(i) - except StopIteration: - iters.remove(i) - - def placeholders(length): return ', '.join('%s' for _ in range(length)) From 2aecb885c6e533a8c1c0921d5197637cbc0b8d0d Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Tue, 18 May 2021 18:26:05 -0400 Subject: [PATCH 13/14] a few more sharectl tests --- share/bin/__init__.py | 1 - share/bin/info.py | 27 ---------- tests/share/bin/test_sharectl.py | 84 ++++++++++++++++++++++++++++++-- 3 files changed, 80 insertions(+), 32 deletions(-) delete mode 100644 share/bin/info.py diff --git a/share/bin/__init__.py b/share/bin/__init__.py index ba1d935b3..8784b16cf 100644 --- a/share/bin/__init__.py +++ b/share/bin/__init__.py @@ -8,7 +8,6 @@ MODULES = ( 'harvest', - 'info', 'ingest', 'search', ) diff --git a/share/bin/info.py b/share/bin/info.py deleted file mode 100644 index fc8c65e98..000000000 --- a/share/bin/info.py +++ /dev/null @@ -1,27 +0,0 @@ -from django.db.models import Q - -from share.models import SourceConfig -from share.bin.util import command - - -@command('List SourceConfigs') -def sources(args, argv): - """ - Usage: {0} sources [-e | -d] - - Options: - -e, --enabled Only list enabled SourceConfigs - -d, --disabled Only list disabled SourceConfigs - - Print out a list of currently installed source configs - """ - configs = SourceConfig.objects.all() - - if args['--enabled']: - configs = configs.exclude(disabled=True).exclude(source__is_deleted=True) - - if args['--disabled']: - configs = configs.filter(Q(disabled=True) | Q(source__is_deleted=True)) - - for config in configs.values_list('label', flat=True): - print(config) diff --git a/tests/share/bin/test_sharectl.py b/tests/share/bin/test_sharectl.py index 71f6a42f3..17dcd3c98 100644 --- a/tests/share/bin/test_sharectl.py +++ b/tests/share/bin/test_sharectl.py @@ -1,6 +1,82 @@ -import subprocess +import io +from contextlib import redirect_stdout -def test_sharectl_at_least_runs(): - completed_process = subprocess.run(['sharectl', '-v']) - assert completed_process.returncode == 0 +from unittest import mock +import pytest + +from share.bin.util import execute_cmd +import share.version + + +def run_sharectl(*args): + """run sharectl, assert that it returned as expected, and return its stdout + """ + fake_stdout = io.StringIO() + try: + with redirect_stdout(fake_stdout): + execute_cmd(args) + except SystemExit: + pass # success! + return fake_stdout.getvalue() + + +def test_sharectl_version(): + assert run_sharectl('-v').strip() == share.version.__version__ + + +class TestSharectlSearch: + @pytest.mark.parametrize('index_names', [ + ['one'], + ['another', 'makes', 'two'], + ]) + def test_purge(self, index_names): + expected_purge_calls = [ + mock.call(index_name) + for index_name in index_names + ] + mock_elastic_manager = mock.Mock() + with mock.patch('share.bin.search.ElasticManager', return_value=mock_elastic_manager): + run_sharectl('search', 'purge', *index_names) + assert mock_elastic_manager.delete_index.mock_calls == expected_purge_calls + + def test_setup_initial(self, settings): + expected_indexes = ['baz', 'bar', 'foo'] + settings.ELASTICSEARCH['ACTIVE_INDEXES'] = expected_indexes + mock_elastic_manager = mock.Mock() + with mock.patch('share.bin.search.ElasticManager', return_value=mock_elastic_manager): + run_sharectl('search', 'setup', '--initial') + + assert mock_elastic_manager.create_index.mock_calls == [ + mock.call(index_name) + for index_name in expected_indexes + ] + assert mock_elastic_manager.update_primary_alias.mock_calls == [mock.call(expected_indexes[0])] + + def test_setup_index(self): + mock_elastic_manager = mock.Mock() + with mock.patch('share.bin.search.ElasticManager', return_value=mock_elastic_manager): + run_sharectl('search', 'setup', 'foo') + assert mock_elastic_manager.create_index.mock_calls == [mock.call('foo')] + assert mock_elastic_manager.update_primary_alias.mock_calls == [] + + def test_set_primary(self): + mock_elastic_manager = mock.Mock() + with mock.patch('share.bin.search.ElasticManager', return_value=mock_elastic_manager): + run_sharectl('search', 'set_primary', 'blazblat') + assert mock_elastic_manager.update_primary_alias.mock_calls == [mock.call('blazblat')] + + def test_daemon(self, settings): + expected_indexes = ['bliz', 'blaz', 'bluz'] + settings.ELASTICSEARCH['ACTIVE_INDEXES'] = expected_indexes + + actual_indexes = [] + + def fake_start_indexer(_, stop_event, __, index_name): + actual_indexes.append(index_name) + stop_event.set() + + with mock.patch('share.bin.search.SearchIndexerDaemon') as mock_daemon: + mock_daemon.start_indexer_in_thread.side_effect = fake_start_indexer + run_sharectl('search', 'daemon') + assert actual_indexes == expected_indexes From 49a823790a7d0ec60e1cbe0aa2a09c8b3c83f7d8 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Wed, 19 May 2021 08:47:19 -0400 Subject: [PATCH 14/14] smoke tests for sharectl.harvest --- tests/share/bin/test_sharectl.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/share/bin/test_sharectl.py b/tests/share/bin/test_sharectl.py index 17dcd3c98..634fe70e5 100644 --- a/tests/share/bin/test_sharectl.py +++ b/tests/share/bin/test_sharectl.py @@ -80,3 +80,23 @@ def fake_start_indexer(_, stop_event, __, index_name): mock_daemon.start_indexer_in_thread.side_effect = fake_start_indexer run_sharectl('search', 'daemon') assert actual_indexes == expected_indexes + + +# TODO unit tests, not just a smoke test +def test_fetch_runs(): + with mock.patch('share.bin.harvest.SourceConfig'): + run_sharectl('fetch', 'foo.sourceconfig', '2021-05-05', '--print') + + +# TODO unit tests, not just a smoke test +def test_harvest_runs(): + with mock.patch('share.bin.harvest.SourceConfig'): + run_sharectl('harvest', 'foo.sourceconfig') + + +# TODO unit tests, not just a smoke test +def test_schedule_runs(): + with mock.patch('share.bin.harvest.SourceConfig'): + with mock.patch('share.bin.harvest.HarvestScheduler'): + with mock.patch('share.bin.harvest.tasks'): + run_sharectl('schedule', 'foo.sourceconfig')