Merge pull request #793 from aaxelb/feature/open-badges

[ENG-3905] add osf_related_resource_types index field
CenterForOpenScience · Aug 8, 2022 · a702759 · a702759
2 parents 4eb2b3c + 9ba5094
commit a702759
Show file tree

Hide file tree

Showing 10 changed files with 137 additions and 6 deletions.
diff --git a/share/apps.py b/share/apps.py
@@ -1,10 +1,11 @@
 from django.apps import AppConfig
 from django.db.models.signals import post_migrate
-from share.signals import post_migrate_load_sources
+from share.signals import ensure_latest_elastic_mappings, post_migrate_load_sources
 
 
 class ShareConfig(AppConfig):
     name = 'share'
 
     def ready(self):
         post_migrate.connect(post_migrate_load_sources, sender=self)
+        post_migrate.connect(ensure_latest_elastic_mappings, sender=self)
diff --git a/share/bin/search.py b/share/bin/search.py
@@ -61,6 +61,14 @@ def setup(args, argv):
         elastic_manager.update_primary_alias(primary_index)
 
 
+@search.subcommand('Update mappings for an existing index')
+def update_mappings(args, argv):
+    """
+    Usage: {0} search update_mappings <index_name>
+    """
+    ElasticManager().update_mappings(args['<index_name>'])
+
+
 @search.subcommand('Set the "primary" index used to serve search results')
 def set_primary(args, argv):
     """

diff --git a/share/metadata_formats/sharev2_elastic.py b/share/metadata_formats/sharev2_elastic.py
@@ -121,6 +121,9 @@ def format(self, normalized_datum):
             'subjects': self._get_subjects(central_work, source_name),
             'subject_synonyms': self._get_subject_synonyms(central_work),
 
+            # osf-specific extra
+            'osf_related_resource_types': (central_work['extra'] or {}).get('osf_related_resource_types'),
+
             # a bunch of nested data because reasons -- used mostly for rendering search results
             'lists': {
                 'affiliations': self._build_related_agent_list(central_work, ['agentworkrelation']),

diff --git a/share/search/elastic_manager.py b/share/search/elastic_manager.py
@@ -49,10 +49,18 @@ def create_index(self, index_name):
             body={'settings': index_setup.index_settings},
         )
 
+        logger.info('Putting Elasticsearch mappings')
+        self.update_mappings(index_name)
+
+        self.es_client.indices.refresh(index_name)
+
         logger.debug('Waiting for yellow status')
         self.es_client.cluster.health(wait_for_status='yellow')
+        logger.info('Finished setting up Elasticsearch index %s', index_name)
+
+    def update_mappings(self, index_name):
+        index_setup = self.get_index_setup(index_name)
 
-        logger.info('Putting Elasticsearch mappings')
         for doc_type, mapping in index_setup.index_mappings.items():
             logger.debug('Putting mapping for %s', doc_type)
             self.es_client.indices.put_mapping(
@@ -61,10 +69,6 @@ def create_index(self, index_name):
                 index=index_name,
             )
 
-        self.es_client.indices.refresh(index_name)
-
-        logger.info('Finished setting up Elasticsearch index %s', index_name)
-
     def stream_actions(self, actions):
         stream = elastic_helpers.streaming_bulk(
             self.es_client,

diff --git a/share/search/index_setup/postrend_backcompat.py b/share/search/index_setup/postrend_backcompat.py
@@ -113,6 +113,7 @@ def index_mappings(self):
                     'type': {'type': 'keyword', 'include_in_all': False},
                     'types': {'type': 'keyword', 'include_in_all': False},
                     'withdrawn': {'type': 'boolean', 'include_in_all': False},
+                    'osf_related_resource_types': {'type': 'object', 'dynamic': True, 'include_in_all': False},
                     'lists': {'type': 'object', 'dynamic': True, 'include_in_all': False},
                 },
                 'dynamic_templates': [

diff --git a/share/signals.py b/share/signals.py
@@ -9,3 +9,11 @@ def post_migrate_load_sources(sender, **kwargs):
     except ProgrammingError:
         return
     management.call_command('loadsources')
+
+
+def ensure_latest_elastic_mappings(sender, **kwargs):
+    from share.search.elastic_manager import ElasticManager
+    elastic_manager = ElasticManager()
+
+    for index_name in elastic_manager.get_primary_indexes():
+        elastic_manager.update_mappings(index_name)
diff --git a/tests/share/bin/test_sharectl.py b/tests/share/bin/test_sharectl.py
@@ -66,6 +66,12 @@ def test_set_primary(self):
             run_sharectl('search', 'set_primary', 'blazblat')
         assert mock_elastic_manager.update_primary_alias.mock_calls == [mock.call('blazblat')]
 
+    def test_update_mappings(self):
+        mock_elastic_manager = mock.Mock()
+        with mock.patch('share.bin.search.ElasticManager', return_value=mock_elastic_manager):
+            run_sharectl('search', 'update_mappings', 'blazblat')
+        assert mock_elastic_manager.update_mappings.mock_calls == [mock.call('blazblat')]
+
     def test_daemon(self, settings):
         expected_indexes = ['bliz', 'blaz', 'bluz']
         settings.ELASTICSEARCH['ACTIVE_INDEXES'] = expected_indexes

diff --git a/tests/share/metadata_formats/base.py b/tests/share/metadata_formats/base.py
@@ -323,6 +323,49 @@
             },
         },
     },
+    'with-osf-extra': {
+        'suid_id': 99,
+        'source_name': 'OsfProbably',
+        'raw_datum_kwargs': {
+            'date_created': dateutil.parser.isoparse('2017-04-07T21:09:05.023090+00:00'),
+        },
+        'normalized_datum_kwargs': {
+            'created_at': dateutil.parser.isoparse('2017-04-07T21:09:05.023090+00:00'),
+            'data': {
+                '@graph': [
+                    {
+                        '@id': '_:p',
+                        '@type': 'person',
+                        'name': 'Open McOperton',
+                    },
+                    {
+                        '@id': '_:c',
+                        '@type': 'creator',
+                        'agent': {'@id': '_:p', '@type': 'person'},
+                        'creative_work': {'@id': '_:w', '@type': 'creativework'},
+                        'cited_as': 'Open McOperton',
+                        'order_cited': 0,
+                    },
+                    {
+                        '@id': '_:i',
+                        '@type': 'workidentifier',
+                        'creative_work': {'@id': '_:w', '@type': 'creativework'},
+                        'uri': 'https://example.com/open',
+                    },
+                    {
+                        '@id': '_:w',
+                        '@type': 'creativework',
+                        'title': 'So open',
+                        'date_updated': '2017-03-31T05:39:48+00:00',
+                        'extra': {
+                            'osf_related_resource_types': {'foo': True, 'bar': False},
+                        },
+                    },
+                ],
+                '@context': {}
+            },
+        },
+    },
 }
 
 

diff --git a/tests/share/metadata_formats/test_oai_dc_formatter.py b/tests/share/metadata_formats/test_oai_dc_formatter.py
@@ -81,4 +81,18 @@ def assert_formatter_outputs_equal(self, actual_output, expected_output):
           <dc:relation>http://staging.osf.io/vroom/</dc:relation>
         </oai_dc:dc>
         ''',
+        'with-osf-extra': '''
+        <oai_dc:dc
+            xmlns:dc="http://purl.org/dc/elements/1.1/"
+            xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
+            xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+            xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"
+        >
+          <dc:title>So open</dc:title>
+          <dc:creator>Open McOperton</dc:creator>
+          <dc:date>2017-03-31T05:39:48Z</dc:date>
+          <dc:type>creativework</dc:type>
+          <dc:identifier>https://example.com/open</dc:identifier>
+        </oai_dc:dc>
+        ''',
     }
diff --git a/tests/share/metadata_formats/test_sharev2_elastic_formatter.py b/tests/share/metadata_formats/test_sharev2_elastic_formatter.py
@@ -214,4 +214,47 @@ def assert_formatter_outputs_equal(self, actual_output, expected_output):
                 'publishers': [],
             },
         },
+        'with-osf-extra': {
+            'affiliations': [],
+            'contributors': ['Open McOperton'],
+            'date': '2017-03-31T05:39:48+00:00',
+            'date_created': '2017-04-07T21:09:05.023090+00:00',
+            'date_modified': '2017-04-07T21:09:05.023090+00:00',
+            'date_updated': '2017-03-31T05:39:48+00:00',
+            'id': 'encoded-99',
+            'identifiers': ['https://example.com/open'],
+            'sources': ['OsfProbably'],
+            'subject_synonyms': [],
+            'subjects': [],
+            'title': 'So open',
+            'type': 'creative work',
+            'types': ['creative work'],
+            'retracted': False,
+            'funders': [],
+            'hosts': [],
+            'publishers': [],
+            'tags': [],
+            'osf_related_resource_types': {
+                'foo': True,
+                'bar': False,
+            },
+            'lists': {
+                'affiliations': [],
+                'contributors': [
+                    {
+                        'cited_as': 'Open McOperton',
+                        'identifiers': [],
+                        'name': 'Open McOperton',
+                        'order_cited': 0,
+                        'relation': 'creator',
+                        'type': 'person',
+                        'types': ['person', 'agent'],
+                    },
+                ],
+                'lineage': [],
+                'funders': [],
+                'hosts': [],
+                'publishers': [],
+            },
+        },
     }