fix: Upgrade internals from legacy search result format to 2019-08-27

The changes here are purposefully simplistic to upgrade a very overdue
API result format. The fields system for processing entries is still
not ideal for a few different reasons, and should be made better for
both the ease of configuring fields in DGPF and handling different
kinds of entries. Those will be addressed later. For now, these changes
simply upgrade to the newer API version while preserving backwards
compatibility.

globus_portal_framework/constants.py

@@ -62,7 +62,7 @@
     'missing'
 ]
 # https://docs.globus.org/api/search/search/#request_documents
-DEFAULT_RESULT_FORMAT_VERSION = '2017-09-01'
+DEFAULT_RESULT_FORMAT_VERSION = '2019-08-27'
 
 BASE_TEMPLATES = 'globus-portal-framework/v2/'
 

globus_portal_framework/gsearch.py

@@ -444,9 +444,10 @@ def get_subject(index, subject, user=None):
     try:
         idata = get_index(index)
         if version.parse(globus_sdk.version.__version__).major < 3:
-            params = dict(result_format_version='2017-09-01')
+            params = dict(result_format_version=DEFAULT_RESULT_FORMAT_VERSION)
         else:
-            params = dict(query_params=dict(result_format_version='2017-09-01'))
+            rfv = dict(result_format_version=DEFAULT_RESULT_FORMAT_VERSION)
+            params = dict(query_params=rfv)
         result = client.get_subject(idata['uuid'], unquote(subject), **params)
         return process_search_data(idata.get('fields', {}), [result.data])[0]
     except globus_sdk.SearchAPIError:
@@ -462,20 +463,24 @@ def process_search_data(field_mappers, results):
     in from a simple query to Globus Search. See here:
     https://docs.globus.org/api/search/schemas/GMetaResult/
     :return: A list of search results:
-
-
     """
     structured_results = []
-    for entry in results:
-        content = entry['content']
+    for gmeta_result in results:
+        if gmeta_result.get('@version') != DEFAULT_RESULT_FORMAT_VERSION:
+            log.warning('Unsupported result format version '
+                        '{}'.format(gmeta_result['@version']))
+            continue
+
+        entries = gmeta_result['entries']
+        content = [e['content'] for e in entries]
         result = {
-            'subject': quote_plus(entry['subject']),
+            'subject': quote_plus(gmeta_result['subject']),
             'all': content
         }
 
         if len(content) == 0:
             log.warning('Subject {} contained no content, skipping...'.format(
-                entry['subject']
+                gmeta_result['subject']
             ))
             continue
         default_content = content[0]

tests/data/get_subject.json

@@ -1,16 +1,18 @@
 {
     "@datatype": "GMetaResult",
-    "@version": "2017-09-01",
-    "content": [
+    "@version": "2019-08-27",
+    "entries": [
         {
-            "globus_group": "b1208858-fc85-11e7-aca1-0ed2d3184bea",
-            "remote_file_manifest": [{
-                "filename": "iozone_log_ccs_dtn36_atlas1_50G_default.txt",
-                "length": 26716,
-                "md5": "48261c5f5f9d677a9ae6e4bed22a9f39",
-                "sha256": "b2e88ea7587409e5ba92a323d7a1c44736fd587bbe00128a95df9cbcd4bcc957",
-                "url": "globus://cf619dbb-623b-41fa-8988-1f3d26ebd887/ORNL/iozone/iozone_log_ccs_dtn36_atlas1_50G_default.txt"
-            }]
+            "content": {
+                "globus_group": "b1208858-fc85-11e7-aca1-0ed2d3184bea",
+                "remote_file_manifest": [{
+                    "filename": "iozone_log_ccs_dtn36_atlas1_50G_default.txt",
+                    "length": 26716,
+                    "md5": "48261c5f5f9d677a9ae6e4bed22a9f39",
+                    "sha256": "b2e88ea7587409e5ba92a323d7a1c44736fd587bbe00128a95df9cbcd4bcc957",
+                    "url": "globus://cf619dbb-623b-41fa-8988-1f3d26ebd887/ORNL/iozone/iozone_log_ccs_dtn36_atlas1_50G_default.txt"
+                }]
+            }
         }
     ],
     "subject": "globus://cf619dbb-623b-41fa-8988-1f3d26ebd887/ORNL/iozone/iozone_log_ccs_dtn36_atlas1_50G_default.txt"

tests/data/search.json

@@ -1,13 +1,13 @@
 {
     "@datatype": "GSearchResult",
-    "@version": "2017-09-01",
+    "@version": "2019-08-27",
     "count": 1,
     "gmeta": [
         {
             "@datatype": "GMetaResult",
-            "@version": "2017-09-01",
-            "content": [
-                {
+            "@version": "2019-08-27",
+            "entries": [{
+                "content": {
                     "globus_group": "ce3ef838-44c1-473e-aecb-c3ecfe6698db",
                     "remote_file_manifest": [
                         {
@@ -29,8 +29,10 @@
                             "md5":"febe6995bad457991331348f7b9c85fa"
                         }
                     ]
-                }
-            ],
+                },
+                "entry_id": null,
+                "matched_principal_sets": []
+            }],
             "subject": "globus://ddb59aef-6d04-11e5-ba46-22000b92c6ec:/share/godata/"
         }
     ],

tests/data/search_legacy.json

@@ -0,0 +1,54 @@
+{
+    "@datatype": "GSearchResult",
+    "@version": "2017-09-01",
+    "count": 1,
+    "gmeta": [
+        {
+            "@datatype": "GMetaResult",
+            "@version": "2017-09-01",
+            "content": [
+                {
+                    "globus_group": "ce3ef838-44c1-473e-aecb-c3ecfe6698db",
+                    "remote_file_manifest": [
+                        {
+                            "url":"globus://ddb59aef-6d04-11e5-ba46-22000b92c6ec/share/godata/file1.txt",
+                            "length":5,
+                            "filename":"file1.txt",
+                            "md5":"5bbf5a52328e7439ae6e719dfe712200"
+                        },
+                        {
+                            "url":"globus://ddb59aef-6d04-11e5-ba46-22000b92c6ec/share/godata/file2.txt",
+                            "length":5,
+                            "filename":"file2.txt",
+                            "md5":"c193497a1a06b2c72230e6146ff47080"
+                        },
+                        {
+                            "url":"globus://ddb59aef-6d04-11e5-ba46-22000b92c6ec/share/godata/file3.txt",
+                            "length":5,
+                            "filename":"file3.txt",
+                            "md5":"febe6995bad457991331348f7b9c85fa"
+                        }
+                    ]
+                }
+            ],
+            "subject": "globus://ddb59aef-6d04-11e5-ba46-22000b92c6ec:/share/godata/"
+        }
+    ],
+    "facet_results": [
+        {
+            "@datatype": "GFacetResult",
+            "@version": "2017-09-01",
+            "buckets": [
+                {
+                    "@datatype": "GBucket",
+                    "@version": "2017-09-01",
+                    "count": 35,
+                    "value": "IOZone"
+                }
+            ],
+            "name": "Subjects"
+        }
+    ],
+    "offset": 0,
+    "total": 1
+}

tests/test_gsearch.py

@@ -170,42 +170,48 @@ def test_process_search_data_with_no_records():
 
 
 def test_process_search_data_zero_length_content(mock_data):
-    sub = mock_data['search']['gmeta'][0]
-    sub['content'] = []
-    mappers, results = [], [sub]
+    gmeta = mock_data['search']['gmeta'][0]
+    gmeta['entries'] = []
+    mappers, results = [], [gmeta]
     data = process_search_data(mappers, results)
     assert data == []
 
 
-def test_process_search_data_with_one_entry(mock_data):
-    sub = mock_data['search']['gmeta'][0]
+def test_process_search_data_skips_legacy(mock_data):
+    sub = mock_data['search_legacy']['gmeta'][0]
     mappers, results = [], [sub]
-    data = process_search_data(mappers, results)[0]
-    assert quote_plus(sub['subject']) == data['subject']
-    assert sub['content'] == data['all']
+    assert process_search_data(mappers, results) == []
+
+
+def test_process_search_data_with_one_entry(mock_data):
+    # First search record
+    gmeta = mock_data['search']['gmeta'][0]
+    processsed_content = process_search_data([], [gmeta])
+    assert processsed_content
+    assert quote_plus(gmeta['subject']) == processsed_content[0]['subject']
+    assert gmeta['entries'][0]['content'] == processsed_content[0]['all'][0]
 
 
 def test_process_search_data_string_field(mock_data):
-    sub = mock_data['search']['gmeta'][0]
-    sub['content'][0]['foo'] = 'bar'
-    mappers, results = ['foo'], [sub]
-    data = process_search_data(mappers, results)[0]
+    gmeta = mock_data['search']['gmeta'][0]
+    gmeta['entries'][0]['content']['foo'] = 'bar'
+    data = process_search_data(['foo'], [gmeta])[0]
     assert data['foo'] == 'bar'
 
 
 def test_process_search_data_func_field(mock_data):
-    sub = mock_data['search']['gmeta'][0]
-    sub['content'][0]['foo'] = 'bar'
+    gmeta = mock_data['search']['gmeta'][0]
+    gmeta['entries'][0]['content']['foo'] = 'bar'
     mappers = [('foo', lambda x: x[0].get('foo').replace('b', 'c'))]
-    results = [sub]
+    results = [gmeta]
     data = process_search_data(mappers, results)[0]
     assert data['foo'] == 'car'
 
 
 def test_process_search_data_string_field_missing(mock_data):
-    sub = mock_data['search']['gmeta'][0]
-    sub['content'][0]['foo'] = 'bar'
-    mappers, results = ['foo'], [sub]
+    gmeta = mock_data['search']['gmeta'][0]
+    gmeta['entries'][0]['content']['foo'] = 'bar'
+    mappers, results = ['foo'], [gmeta]
     data = process_search_data(mappers, results)[0]
     assert data['foo'] == 'bar'