Merge 035eb1a into 0a42bfb

dcicutils/_version.py

@@ -1,4 +1,4 @@
 """Version information."""
 
 # The following line *must* be the last in the module, exactly as formatted:
-__version__ = "0.5.7"
+__version__ = "0.5.8"

dcicutils/ff_utils.py

@@ -125,6 +125,58 @@ def search_request_with_retries(request_fxn, url, auth, verb, **kwargs):
     return final_res
 
 
+def purge_request_with_retries(request_fxn, url, auth, verb, **kwargs):
+    """
+    Example of using a non-standard retry function. This one is for purges,
+    which return a 423 if the item is locked. This function returns a list of
+    locked items to faciliate easier purging
+    """
+    final_res = None
+    error = None
+    retry = 0
+    # 423 is not included here because it is handled specially
+    non_retry_statuses = [401, 402, 403, 404, 405, 422]
+    retry_timeouts = [0, 1, 2, 3, 4]
+    while final_res is None and retry < len(retry_timeouts):
+        time.sleep(retry_timeouts[retry])
+        try:
+            res = request_fxn(url, auth=auth, **kwargs)
+        except Exception as e:
+            retry += 1
+            error = 'Error with %s request for %s: %s' % (verb.upper(), url, e)
+            continue
+        if res.status_code >= 400:
+            # attempt to get reason from res.json. then try raise_for_status
+            try:
+                err_reason = res.json()
+            except ValueError:
+                try:
+                    res.raise_for_status()
+                except Exception as e:
+                    err_reason = repr(e)
+                else:
+                    err_reason = res.reason
+            else:
+                # handle locked items
+                if res.status_code == 423:
+                    locked_items = err_reason.get('comment', [])
+                    if locked_items:
+                        final_res = res
+                        error = None
+                        break
+            retry += 1
+            error = ('Bad status code for %s request for %s: %s. Reason: %s'
+                     % (verb.upper(), url, res.status_code, err_reason))
+            if res.status_code in non_retry_statuses:
+                break
+        else:
+            final_res = res
+            error = None
+    if error and not final_res:
+        raise Exception(error)
+    return final_res
+
+
 def authorized_request(url, auth=None, ff_env=None, verb='GET',
                        retry_fxn=standard_request_with_retries, **kwargs):
     """
@@ -178,7 +230,7 @@ def get_metadata(obj_id, key=None, ff_env=None, check_queue=False, add_on=''):
     Takes an optional string add_on that should contain things like
     "frame=object". Join query parameters in the add_on using "&", e.g.
     "frame=object&force_md5"
-    *REQUIRES ff_env if check_queue is used.*
+    *REQUIRES ff_env if check_queue is used*
     """
     auth = get_authentication_with_server(key, ff_env)
     if check_queue and stuff_in_queues(ff_env, check_secondary=False):
@@ -329,6 +381,38 @@ def search_metadata(search, key=None, ff_env=None, page_limit=50, is_generator=F
         return search_res
 
 
+def get_metadata_links(obj_id, key=None, ff_env=None):
+    """
+    Given standard key/ff_env authentication, return result for @@links view
+    """
+    auth = get_authentication_with_server(key, ff_env)
+    purge_url = '/'.join([auth['server'], obj_id, '@@links'])
+    response = authorized_request(purge_url, auth=auth, verb='GET')
+    return get_response_json(response)
+
+
+def delete_metadata(obj_id, key=None, ff_env=None):
+    """
+    Given standard key/ff_env authentication, simply set the status of the
+    given object to 'deleted'
+    """
+    return patch_metadata({'status': 'deleted'}, obj_id, key, ff_env)
+
+
+def purge_metadata(obj_id, key=None, ff_env=None):
+    """
+    Given standard key/ff_env authentication, attempt to purge the item from
+    the DB (FULL delete). If the item cannot be deleted due to other items
+    still linking it, this function provides information in the response
+    `@graph`
+    """
+    auth = get_authentication_with_server(key, ff_env)
+    purge_url = '/'.join([auth['server'], obj_id]) + '?purge=True'
+    response = authorized_request(purge_url, auth=auth, verb='DELETE',
+                                  retry_fxn=purge_request_with_retries)
+    return get_response_json(response)
+
+
 def delete_field(obj_id, del_field, key=None, ff_env=None):
     """
     Given string obj_id and string del_field, delete a field(or fields seperated

dcicutils/jh_utils.py

@@ -44,6 +44,9 @@
     'patch_metadata',
     'upsert_metadata',
     'search_metadata',
+    'delete_metadata',
+    'purge_metadata',
+    'get_metadata_links',
     'delete_field'
 ]
 

test/test_ff_utils.py

@@ -1,6 +1,7 @@
 from dcicutils import ff_utils
 import pytest
 import json
+import time
 pytestmark = pytest.mark.working
 
 
@@ -158,7 +159,6 @@ def test_stuff_in_queues(integrated_ff):
     """
     Gotta index a bunch of stuff to make this work
     """
-    import time
     search_res = ff_utils.search_metadata('search/?limit=all&type=File', key=integrated_ff['ff_key'])
     # just take the first handful
     for item in search_res[:8]:
@@ -204,7 +204,6 @@ def test_authorized_request_integrated(integrated_ff):
 
 @pytest.mark.integrated
 def test_get_metadata(integrated_ff, basestring):
-    import time
     # use this test biosource
     test_item = '331111bc-8535-4448-903e-854af460b254'
     res_w_key = ff_utils.get_metadata(test_item, key=integrated_ff['ff_key'])
@@ -257,19 +256,72 @@ def test_patch_metadata(integrated_ff):
 
 
 @pytest.mark.integrated
-def test_post_metadata(integrated_ff):
-    test_data = {'biosource_type': 'immortalized cell line', 'award': '1U01CA200059-01',
-                 'lab': '4dn-dcic-lab', 'status': 'deleted'}
-    post_res = ff_utils.post_metadata(test_data, 'biosource', key=integrated_ff['ff_key'])
+def test_post_delete_purge_links_metadata(integrated_ff):
+    """
+    Combine all of these tests because they logically fit
+    """
+    post_data = {'biosource_type': 'immortalized cell line', 'award': '1U01CA200059-01',
+                 'lab': '4dn-dcic-lab'}
+    post_res = ff_utils.post_metadata(post_data, 'biosource', key=integrated_ff['ff_key'])
     post_item = post_res['@graph'][0]
     assert 'uuid' in post_item
-    assert post_item['biosource_type'] == test_data['biosource_type']
+    assert post_item['biosource_type'] == post_data['biosource_type']
     # make sure there is a 409 when posting to an existing item
-    test_data['uuid'] = post_item['uuid']
+    post_data['uuid'] = post_item['uuid']
     with pytest.raises(Exception) as exec_info:
-        ff_utils.post_metadata(test_data, 'biosource', key=integrated_ff['ff_key'])
+        ff_utils.post_metadata(post_data, 'biosource', key=integrated_ff['ff_key'])
     assert '409' in str(exec_info.value)  # 409 is conflict error
 
+    # make a biosample that links to the biosource
+    bios_data = {'biosource': [post_data['uuid']], 'status': 'deleted',
+                 'lab': '4dn-dcic-lab', 'award': '1U01CA200059-01'}
+    bios_res = ff_utils.post_metadata(bios_data, 'biosample', key=integrated_ff['ff_key'])
+    bios_item = bios_res['@graph'][0]
+    assert 'uuid' in bios_item
+
+    # delete the biosource
+    del_res = ff_utils.delete_metadata(post_item['uuid'], key=integrated_ff['ff_key'])
+    assert del_res['status'] == 'success'
+    assert del_res['@graph'][0]['status'] == 'deleted'
+
+    # test get_metadata_links function (this will ensure everything is indexed, as well)
+    links = []
+    tries = 0
+    while not links and tries < 10:
+        time.sleep(2)
+        post_links = ff_utils.get_metadata_links(post_item['uuid'], key=integrated_ff['ff_key'])
+        links = post_links.get('uuids_linking_to', [])
+        tries += 1
+    assert len(links) == 1
+    assert links[0]['uuid'] == bios_item['uuid']
+    assert links[0]['field'] == 'biosource[0].uuid'
+
+    # purge biosource first, which will failed because biosample is still linked
+    purge_res1 = ff_utils.purge_metadata(post_item['uuid'], key=integrated_ff['ff_key'])
+    assert purge_res1['status'] == 'error'
+    assert bios_item['uuid'] in [purge['uuid'] for purge in purge_res1['comment']]
+
+    # purge biosample and then biosource
+    purge_res2 = ff_utils.purge_metadata(bios_item['uuid'], key=integrated_ff['ff_key'])
+    assert purge_res2['status'] == 'success'
+
+    # wait for indexing to catch up
+    tries = 0
+    while len(links) > 0 and tries < 10:
+        time.sleep(2)
+        post_links = ff_utils.get_metadata_links(post_item['uuid'], key=integrated_ff['ff_key'])
+        links = post_links.get('uuids_linking_to', [])
+        tries += 1
+    assert len(links) == 0
+
+    purge_res3 = ff_utils.purge_metadata(post_item['uuid'], key=integrated_ff['ff_key'])
+    assert purge_res3['status'] == 'success'
+    # make sure it is purged
+    with pytest.raises(Exception) as exec_info:
+        ff_utils.get_metadata(post_item['uuid'], key=integrated_ff['ff_key'],
+                              add_on='datastore=database')
+    assert 'The resource could not be found' in str(exec_info.value)
+
 
 @pytest.mark.integrated
 def test_upsert_metadata(integrated_ff):