From 801efd95ff3038a23f962ab98afdad3b23607938 Mon Sep 17 00:00:00 2001 From: Calvin Nhieu Date: Thu, 19 Jul 2018 16:29:44 -0700 Subject: [PATCH 1/3] Refetch DSS API definition weekly (#79) --- hca/util/__init__.py | 12 ++++- test/test_dss_swagger_spec.py | 88 +++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 test/test_dss_swagger_spec.py diff --git a/hca/util/__init__.py b/hca/util/__init__.py index 6dab948e..1059d9a1 100755 --- a/hca/util/__init__.py +++ b/hca/util/__init__.py @@ -83,7 +83,7 @@ def special_cli_command(self, required_argument, optional_argument=""): from __future__ import absolute_import, division, print_function, unicode_literals -import os, types, collections, typing, json, errno, base64, argparse +import os, datetime, types, collections, typing, json, errno, base64, argparse try: from inspect import signature, Signature, Parameter @@ -176,6 +176,7 @@ class SwaggerClient(object): _authenticated_session = None _session = None _swagger_spec = None + _spec_valid_for_days = 7 _type_map = { "string": str, "number": float, @@ -238,7 +239,9 @@ def swagger_spec(self): else: swagger_filename = base64.urlsafe_b64encode(swagger_url.encode()).decode() + ".json" swagger_filename = os.path.join(self.config.user_config_dir, swagger_filename) - if not os.path.exists(swagger_filename): + is_cached = os.path.exists(swagger_filename) + if (not is_cached) or (is_cached and + self._get_days_since_last_modified(swagger_filename) >= self._spec_valid_for_days): try: os.makedirs(self.config.user_config_dir) except OSError as e: @@ -298,6 +301,11 @@ def login(self, access_token=""): token_type="Bearer") print("Storing access credentials") + def _get_days_since_last_modified(self, filename): + now = datetime.datetime.now() + last_modified = datetime.datetime.fromtimestamp(os.path.getmtime(filename)) + return (now - last_modified).days + def _get_oauth_token_from_service_account_credentials(self): scopes = ["https://www.googleapis.com/auth/userinfo.email"] assert 'GOOGLE_APPLICATION_CREDENTIALS' in os.environ diff --git a/test/test_dss_swagger_spec.py b/test/test_dss_swagger_spec.py new file mode 100644 index 00000000..c4e29ea5 --- /dev/null +++ b/test/test_dss_swagger_spec.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# coding: utf-8 + +import unittest +import requests +from hca.util.compat import USING_PYTHON2 + +import hca.dss + +if USING_PYTHON2: + import mock + from mock import mock_open +else: + from unittest import mock + from unittest.mock import mock_open + + +class TestDssSwaggerSpec(unittest.TestCase): + client = hca.dss.DSSClient() + swagger_url = "" + dummy_response = None + open_fn_name = "__builtin__.open" if USING_PYTHON2 else "builtins.open" + + def setUp(self): + self.client._swagger_spec = None + self.client._spec_valid_for_days = 1 + self.swagger_url = "test_swagger_url" + self.client._session = requests.Session() + self.client.config.__dict__.update({ + 'DSSClient': { + 'swagger_url': self.swagger_url + }, + 'swagger_filename': "test_filename" + }) + self.dummy_response = requests.models.Response() + self.dummy_response._content = b'{"swagger": ""}' + self.dummy_response.status_code = 200 + + def test_get_swagger_spec_new(self): + with mock.patch('os.path.exists') as mock_exists, \ + mock.patch('hca.dss.SwaggerClient._get_days_since_last_modified') as mock_days_since_last_modified, \ + mock.patch('os.makedirs') as mock_makedirs, \ + mock.patch('requests.Session.get') as mock_get, \ + mock.patch('hca.dss.SwaggerClient.load_swagger_json'), \ + mock.patch(self.open_fn_name, mock_open()) as open_mock: + mock_exists.return_value = False + mock_days_since_last_modified.return_value = 0 + mock_get.return_value = self.dummy_response + + test = self.client.swagger_spec + self.assertTrue(mock_makedirs.called) + self.assertTrue(mock_get.calledWith(self.swagger_url)) + self.assertTrue(open_mock().write.calledWith(self.dummy_response._content)) + + def test_get_swagger_spec_cache_valid(self): + with mock.patch('os.path.exists') as mock_exists, \ + mock.patch('hca.dss.SwaggerClient._get_days_since_last_modified') as mock_days_since_last_modified, \ + mock.patch('os.makedirs') as mock_makedirs, \ + mock.patch('requests.Session.get') as mock_get, \ + mock.patch('hca.dss.SwaggerClient.load_swagger_json'), \ + mock.patch(self.open_fn_name, mock_open()) as open_mock: + mock_exists.return_value = True + mock_days_since_last_modified.return_value = 0 + mock_get.return_value = self.dummy_response + + test = self.client.swagger_spec + self.assertFalse(mock_makedirs.called) + self.assertFalse(mock_get.called) + self.assertFalse(open_mock().write.called) + + def test_get_swagger_spec_cache_expired(self): + with mock.patch('os.path.exists') as mock_exists, \ + mock.patch('hca.dss.SwaggerClient._get_days_since_last_modified') as mock_days_since_last_modified, \ + mock.patch('os.makedirs') as mock_makedirs, \ + mock.patch('requests.Session.get') as mock_get, \ + mock.patch('hca.dss.SwaggerClient.load_swagger_json'), \ + mock.patch(self.open_fn_name, mock_open()) as open_mock: + mock_exists.return_value = True + mock_days_since_last_modified.return_value = 2 + mock_get.return_value = self.dummy_response + + test = self.client.swagger_spec + self.assertTrue(mock_get.calledWith(self.swagger_url)) + self.assertTrue(open_mock().write.calledWith(self.dummy_response._content)) + + +if __name__ == '__main__': + unittest.main() From 745e0880d714aab95465c868cc621d0d4f580cd8 Mon Sep 17 00:00:00 2001 From: Calvin Nhieu Date: Mon, 23 Jul 2018 09:09:04 -0700 Subject: [PATCH 2/3] Fix broken DSS api retry test --- hca/util/__init__.py | 2 +- test/test_dss_api_retry.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hca/util/__init__.py b/hca/util/__init__.py index 1059d9a1..20a23f5f 100755 --- a/hca/util/__init__.py +++ b/hca/util/__init__.py @@ -241,7 +241,7 @@ def swagger_spec(self): swagger_filename = os.path.join(self.config.user_config_dir, swagger_filename) is_cached = os.path.exists(swagger_filename) if (not is_cached) or (is_cached and - self._get_days_since_last_modified(swagger_filename) >= self._spec_valid_for_days): + self._get_days_since_last_modified(swagger_filename) >= self._spec_valid_for_days): try: os.makedirs(self.config.user_config_dir) except OSError as e: diff --git a/test/test_dss_api_retry.py b/test/test_dss_api_retry.py index e9888317..1d463032 100644 --- a/test/test_dss_api_retry.py +++ b/test/test_dss_api_retry.py @@ -10,6 +10,7 @@ from requests import ConnectTimeout from urllib3 import Timeout +from datetime import datetime from hca.util import RetryPolicy @@ -61,10 +62,12 @@ def test_get_retry(self): client = hca.dss.DSSClient() file_uuid = str(uuid.uuid4()) creator_uid = client.config.get("creator_uid", 0) + version = datetime.utcnow().strftime("%Y-%m-%dT%H%M%S.%fZ") client.put_file._request( dict( uuid=file_uuid, + version=version, bundle_uuid=str(uuid.uuid4()), creator_uid=creator_uid, source_url=TestDssApiRetry.source_url, From e85f706697cb80530d084343fd55cc141befcda0 Mon Sep 17 00:00:00 2001 From: Calvin Nhieu Date: Mon, 23 Jul 2018 10:17:18 -0700 Subject: [PATCH 3/3] Fix broken DSS API unit tests --- hca/util/__init__.py | 7 ++++--- test/test_dss_swagger_spec.py | 39 +++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/hca/util/__init__.py b/hca/util/__init__.py index 20a23f5f..b883433c 100755 --- a/hca/util/__init__.py +++ b/hca/util/__init__.py @@ -83,7 +83,7 @@ def special_cli_command(self, required_argument, optional_argument=""): from __future__ import absolute_import, division, print_function, unicode_literals -import os, datetime, types, collections, typing, json, errno, base64, argparse +import os, types, collections, typing, json, errno, base64, argparse try: from inspect import signature, Signature, Parameter @@ -95,6 +95,7 @@ def special_cli_command(self, required_argument, optional_argument=""): from requests_oauthlib import OAuth2Session from urllib3.util import retry, timeout from jsonpointer import resolve_pointer +from datetime import datetime from .. import get_config, logger from .compat import USING_PYTHON2 @@ -302,8 +303,8 @@ def login(self, access_token=""): print("Storing access credentials") def _get_days_since_last_modified(self, filename): - now = datetime.datetime.now() - last_modified = datetime.datetime.fromtimestamp(os.path.getmtime(filename)) + now = datetime.now() + last_modified = datetime.fromtimestamp(os.path.getmtime(filename)) return (now - last_modified).days def _get_oauth_token_from_service_account_credentials(self): diff --git a/test/test_dss_swagger_spec.py b/test/test_dss_swagger_spec.py index c4e29ea5..d79722bd 100644 --- a/test/test_dss_swagger_spec.py +++ b/test/test_dss_swagger_spec.py @@ -22,7 +22,7 @@ class TestDssSwaggerSpec(unittest.TestCase): open_fn_name = "__builtin__.open" if USING_PYTHON2 else "builtins.open" def setUp(self): - self.client._swagger_spec = None + self.client.__class__._swagger_spec = None self.client._spec_valid_for_days = 1 self.swagger_url = "test_swagger_url" self.client._session = requests.Session() @@ -36,6 +36,9 @@ def setUp(self): self.dummy_response._content = b'{"swagger": ""}' self.dummy_response.status_code = 200 + def tearDown(self): + self.client.__class__._swagger_spec = None + def test_get_swagger_spec_new(self): with mock.patch('os.path.exists') as mock_exists, \ mock.patch('hca.dss.SwaggerClient._get_days_since_last_modified') as mock_days_since_last_modified, \ @@ -54,27 +57,27 @@ def test_get_swagger_spec_new(self): def test_get_swagger_spec_cache_valid(self): with mock.patch('os.path.exists') as mock_exists, \ - mock.patch('hca.dss.SwaggerClient._get_days_since_last_modified') as mock_days_since_last_modified, \ - mock.patch('os.makedirs') as mock_makedirs, \ - mock.patch('requests.Session.get') as mock_get, \ - mock.patch('hca.dss.SwaggerClient.load_swagger_json'), \ - mock.patch(self.open_fn_name, mock_open()) as open_mock: - mock_exists.return_value = True - mock_days_since_last_modified.return_value = 0 - mock_get.return_value = self.dummy_response + mock.patch('hca.dss.SwaggerClient._get_days_since_last_modified') as mock_days_since_last_modified, \ + mock.patch('os.makedirs') as mock_makedirs, \ + mock.patch('requests.Session.get') as mock_get, \ + mock.patch('hca.dss.SwaggerClient.load_swagger_json'), \ + mock.patch(self.open_fn_name, mock_open()) as open_mock: + mock_exists.return_value = True + mock_days_since_last_modified.return_value = 0 + mock_get.return_value = self.dummy_response - test = self.client.swagger_spec - self.assertFalse(mock_makedirs.called) - self.assertFalse(mock_get.called) - self.assertFalse(open_mock().write.called) + test = self.client.swagger_spec + self.assertFalse(mock_makedirs.called) + self.assertFalse(mock_get.called) + self.assertFalse(open_mock().write.called) def test_get_swagger_spec_cache_expired(self): with mock.patch('os.path.exists') as mock_exists, \ - mock.patch('hca.dss.SwaggerClient._get_days_since_last_modified') as mock_days_since_last_modified, \ - mock.patch('os.makedirs') as mock_makedirs, \ - mock.patch('requests.Session.get') as mock_get, \ - mock.patch('hca.dss.SwaggerClient.load_swagger_json'), \ - mock.patch(self.open_fn_name, mock_open()) as open_mock: + mock.patch('hca.dss.SwaggerClient._get_days_since_last_modified') as mock_days_since_last_modified, \ + mock.patch('os.makedirs') as mock_makedirs, \ + mock.patch('requests.Session.get') as mock_get, \ + mock.patch('hca.dss.SwaggerClient.load_swagger_json'), \ + mock.patch(self.open_fn_name, mock_open()) as open_mock: mock_exists.return_value = True mock_days_since_last_modified.return_value = 2 mock_get.return_value = self.dummy_response