From 256de5d2ccb56bf4782bd5138aa0ccb0a6d0699d Mon Sep 17 00:00:00 2001 From: tifayuki Date: Wed, 16 Nov 2016 14:24:50 -0800 Subject: [PATCH 1/6] add namespace to all apis --- dockercloud/api/action.py | 2 +- dockercloud/api/auth.py | 4 +- dockercloud/api/base.py | 96 ++++++------- dockercloud/api/events.py | 9 +- dockercloud/api/node.py | 2 +- dockercloud/api/nodeaz.py | 2 +- dockercloud/api/nodecluster.py | 4 +- dockercloud/api/nodeprovider.py | 2 +- dockercloud/api/noderegion.py | 2 +- dockercloud/api/nodetype.py | 2 +- dockercloud/api/stack.py | 4 +- dockercloud/api/tag.py | 2 +- dockercloud/api/trigger.py | 21 +-- dockercloud/api/utils.py | 6 +- tests/test_action.py | 5 +- tests/test_auth.py | 6 +- tests/test_base.py | 237 ++++++++++++++++---------------- tests/test_container.py | 5 +- tests/test_http.py | 7 +- tests/test_node.py | 5 +- tests/test_nodecluster.py | 9 +- tests/test_nodeprovider.py | 5 +- tests/test_noderegion.py | 5 +- tests/test_nodetype.py | 5 +- tests/test_repository.py | 5 +- tests/test_service.py | 5 +- tests/test_utils.py | 5 +- 27 files changed, 253 insertions(+), 209 deletions(-) diff --git a/dockercloud/api/action.py b/dockercloud/api/action.py index 214827f..230bc51 100644 --- a/dockercloud/api/action.py +++ b/dockercloud/api/action.py @@ -6,7 +6,7 @@ class Action(Immutable): subsystem = 'audit' endpoint = "/action" - namespaced = False + is_namespaced = False @classmethod def _pk_key(cls): diff --git a/dockercloud/api/auth.py b/dockercloud/api/auth.py index 65d6668..ccfec44 100644 --- a/dockercloud/api/auth.py +++ b/dockercloud/api/auth.py @@ -12,6 +12,7 @@ HUB_INDEX = "https://index.docker.io/v1/" + def authenticate(username, password): verify_credential(username, password) dockercloud.basic_auth = base64.b64encode("%s:%s" % (username, password)) @@ -55,7 +56,8 @@ def load_from_file(f="~/.docker/config.json"): p = subprocess.Popen([cmd, 'get'], stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT) out = p.communicate(input=HUB_INDEX)[0] except: - raise dockercloud.AuthError('error getting credentials - err: exec: "%s": executable file not found in $PATH, out: ``' % cmd) + raise dockercloud.AuthError( + 'error getting credentials - err: exec: "%s": executable file not found in $PATH, out: ``' % cmd) try: credential = json.loads(out) diff --git a/dockercloud/api/base.py b/dockercloud/api/base.py index f889f69..b96ab9f 100644 --- a/dockercloud/api/base.py +++ b/dockercloud/api/base.py @@ -16,18 +16,19 @@ class BasicObject(object): _api_version = 'v1' - def __init__(self, **kwargs): - pass - class Restful(BasicObject): - _detail_uri = None - namespaced = True + is_namespaced = True - def __init__(self, **kwargs): + def __init__(self, namespace="", **kwargs): """Simply reflect all the values in kwargs""" for k, v in list(kwargs.items()): setattr(self, k, v) + if self.is_namespaced and namespace: + self._namespace = namespace + else: + self._namespace = dockercloud.namespace + self._resource_uri = "" def __addchanges__(self, name): changed_attrs = self.__getchanges__() @@ -38,7 +39,7 @@ def __addchanges__(self, name): def __setattr__(self, name, value): """Keeps track of what attributes have been set""" current_value = getattr(self, name, None) - if value != current_value: + if value != current_value and not name.startswith("_"): self.__addchanges__(name) super(Restful, self).__setattr__(name, value) @@ -53,17 +54,10 @@ def __setchanges__(self, val): def _loaddict(self, dict): """Internal. Sets the model attributes to the dictionary values passed""" - endpoint = getattr(self, 'endpoint', None) - subsystem = getattr(self, 'subsystem', None) - assert endpoint, "Endpoint not specified for %s" % self.__class__.__name__ - assert subsystem, "Subsystem not specified for %s" % self.__class__.__name__ for k, v in list(dict.items()): setattr(self, k, v) - if self.namespaced and dockercloud.namespace: - self._detail_uri = "/".join(["api", subsystem, self._api_version, dockercloud.namespace, - endpoint.strip("/"), self.pk]) - else: - self._detail_uri = "/".join(["api", subsystem, self._api_version, endpoint.strip("/"), self.pk]) + + self._resource_uri = getattr(self, "resource_uri", None) self.__setchanges__([]) @property @@ -93,9 +87,9 @@ def is_dirty(self): def _perform_action(self, action, params=None, data={}): """Internal. Performs the specified action on the object remotely""" success = False - if not self._detail_uri: + if not self._resource_uri: raise ApiError("You must save the object before performing this operation") - path = "/".join([self._detail_uri.rstrip("/"), action.lstrip("/")]) + path = "/".join([self._resource_uri.rstrip("/"), action.lstrip("/")]) json = send_request("POST", path, params=params, data=data) if json: self._loaddict(json) @@ -104,9 +98,9 @@ def _perform_action(self, action, params=None, data={}): def _expand_attribute(self, attribute): """Internal. Expands the given attribute from remote information""" - if not self._detail_uri: + if not self._resource_uri: raise ApiError("You must save the object before performing this operation") - path = "/".join([self._detail_uri, attribute]) + path = "/".join([self._resource_uri, attribute]) json = send_request("GET", path) if json: return json[attribute] @@ -125,39 +119,43 @@ def get_all_attributes(self): class Immutable(Restful): @classmethod - def fetch(cls, pk): - instance = None + def fetch(cls, pk, namespace=""): endpoint = getattr(cls, 'endpoint', None) subsystem = getattr(cls, 'subsystem', None) assert endpoint, "Endpoint not specified for %s" % cls.__name__ assert subsystem, "Subsystem not specified for %s" % cls.__name__ - if cls.namespaced and dockercloud.namespace: - detail_uri = "/".join(["api", subsystem, cls._api_version, dockercloud.namespace, endpoint.strip("/"), pk]) + + if not namespace: + namespace = dockercloud.namespace + if cls.is_namespaced and namespace: + resource_uri = "/".join(["api", subsystem, cls._api_version, namespace, endpoint.strip("/"), pk]) else: - detail_uri = "/".join(["api", subsystem, cls._api_version, endpoint.strip("/"), pk]) - json = send_request('GET', detail_uri) + resource_uri = "/".join(["api", subsystem, cls._api_version, endpoint.strip("/"), pk]) + json = send_request('GET', resource_uri) if json: instance = cls() instance._loaddict(json) return instance @classmethod - def list(cls, limit=None, **kwargs): + def list(cls, limit=None, namespace="", **kwargs): restful = [] endpoint = getattr(cls, 'endpoint', None) subsystem = getattr(cls, 'subsystem', None) assert endpoint, "Endpoint not specified for %s" % cls.__name__ assert subsystem, "Subsystem not specified for %s" % cls.__name__ - if cls.namespaced and dockercloud.namespace: - detail_uri = "/".join(["api", subsystem, cls._api_version, dockercloud.namespace, endpoint.strip("/")]) + if not namespace: + namespace = dockercloud.namespace + if cls.is_namespaced and namespace: + resource_uri = "/".join(["api", subsystem, cls._api_version, namespace, endpoint.strip("/")]) else: - detail_uri = "/".join(["api", subsystem, cls._api_version, endpoint.strip("/")]) + resource_uri = "/".join(["api", subsystem, cls._api_version, endpoint.strip("/")]) objects = [] while True: if limit and len(objects) >= limit: break - json = send_request('GET', detail_uri, params=kwargs) + json = send_request('GET', resource_uri, params=kwargs) objs = json.get('objects', []) meta = json.get('meta', {}) next_url = meta.get('next', '') @@ -182,10 +180,10 @@ def refresh(self, force=False): if self.is_dirty and not force: # We have local non-committed changes - rejecting the refresh success = False - elif not self._detail_uri: + elif not self._resource_uri: raise ApiError("You must save the object before performing this operation") else: - json = send_request("GET", self._detail_uri) + json = send_request("GET", self._resource_uri) if json: self._loaddict(json) success = True @@ -202,16 +200,17 @@ def create(cls, **kwargs): return cls(**kwargs) def delete(self): - if not self._detail_uri: + if not self._resource_uri: raise ApiError("You must save the object before performing this operation") action = "DELETE" - url = self._detail_uri + url = self._resource_uri json = send_request(action, url) if json: self._loaddict(json) + self._resource_uri = None else: # Object deleted successfully and nothing came back - deleting PK reference. - self._detail_uri = None + self._resource_uri = None # setattr(self, self._pk_key(), None) -- doesn't work self.__setchanges__([]) return True @@ -228,15 +227,15 @@ def save(self): assert endpoint, "Endpoint not specified for %s" % self.__class__.__name__ assert subsystem, "Subsystem not specified for %s" % self.__class__.__name__ # Figure out whether we should do a create or update - if not self._detail_uri: + if not self._resource_uri: action = "POST" - if cls.namespaced and dockercloud.namespace: - path = "/".join(["api", subsystem, self._api_version, dockercloud.namespace, endpoint.lstrip("/")]) + if cls.is_namespaced and self._namespace: + path = "/".join(["api", subsystem, self._api_version, self._namespace, endpoint.lstrip("/")]) else: path = "/".join(["api", subsystem, self._api_version, endpoint.lstrip("/")]) else: action = "PATCH" - path = self._detail_uri + path = self._resource_uri # Construct the necessary params params = {} for attr in self.__getchanges__(): @@ -322,13 +321,16 @@ def run_forever(self, *args, **kwargs): class StreamingLog(StreamingAPI): - def __init__(self, subsystem, resource, uuid, tail, follow): + def __init__(self, subsystem, resource, uuid, tail, follow, namespace=""): endpoint = "%s/%s/logs/?follow=%s" % (resource, uuid, str(follow).lower()) if tail: endpoint = "%s&tail=%d" % (endpoint, tail) - if dockercloud.namespace: + + if not namespace: + namespace = dockercloud.namespace + if namespace: url = "/".join([dockercloud.stream_host.rstrip("/"), "api", subsystem, self._api_version, - dockercloud.namespace, endpoint.lstrip("/")]) + self._namespace, endpoint.lstrip("/")]) else: url = "/".join([dockercloud.stream_host.rstrip("/"), "api", subsystem, self._api_version, endpoint.lstrip("/")]) @@ -348,11 +350,13 @@ def run_forever(self, *args, **kwargs): class Exec(StreamingAPI): - def __init__(self, uuid, cmd='sh'): + def __init__(self, uuid, cmd='sh', namespace=""): endpoint = "container/%s/exec/?command=%s" % (uuid, urllib.quote_plus(cmd)) - if dockercloud.namespace: + if not namespace: + namespace = dockercloud.namespace + if namespace: url = "/".join([dockercloud.stream_host.rstrip("/"), "api", "app", self._api_version, - dockercloud.namespace, endpoint.lstrip("/")]) + namespace, endpoint.lstrip("/")]) else: url = "/".join([dockercloud.stream_host.rstrip("/"), "api", "app", self._api_version, endpoint.lstrip("/")]) super(self.__class__, self).__init__(url) diff --git a/dockercloud/api/events.py b/dockercloud/api/events.py index 4440f72..c16ec31 100644 --- a/dockercloud/api/events.py +++ b/dockercloud/api/events.py @@ -13,11 +13,14 @@ class Events(StreamingAPI): - def __init__(self): + def __init__(self, namespace=""): endpoint = "events" - if dockercloud.namespace: + + if not namespace: + namespace = dockercloud.namespace + if namespace: url = "/".join([dockercloud.stream_host.rstrip("/"), "api", "audit", self._api_version, - dockercloud.namespace, endpoint.lstrip("/")]) + namespace, endpoint.lstrip("/")]) else: url = "/".join([dockercloud.stream_host.rstrip("/"), "api", "audit", self._api_version, endpoint.lstrip("/")]) diff --git a/dockercloud/api/node.py b/dockercloud/api/node.py index 31f80ff..1d65901 100644 --- a/dockercloud/api/node.py +++ b/dockercloud/api/node.py @@ -8,7 +8,7 @@ class Node(Mutable, Taggable): endpoint = "/node" def save(self): - if not self._detail_uri: + if not self.resource_uri: raise AttributeError("Adding a new node is not supported via 'save' method") super(Node, self).save() diff --git a/dockercloud/api/nodeaz.py b/dockercloud/api/nodeaz.py index 25a153b..76ec50a 100644 --- a/dockercloud/api/nodeaz.py +++ b/dockercloud/api/nodeaz.py @@ -6,7 +6,7 @@ class AZ(Immutable): subsystem = "infra" endpoint = "/az" - namespaced = False + is_namespaced = False @classmethod def _pk_key(cls): diff --git a/dockercloud/api/nodecluster.py b/dockercloud/api/nodecluster.py index 2443b40..5b43bc7 100644 --- a/dockercloud/api/nodecluster.py +++ b/dockercloud/api/nodecluster.py @@ -16,7 +16,7 @@ def deploy(self, tag=None): def create(cls, **kwargs): for key, value in kwargs.items(): if key == "node_type" and isinstance(value, NodeType): - kwargs[key] = getattr(value, "resource_uri", "") + kwargs[key] = getattr(value, "_resource_uri", "") if key == "region" and isinstance(value, Region): - kwargs[key] = getattr(value, "resource_uri", "") + kwargs[key] = getattr(value, "_resource_uri", "") return cls(**kwargs) diff --git a/dockercloud/api/nodeprovider.py b/dockercloud/api/nodeprovider.py index 168bc72..d3ed81e 100644 --- a/dockercloud/api/nodeprovider.py +++ b/dockercloud/api/nodeprovider.py @@ -6,7 +6,7 @@ class Provider(Immutable): subsystem = "infra" endpoint = "/provider" - namespaced = False + is_namespaced = False @classmethod def _pk_key(cls): diff --git a/dockercloud/api/noderegion.py b/dockercloud/api/noderegion.py index 9a9c97a..8e1db4d 100644 --- a/dockercloud/api/noderegion.py +++ b/dockercloud/api/noderegion.py @@ -6,7 +6,7 @@ class Region(Immutable): subsystem = "infra" endpoint = "/region" - namespaced = False + is_namespaced = False @classmethod def _pk_key(cls): diff --git a/dockercloud/api/nodetype.py b/dockercloud/api/nodetype.py index 7416d63..867f36f 100644 --- a/dockercloud/api/nodetype.py +++ b/dockercloud/api/nodetype.py @@ -6,7 +6,7 @@ class NodeType(Immutable): subsystem = "infra" endpoint = "/nodetype" - namespaced = False + is_namespaced = False @classmethod def _pk_key(cls): diff --git a/dockercloud/api/stack.py b/dockercloud/api/stack.py index 331b690..71d035b 100644 --- a/dockercloud/api/stack.py +++ b/dockercloud/api/stack.py @@ -20,7 +20,7 @@ def redeploy(self, reuse_volumes=True): return self._perform_action("redeploy", params=params) def export(self): - if not self._detail_uri: + if not self.resource_uri: raise ApiError("You must save the object before performing this operation") - url = "/".join([self._detail_uri, "export"]) + url = "/".join([self.resource_uri, "export"]) return send_request("GET", url, inject_header=False) diff --git a/dockercloud/api/tag.py b/dockercloud/api/tag.py index 10d5354..a08371f 100644 --- a/dockercloud/api/tag.py +++ b/dockercloud/api/tag.py @@ -57,7 +57,7 @@ def delete(self, tagname): def fetch(cls, taggable): if not isinstance(taggable, Taggable): raise ApiError("The object does not support tag") - if not taggable._detail_uri: + if not taggable.resource_uri: raise ApiError("You must save the taggable object before performing this operation") tag = cls() diff --git a/dockercloud/api/trigger.py b/dockercloud/api/trigger.py index 1c5aa73..46e36b7 100644 --- a/dockercloud/api/trigger.py +++ b/dockercloud/api/trigger.py @@ -8,6 +8,7 @@ class Trigger(BasicObject): def __init__(self): self.trigger = None + self.resource_uri = None def add(self, name=None, operation=None): @@ -30,11 +31,11 @@ def create(cls, **kwargs): return cls(**kwargs) def delete(self, uuid): - if not self.endpoint: + if not self.resource_uri: raise ApiError("You must initialize the Trigger object before performing this operation") action = "DELETE" - url = "/".join([self.endpoint, uuid]) + url = "/".join([self.resource_uri, uuid]) send_request(action, url) return True @@ -43,11 +44,11 @@ def fetch(cls, triggerable): if not isinstance(triggerable, Triggerable): raise ApiError("The object does not support trigger") - if not triggerable._detail_uri: + if not triggerable.resource_uri: raise ApiError("You must save the triggerable object before performing this operation") trigger = cls() - trigger.endpoint = "/".join([triggerable._detail_uri, "trigger"]) + trigger.resource_uri = "/".join([triggerable.resource_uri, "trigger"]) handlers = [] for t in trigger.list(): triggername = t.get("name", "") @@ -56,12 +57,12 @@ def fetch(cls, triggerable): return trigger def list(self, **kwargs): - if not self.endpoint: + if not self.resource_uri: raise ApiError("You must initialize the Trigger object before performing this operation") objects = [] while True: - json = send_request('GET', self.endpoint, params=kwargs) + json = send_request('GET', self.resource_uri, params=kwargs) objs = json.get('objects', []) meta = json.get('meta', {}) next_url = meta.get('next', '') @@ -77,23 +78,23 @@ def list(self, **kwargs): return objects def save(self): - if not self.endpoint: + if not self.resource_uri: raise ApiError("You must initialize the Trigger object before performing this operation") if self.trigger is None: return True - json = send_request("POST", self.endpoint, data=json_parser.dumps(self.trigger)) + json = send_request("POST", self.resource_uri, data=json_parser.dumps(self.trigger)) if json: self.clear() self.clear() return True def call(self, uuid): - if not self.endpoint: + if not self.resource_uri: raise ApiError("You must initialize the Trigger object before performing this operation") - json = send_request("POST", "/".join([self.endpoint, uuid + "/call"])) + json = send_request("POST", "/".join([self.resource_uri, uuid + "/call"])) if json: return True return False diff --git a/dockercloud/api/utils.py b/dockercloud/api/utils.py index 8719aa2..20d5c68 100644 --- a/dockercloud/api/utils.py +++ b/dockercloud/api/utils.py @@ -43,7 +43,7 @@ def fetch_by_resource_uri(uri): return Action.fetch(id) else: raise ApiError( - "Unsupported resource type. Only support: action, container, node, nodecluster, service, stack") + "Unsupported resource type. Only support: action, container, node, nodecluster, service, stack") @staticmethod def fetch_remote_container(identifier, raise_exceptions=True): @@ -162,7 +162,7 @@ def fetch_remote_nodecluster(identifier, raise_exceptions=True): elif len(objects_same_identifier) == 0: raise ObjectNotFound("Cannot find a node cluster with the identifier '%s'" % identifier) raise NonUniqueIdentifier( - "More than one node cluster has the same identifier, please use the long uuid") + "More than one node cluster has the same identifier, please use the long uuid") except (NonUniqueIdentifier, ObjectNotFound) as e: if not raise_exceptions: @@ -185,7 +185,7 @@ def fetch_remote_action(identifier, raise_exceptions=True): elif len(objects_same_identifier) == 0: raise ObjectNotFound("Cannot find an action cluster with the identifier '%s'" % identifier) raise NonUniqueIdentifier( - "More than one action has the same identifier, please use the long uuid") + "More than one action has the same identifier, please use the long uuid") except (NonUniqueIdentifier, ObjectNotFound) as e: if not raise_exceptions: diff --git a/tests/test_action.py b/tests/test_action.py index f5893a7..faba154 100644 --- a/tests/test_action.py +++ b/tests/test_action.py @@ -2,7 +2,10 @@ import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from .fake_api import * diff --git a/tests/test_auth.py b/tests/test_auth.py index dd81a01..6ce8ab5 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -4,7 +4,10 @@ import tempfile import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from .fake_api import * @@ -75,7 +78,6 @@ def test_auth_get_auth_header(self): dockercloud.basic_auth = FAKE_BASIC_AUTH self.assertEqual({'Authorization': FAKE_DOCKERCLOUD_AUTH}, dockercloud.auth.get_auth_header()) - print "====================" dockercloud.dockercloud_auth = None dockercloud.basic_auth = FAKE_BASIC_AUTH self.assertEqual({'Authorization': 'Basic %s' % (FAKE_BASIC_AUTH)}, dockercloud.auth.get_auth_header()) diff --git a/tests/test_base.py b/tests/test_base.py index d2be83a..5e3b4f3 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -3,7 +3,10 @@ import json import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from dockercloud.api.base import Restful, Mutable, Immutable @@ -54,14 +57,12 @@ def test_restful_setchanges(self): def test_restful_loaddict(self): model = Restful() - self.assertRaises(AssertionError, model._loaddict, {'key': 'value'}) - - model.endpoint = 'endpoint' + model.endpoint = 'resource_uri' model.subsystem = "subsystem" - model._loaddict({'key': 'value'}) + resource_uri = "/".join(["api", model.subsystem, model._api_version, model.endpoint.lstrip("/"), model.pk]) + model._loaddict({'key': 'value', "resource_uri": resource_uri}) self.assertEqual('value', model.key) - self.assertEqual("/".join(["api", model.subsystem, model._api_version, model.endpoint.lstrip("/"), model.pk]), - model._detail_uri) + self.assertEqual(resource_uri, model._resource_uri) self.assertEqual([], model.__getchanges__()) def test_restful_pk(self): @@ -77,32 +78,29 @@ def test_restful_is_dirty(self): @mock.patch('dockercloud.api.base.send_request') def test_restful_perform_action(self, mock_send_request): - try: - model = Restful() - self.assertRaises(dockercloud.ApiError, model._perform_action, 'action') - model.endpoint = 'fake' - model.subsystem = "subsystem" - model._detail_uri = "/".join( - ["api", model.subsystem, model._api_version, model.endpoint.lstrip("/"), model.pk]) - mock_send_request.side_effect = [{'key': 'value'}, None] - self.assertTrue(model._perform_action('action', params={'k': 'v'}, data={'key': 'value'})) - self.assertEqual('value', model.key) - mock_send_request.assert_called_with('POST', "/".join([model._detail_uri, "action"]), data={'key': 'value'}, - params={'k': 'v'}) + model = Restful() + self.assertRaises(dockercloud.ApiError, model._perform_action, 'action') - self.assertFalse(model._perform_action('action', {'key': 'value'})) + model.endpoint = 'fake' + model.subsystem = "subsystem" + model.resource_uri = "/".join( + ["api", model.subsystem, model._api_version, model.endpoint.lstrip("/"), model.pk]) + model._resource_uri = model.resource_uri + mock_send_request.side_effect = [{'key': 'value'}, None] + self.assertTrue(model._perform_action('action', params={'k': 'v'}, data={'key': 'value'})) + self.assertEqual('value', model.key) + mock_send_request.assert_called_with('POST', "/".join([model._resource_uri, "action"]), data={'key': 'value'}, + params={'k': 'v'}) - finally: - if hasattr(Restful, 'endpoint'): - delattr(Restful, 'endpoint') + self.assertFalse(model._perform_action('action', {'key': 'value'})) @mock.patch('dockercloud.api.base.send_request') def test_restful_expand_attribute(self, mock_send_request): model = Restful() self.assertRaises(dockercloud.ApiError, model._expand_attribute, 'attribute') - model._detail_uri = 'fake/uuid' + model._resource_uri = 'fake/uuid' mock_send_request.side_effect = [{'key': 'value'}, None] self.assertEqual('value', model._expand_attribute('key')) @@ -124,63 +122,65 @@ def tearDown(self): @mock.patch('dockercloud.api.base.send_request') def test_immutable_fetch(self, mock_send_request): + try: + delattr(Immutable, 'endpoint') + delattr(Immutable, 'subsystem') + except: + pass self.assertRaises(AssertionError, Immutable.fetch, 'uuid') - try: - ret_json = {"key": "value"} - mock_send_request.return_value = ret_json - Immutable.endpoint = 'endpoint' - Immutable.subsystem = "subsystem" - model = Immutable.fetch('uuid') - mock_send_request.assert_called_with('GET', 'api/subsystem/%s/endpoint/uuid' % Immutable._api_version) - self.assertIsInstance(model, Immutable) - self.assertEqual('value', model.key) - finally: - if hasattr(Immutable, 'endpoint'): - delattr(Immutable, 'endpoint') + ret_json = {"key": "value"} + mock_send_request.return_value = ret_json + Immutable.endpoint = 'resource_uri' + Immutable.subsystem = "subsystem" + model = Immutable.fetch('uuid') + mock_send_request.assert_called_with('GET', 'api/subsystem/%s/resource_uri/uuid' % Immutable._api_version) + self.assertIsInstance(model, Immutable) + self.assertEqual('value', model.key) @mock.patch('dockercloud.api.base.send_request') def test_immutable_list(self, mock_send_request): - self.assertRaises(AssertionError, Immutable.list) try: - kwargs = {'key': 'value'} - ret_json = {"meta": {"limit": 25, "next": None, "offset": 0, "previous": None, "total_count": 1}, - "objects": [{"key": "value1"}, {"key": "value2"}]} - mock_send_request.return_value = ret_json - Immutable.endpoint = 'fake' - models = Immutable.list(**kwargs) - mock_send_request.assert_called_with('GET', 'api/subsystem/%s/fake' % Immutable._api_version, params=kwargs) - self.assertEqual(2, len(models)) - self.assertIsInstance(models[0], Immutable) - self.assertEqual('value1', models[0].key) - self.assertIsInstance(models[1], Immutable) - self.assertEqual('value2', models[1].key) - finally: - if hasattr(Immutable, 'endpoint'): - delattr(Immutable, 'endpoint') + delattr(Immutable, 'endpoint') + delattr(Immutable, 'subsystem') + except: + pass + self.assertRaises(AssertionError, Immutable.list) + + kwargs = {'key': 'value'} + ret_json = {"meta": {"limit": 25, "next": None, "offset": 0, "previous": None, "total_count": 1}, + "objects": [{"key": "value1"}, {"key": "value2"}]} + mock_send_request.return_value = ret_json + Immutable.endpoint = 'fake' + Immutable.subsystem = "subsystem" + models = Immutable.list(**kwargs) + mock_send_request.assert_called_with('GET', 'api/subsystem/%s/fake' % Immutable._api_version, params=kwargs) + self.assertEqual(2, len(models)) + self.assertIsInstance(models[0], Immutable) + self.assertEqual('value1', models[0].key) + self.assertIsInstance(models[1], Immutable) + self.assertEqual('value2', models[1].key) @mock.patch('dockercloud.api.base.send_request') def test_immutable_refresh(self, mock_send_request): - try: - model = Immutable() - model.key = 'value' - self.assertFalse(model.refresh(force=False)) - self.assertRaises(dockercloud.ApiError, model.refresh, force=True) + model = Immutable() + model.key = 'value' + self.assertFalse(model.refresh(force=False)) + + self.assertRaises(dockercloud.ApiError, model.refresh, force=True) - Immutable.endpoint = 'endpoint' - Immutable.subsystem = 'subsystem' - model._detail_uri = 'api/subsystem/%s/endpoint/uuid' % Immutable._api_version - mock_send_request.side_effect = [{'newkey': 'newvalue'}, None] - self.assertTrue(model.refresh(force=True)) - self.assertEqual('newvalue', model.newkey) - mock_send_request.assert_called_with('GET', model._detail_uri) + Immutable.endpoint = 'resource_uri' + Immutable.subsystem = 'subsystem' + model.resource_uri = 'api/subsystem/%s/resource_uri/uuid' % Immutable._api_version + model._resource_uri = model.resource_uri + mock_send_request.side_effect = [{'newkey': 'newvalue'}, None] + self.assertTrue(model.refresh(force=True)) + self.assertEqual('newvalue', model.newkey) + mock_send_request.assert_called_with('GET', model._resource_uri) - self.assertFalse(model.refresh(force=True)) - mock_send_request.assert_called_with('GET', model._detail_uri) - finally: - if hasattr(Immutable, 'endpoint'): - delattr(Immutable, 'endpoint') + self.assertFalse(model.refresh(force=True)) + mock_send_request.assert_called_with('GET', model._resource_uri) class MutableTestCase(unittest.TestCase): @@ -196,57 +196,56 @@ def test_mutable_create(self): @mock.patch('dockercloud.api.base.send_request') def test_mutable_delete(self, mock_send_request): - try: - model = Mutable() - self.assertRaises(dockercloud.ApiError, model.delete) - - Mutable.endpoint = 'fake' - model._detail_uri = 'fake/uuid' - mock_send_request.side_effect = [{'key': 'value'}, None] - self.assertTrue(model.delete()) - self.assertEqual('value', model.key) - mock_send_request.assert_called_with('DELETE', 'fake/uuid') - - self.assertTrue(model.delete()) - self.assertIsNone(model._detail_uri) - self.assertFalse(model.is_dirty) - finally: - if hasattr(Mutable, 'endpoint'): - delattr(Mutable, 'endpoint') + model = Mutable() + self.assertRaises(dockercloud.ApiError, model.delete) + + Mutable.endpoint = 'fake' + model._resource_uri = 'fake/uuid' + mock_send_request.side_effect = [{'key': 'value'}, None] + self.assertTrue(model.delete()) + self.assertEqual('value', model.key) + mock_send_request.assert_called_with('DELETE', 'fake/uuid') + + model = Mutable() + model._resource_uri = 'fake/uuid' + self.assertTrue(model.delete()) + self.assertIsNone(model._resource_uri) + self.assertFalse(model.is_dirty) @mock.patch('dockercloud.api.base.send_request') def test_mutable_save(self, mock_send_request): + self.assertTrue(Mutable().save()) + + model = Mutable() + model.key = 'value' try: - self.assertTrue(Mutable().save()) - - model = Mutable() - model.key = 'value' - self.assertRaises(AssertionError, model.save) - - Mutable.endpoint = 'endpoint' - Mutable.subsystem = 'subsystem' - mock_send_request.return_value = None - result = model.save() - mock_send_request.assert_called_with('POST', 'api/subsystem/%s/endpoint' % Mutable._api_version, - data=json.dumps({'key': 'value'})) - self.assertFalse(result) - - mock_send_request.return_value = {'newkey': 'newvalue'} - result = model.save() - mock_send_request.assert_called_with('POST', 'api/subsystem/%s/endpoint' % Mutable._api_version, - data=json.dumps({'key': 'value'})) - self.assertTrue(result) - self.assertEqual('newvalue', model.newkey) - - model.key = 'another value' - mock_send_request.return_value = {'newkey2': 'newvalue2'} - model._detail_uri = 'api/subsystem/%s/endpoint/uuid' % Immutable._api_version - result = model.save() - mock_send_request.assert_called_with('PATCH', 'api/subsystem/%s/endpoint/uuid' % Mutable._api_version, - data=json.dumps({'key': 'another value'})) - self.assertTrue(result) - self.assertEqual('another value', model.key) - self.assertEqual('newvalue2', model.newkey2) - finally: - if hasattr(Mutable, 'endpoint'): - delattr(Mutable, 'endpoint') + delattr(Immutable, 'endpoint') + delattr(Immutable, 'subsystem') + except: + pass + self.assertRaises(AssertionError, model.save) + + Mutable.endpoint = 'resource_uri' + Mutable.subsystem = 'subsystem' + mock_send_request.return_value = None + result = model.save() + mock_send_request.assert_called_with('POST', 'api/subsystem/%s/resource_uri' % Mutable._api_version, + data=json.dumps({'key': 'value'})) + self.assertFalse(result) + + mock_send_request.return_value = {'newkey': 'newvalue'} + result = model.save() + mock_send_request.assert_called_with('POST', 'api/subsystem/%s/resource_uri' % Mutable._api_version, + data=json.dumps({'key': 'value'})) + self.assertTrue(result) + self.assertEqual('newvalue', model.newkey) + + model.key = 'another value' + mock_send_request.return_value = {'newkey2': 'newvalue2'} + model._resource_uri = 'api/subsystem/%s/resource_uri/uuid' % Immutable._api_version + result = model.save() + mock_send_request.assert_called_with('PATCH', 'api/subsystem/%s/resource_uri/uuid' % Mutable._api_version, + data=json.dumps({'key': 'another value'})) + self.assertTrue(result) + self.assertEqual('another value', model.key) + self.assertEqual('newvalue2', model.newkey2) diff --git a/tests/test_container.py b/tests/test_container.py index 9fa54a3..5e8c0e7 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -2,7 +2,10 @@ import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from .fake_api import * diff --git a/tests/test_http.py b/tests/test_http.py index 88802a4..ee09e80 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -1,9 +1,12 @@ from __future__ import absolute_import +import requests import unittest -import requests -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from dockercloud.api.base import send_request diff --git a/tests/test_node.py b/tests/test_node.py index 6afa961..61e1bdf 100644 --- a/tests/test_node.py +++ b/tests/test_node.py @@ -2,7 +2,10 @@ import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from .fake_api import * diff --git a/tests/test_nodecluster.py b/tests/test_nodecluster.py index 20548d7..77105f3 100644 --- a/tests/test_nodecluster.py +++ b/tests/test_nodecluster.py @@ -2,7 +2,10 @@ import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from .fake_api import * @@ -40,7 +43,7 @@ def test_nodecluster_save(self, mock_send): ) mock_send.return_value = fake_resp(fake_nodecluster_save) cluster = dockercloud.NodeCluster.create(name="my_cluster", region="/api/v1/region/digitalocean/lon1/", - node_type="/api/v1/nodetype/digitalocean/1gb/") + node_type="/api/v1/nodetype/digitalocean/1gb/") self.assertTrue(cluster.save()) result = json.loads(json.dumps(cluster.get_all_attributes())) target = json.loads(json.dumps(attribute)) @@ -53,7 +56,7 @@ def test_nodecluster_deploy(self, mock_send): ) mock_send.side_effect = [fake_resp(fake_nodecluster_save), fake_resp(fake_nodecluster_deploy)] cluster = dockercloud.NodeCluster.create(name="my_cluster", region="/api/v1/region/digitalocean/lon1/", - node_type="/api/v1/nodetype/digitalocean/1gb/") + node_type="/api/v1/nodetype/digitalocean/1gb/") cluster.save() self.assertTrue(cluster.deploy()) result = json.loads(json.dumps(cluster.get_all_attributes())) diff --git a/tests/test_nodeprovider.py b/tests/test_nodeprovider.py index 357b752..afa28ad 100644 --- a/tests/test_nodeprovider.py +++ b/tests/test_nodeprovider.py @@ -2,7 +2,10 @@ import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from .fake_api import * diff --git a/tests/test_noderegion.py b/tests/test_noderegion.py index bc40a7e..add65a9 100644 --- a/tests/test_noderegion.py +++ b/tests/test_noderegion.py @@ -2,7 +2,10 @@ import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from .fake_api import * diff --git a/tests/test_nodetype.py b/tests/test_nodetype.py index 1cfc4f3..38cae1e 100644 --- a/tests/test_nodetype.py +++ b/tests/test_nodetype.py @@ -2,7 +2,10 @@ import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from .fake_api import * diff --git a/tests/test_repository.py b/tests/test_repository.py index 82cb209..ca4aa84 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -2,7 +2,10 @@ import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from .fake_api import * diff --git a/tests/test_service.py b/tests/test_service.py index e210625..47d86cc 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -2,7 +2,10 @@ import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from .fake_api import * diff --git a/tests/test_utils.py b/tests/test_utils.py index f962b6f..ca9140f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,9 @@ import unittest -import unittest.mock as mock +try: + import mock +except ImportError: + import unittest.mock as mock import dockercloud from dockercloud.api.exceptions import ObjectNotFound, ApiError, NonUniqueIdentifier From 2c4ac392ef879151d94ac7070a8fda3676d966a7 Mon Sep 17 00:00:00 2001 From: Eugene Khashin Date: Tue, 3 Jan 2017 03:17:38 +0300 Subject: [PATCH 2/6] reconnection config --- .gitignore | 3 ++ README.md | 14 ++++++++ dockercloud/__init__.py | 2 ++ dockercloud/api/http.py | 12 +++++-- tests/test_http_reconnection.py | 63 +++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 tests/test_http_reconnection.py diff --git a/.gitignore b/.gitignore index 7d0d1f4..905d844 100644 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,7 @@ target/ # IDE .idea/ +# vim +*.swp + venv/ diff --git a/README.md b/README.md index aea721e..e704500 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,20 @@ The authentication can be configured in the following ways: export DOCKERCLOUD_USER=username export DOCKERCLOUD_APIKEY=apikey +## Optional parameters + +You may set the reconnection interval (Integer, in seconds) using the variable DOCKERCLOUD_RECONNECTION_INTERVAL: + + export DOCKERCLOUD_RECONNECTION_INTERVAL=240 + +Session uses a socket that may be closed by some peer. To prevent the "Read timed out" issue you should use this option. + +Possible values: + +* `-1` (by default) means no reconnect (as usually it works) +* `0` means reconnect on each request +* any positive value means that the connection will be reopened if the time diff between last 2 requests is more than that value + ## Namespace To support teams and orgs, you can specify the namespace in the following ways: diff --git a/dockercloud/__init__.py b/dockercloud/__init__.py index 8bb6aa5..d4df62c 100644 --- a/dockercloud/__init__.py +++ b/dockercloud/__init__.py @@ -40,6 +40,8 @@ namespace = os.environ.get('DOCKERCLOUD_NAMESPACE') +reconnection_interval = int(os.environ.get('DOCKERCLOUD_RECONNECTION_INTERVAL', '-1')) # in seconds, if the connection is inactive more than that value it will be recreated + user_agent = None logging.basicConfig() diff --git a/dockercloud/api/http.py b/dockercloud/api/http.py index f1d0696..f255dc8 100644 --- a/dockercloud/api/http.py +++ b/dockercloud/api/http.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import logging +import time from requests import Request, Session from requests import utils @@ -12,9 +13,14 @@ logger = logging.getLogger("python-dockercloud") global_session = Session() - - -def get_session(): +last_connection_time = time.time() + +def get_session(time=time): + if (dockercloud.reconnection_interval >= 0): + global last_connection_time + if (time.time() - last_connection_time > dockercloud.reconnection_interval): + new_session() + last_connection_time = time.time() return global_session diff --git a/tests/test_http_reconnection.py b/tests/test_http_reconnection.py new file mode 100644 index 0000000..917a4da --- /dev/null +++ b/tests/test_http_reconnection.py @@ -0,0 +1,63 @@ +import unittest +import time +import dockercloud +from dockercloud.api import http + +_reconnection_interval = None + +class FakeTime(object): + def __init__(self, val=None): + self.val = val + def time(self): + return self.val + +class SessionReconnectionTestCase(unittest.TestCase): + # a few helpers + def setUp(self): + global _reconnection_interval + global _last_connection_time + _reconnection_interval = dockercloud.reconnection_interval + _last_connection_time = http.last_connection_time + http.last_connection_time = 0 + + def tearDown(self): + global old_reconnection_interval + global _last_connection_time + dockercloud.reconnection_interval = _reconnection_interval + dockercloud.http = _last_connection_time + # + + def test_logic_without_interval(self): + dockercloud.reconnection_interval = None + session1 = http.get_session() + session2 = http.get_session() + self.assertEqual(id(session1), id(session2)) + + def test_logic_with_negative_interval(self): + dockercloud.reconnection_interval = -1 + session1 = http.get_session() + session2 = http.get_session() + self.assertEqual(id(session1), id(session2)) + + def test_logic_with_zero_interval(self): + dockercloud.reconnection_interval = 0 + session1 = http.get_session() + session2 = http.get_session() + self.assertNotEqual(id(session1), id(session2)) + + def test_logic_with_positive_interval(self): + dockercloud.reconnection_interval = 30 + + # diff is less than 30 secs + session1 = http.get_session(FakeTime(0)) + session2 = http.get_session(FakeTime(10)) + self.assertEqual(id(session1), id(session2)) + + # diff is equal to 30 secs + session3 = http.get_session(FakeTime(40)) + self.assertEqual(id(session2), id(session3)) + + # diff is more that 30 secs + session4 = http.get_session(FakeTime(71)) + self.assertNotEqual(id(session3), id(session4)) + From ee63f3badb981e1c7b1c0a5781deeb4a2c6d07dd Mon Sep 17 00:00:00 2001 From: TortueMat Date: Wed, 11 Jan 2017 21:51:17 -0500 Subject: [PATCH 3/6] Update dockercloud/api/events.py :: Add signal interrupt on 'run_forever' --- dockercloud/api/events.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/dockercloud/api/events.py b/dockercloud/api/events.py index 4440f72..9f31b5a 100644 --- a/dockercloud/api/events.py +++ b/dockercloud/api/events.py @@ -2,6 +2,7 @@ import json import logging +import signal import websocket @@ -41,15 +42,24 @@ def _on_error(self, ws, e): super(self.__class__, self)._on_error(ws, e) + + def _on_stop(self, signal, frame): + self.ws.close() + self.run_forever_flag = not self.run_forever_flag + def run_forever(self, *args, **kwargs): - while True: + + self.run_forever_flag = True + while self.run_forever_flag: if self.auth_error: self.auth_error = False raise AuthError("Not Authorized") - ws = websocket.WebSocketApp(self.url, header=self.header, + self.ws = websocket.WebSocketApp(self.url, header=self.header, on_open=self._on_open, on_message=self._on_message, on_error=self._on_error, on_close=self._on_close) - ws.run_forever(ping_interval=10, ping_timeout=5, *args, **kwargs) + signal.signal(signal.SIGINT, self._on_stop) + self.ws.run_forever(ping_interval=10, ping_timeout=5, *args, **kwargs) + From 8d7e23a07fd302cd561dbf693a620b63bcf0753a Mon Sep 17 00:00:00 2001 From: Sebastian Gumprich Date: Tue, 28 Mar 2017 19:01:45 +0200 Subject: [PATCH 4/6] exclude tests in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2ab4865..1b6b899 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ def find_version(*file_paths): setup( name="python-dockercloud", version=find_version('dockercloud', '__init__.py'), - packages=find_packages(), + packages=find_packages(exclude=['tests']), install_requires=requirements, tests_require=test_requirements, provides=['docker'], From 2d86f5c371dfdebdf5655201e5df0bd61d1015ff Mon Sep 17 00:00:00 2001 From: tifayuki Date: Tue, 11 Apr 2017 16:32:30 -0700 Subject: [PATCH 5/6] add timeout for api call --- dockercloud/__init__.py | 6 +++++- dockercloud/api/events.py | 10 ++++------ dockercloud/api/http.py | 8 +++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/dockercloud/__init__.py b/dockercloud/__init__.py index d4df62c..32042f4 100644 --- a/dockercloud/__init__.py +++ b/dockercloud/__init__.py @@ -40,10 +40,14 @@ namespace = os.environ.get('DOCKERCLOUD_NAMESPACE') -reconnection_interval = int(os.environ.get('DOCKERCLOUD_RECONNECTION_INTERVAL', '-1')) # in seconds, if the connection is inactive more than that value it will be recreated +# in seconds, if the connection is inactive more than that value it will be recreated +reconnection_interval = int(os.environ.get('DOCKERCLOUD_RECONNECTION_INTERVAL', '-1')) user_agent = None +# in seconds, make the api call timeout after X seconds, None usually is 15 mins +api_timeout = None + logging.basicConfig() logger = logging.getLogger("python-dockercloud") diff --git a/dockercloud/api/events.py b/dockercloud/api/events.py index 0caba0d..1d1fce9 100644 --- a/dockercloud/api/events.py +++ b/dockercloud/api/events.py @@ -45,7 +45,6 @@ def _on_error(self, ws, e): super(self.__class__, self)._on_error(ws, e) - def _on_stop(self, signal, frame): self.ws.close() self.run_forever_flag = not self.run_forever_flag @@ -59,10 +58,9 @@ def run_forever(self, *args, **kwargs): raise AuthError("Not Authorized") self.ws = websocket.WebSocketApp(self.url, header=self.header, - on_open=self._on_open, - on_message=self._on_message, - on_error=self._on_error, - on_close=self._on_close) + on_open=self._on_open, + on_message=self._on_message, + on_error=self._on_error, + on_close=self._on_close) signal.signal(signal.SIGINT, self._on_stop) self.ws.run_forever(ping_interval=10, ping_timeout=5, *args, **kwargs) - diff --git a/dockercloud/api/http.py b/dockercloud/api/http.py index f255dc8..b87b154 100644 --- a/dockercloud/api/http.py +++ b/dockercloud/api/http.py @@ -15,6 +15,7 @@ global_session = Session() last_connection_time = time.time() + def get_session(time=time): if (dockercloud.reconnection_interval >= 0): global last_connection_time @@ -61,7 +62,12 @@ def send_request(method, path, inject_header=True, **kwargs): # make the request req = s.prepare_request(request) logger.info("Prepared Request: %s, %s, %s, %s" % (req.method, req.url, req.headers, kwargs)) - response = s.send(req, **kw_args) + + if dockercloud.api_timeout: + response = s.send(req, timeout=dockercloud.api_timeout, **kw_args) + else: + response = s.send(req, **kw_args) + status_code = getattr(response, 'status_code', None) logger.info("Response: Status %s, %s, %s" % (str(status_code), response.headers, response.text)) From 362a27c43a1c23f0792dca88c3c79f4a9c0d9280 Mon Sep 17 00:00:00 2001 From: tifayuki Date: Tue, 11 Apr 2017 16:40:05 -0700 Subject: [PATCH 6/6] bump version --- dockercloud/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockercloud/__init__.py b/dockercloud/__init__.py index 32042f4..b18077c 100644 --- a/dockercloud/__init__.py +++ b/dockercloud/__init__.py @@ -25,7 +25,7 @@ from dockercloud.api.events import Events from dockercloud.api.nodeaz import AZ -__version__ = '1.0.9' +__version__ = '1.0.10' dockercloud_auth = os.environ.get('DOCKERCLOUD_AUTH') basic_auth = auth.load_from_file("~/.docker/config.json")