diff --git a/code/daemon/dependencies/cloudfiles/COPYING b/code/daemon/dependencies/cloudfiles/COPYING new file mode 100644 index 0000000..bb367c4 --- /dev/null +++ b/code/daemon/dependencies/cloudfiles/COPYING @@ -0,0 +1,26 @@ +Unless otherwise noted, all files are released under the MIT license, +exceptions contain licensing information in them. + + Copyright (C) 2008 Rackspace US, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Except as contained in this notice, the name of Rackspace US, Inc. shall not +be used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization from Rackspace US, Inc. diff --git a/code/daemon/dependencies/cloudfiles/__init__.py b/code/daemon/dependencies/cloudfiles/__init__.py new file mode 100644 index 0000000..3548cc0 --- /dev/null +++ b/code/daemon/dependencies/cloudfiles/__init__.py @@ -0,0 +1,79 @@ +""" +Cloud Files python client API. + +Working with result sets: + + >>> import cloudfiles + >>> # conn = cloudfiles.get_connection(username='jsmith', api_key='1234567890') + >>> conn = cloudfiles.get_connection('jsmith', '1234567890') + >>> containers = conn.get_all_containers() + >>> type(containers) + + >>> len(containers) + 2 + >>> for container in containers: + >>> print container.name + fruit + vegitables + >>> print container[0].name + fruit + >>> fruit_container = container[0] + >>> objects = fruit_container.get_objects() + >>> for storage_object in objects: + >>> print storage_object.name + apple + orange + bannana + >>> + +Creating Containers and adding Objects to them: + + >>> pic_container = conn.create_container('pictures') + >>> my_dog = pic_container.create_object('fido.jpg') + >>> my_dog.load_from_file('images/IMG-0234.jpg') + >>> text_obj = pic_container.create_object('sample.txt') + >>> text_obj.write('This is not the object you are looking for.\\n') + >>> text_obj.read() + 'This is not the object you are looking for.' + +Object instances support streaming through the use of a generator: + + >>> deb_iso = pic_container.get_object('debian-40r3-i386-netinst.iso') + >>> f = open('/tmp/debian.iso', 'w') + >>> for chunk in deb_iso.stream(): + .. f.write(chunk) + >>> f.close() + +Marking a Container as CDN-enabled/public with a TTL of 30 days + + >>> pic_container.make_public(2592000) + >>> pic_container.public_uri() + 'http://c0001234.cdn.cloudfiles.rackspacecloud.com' + >>> my_dog.public_uri() + 'http://c0001234.cdn.cloudfiles.rackspacecloud.com/fido.jpg' + +Set the logs retention on CDN-enabled/public Container + + >>> pic_container.log_retention(True) + +See COPYING for license information. +""" + +from cloudfiles.connection import Connection, ConnectionPool +from cloudfiles.container import Container +from cloudfiles.storage_object import Object +from cloudfiles.consts import __version__ + +def get_connection(*args, **kwargs): + """ + Helper function for creating connection instances. + + @type username: string + @param username: a Mosso username + @type api_key: string + @param api_key: a Mosso API key + @rtype: L{Connection} + @returns: a connection object + """ + return Connection(*args, **kwargs) + diff --git a/code/daemon/dependencies/cloudfiles/__init__.pyc b/code/daemon/dependencies/cloudfiles/__init__.pyc new file mode 100644 index 0000000..7df2bda Binary files /dev/null and b/code/daemon/dependencies/cloudfiles/__init__.pyc differ diff --git a/code/daemon/dependencies/cloudfiles/authentication.py b/code/daemon/dependencies/cloudfiles/authentication.py new file mode 100644 index 0000000..6dd2aa3 --- /dev/null +++ b/code/daemon/dependencies/cloudfiles/authentication.py @@ -0,0 +1,89 @@ +""" +authentication operations + +Authentication instances are used to interact with the remote +authentication service, retreiving storage system routing information +and session tokens. + +See COPYING for license information. +""" + +import urllib +from httplib import HTTPSConnection, HTTPConnection, HTTPException +from utils import parse_url +from errors import ResponseError, AuthenticationError, AuthenticationFailed +from consts import user_agent, default_authurl + +class BaseAuthentication(object): + """ + The base authentication class from which all others inherit. + """ + def __init__(self, username, api_key, authurl=default_authurl): + self.authurl = authurl + self.headers = dict() + self.headers['x-auth-user'] = username + self.headers['x-auth-key'] = api_key + self.headers['User-Agent'] = user_agent + (self.host, self.port, self.uri, self.is_ssl) = parse_url(self.authurl) + self.conn_class = self.is_ssl and HTTPSConnection or HTTPConnection + + def authenticate(self): + """ + Initiates authentication with the remote service and returns a + two-tuple containing the storage system URL and session token. + + Note: This is a dummy method from the base class. It must be + overridden by sub-classes. + """ + return (None, None, None) + +class MockAuthentication(BaseAuthentication): + """ + Mock authentication class for testing + """ + def authenticate(self): + return ('http://localhost/v1/account', None, 'xxxxxxxxx') + +class Authentication(BaseAuthentication): + """ + Authentication, routing, and session token management. + """ + def authenticate(self): + """ + Initiates authentication with the remote service and returns a + two-tuple containing the storage system URL and session token. + """ + conn = self.conn_class(self.host, self.port) + conn.request('GET', self.authurl, '', self.headers) + response = conn.getresponse() + buff = response.read() + + # A status code of 401 indicates that the supplied credentials + # were not accepted by the authentication service. + if response.status == 401: + raise AuthenticationFailed() + + if response.status != 204: + raise ResponseError(response.status, response.reason) + + storage_url = cdn_url = auth_token = None + + for hdr in response.getheaders(): + if hdr[0].lower() == "x-storage-url": + storage_url = hdr[1] + if hdr[0].lower() == "x-cdn-management-url": + cdn_url = hdr[1] + if hdr[0].lower() == "x-storage-token": + auth_token = hdr[1] + if hdr[0].lower() == "x-auth-token": + auth_token = hdr[1] + + conn.close() + + if not (auth_token and storage_url): + raise AuthenticationError("Invalid response from the " \ + "authentication service.") + + return (storage_url, cdn_url, auth_token) + +# vim:set ai ts=4 sw=4 tw=0 expandtab: diff --git a/code/daemon/dependencies/cloudfiles/authentication.pyc b/code/daemon/dependencies/cloudfiles/authentication.pyc new file mode 100644 index 0000000..0551994 Binary files /dev/null and b/code/daemon/dependencies/cloudfiles/authentication.pyc differ diff --git a/code/daemon/dependencies/cloudfiles/connection.py b/code/daemon/dependencies/cloudfiles/connection.py new file mode 100644 index 0000000..1e285b0 --- /dev/null +++ b/code/daemon/dependencies/cloudfiles/connection.py @@ -0,0 +1,441 @@ +""" +connection operations + +Connection instances are used to communicate with the remote service at +the account level creating, listing and deleting Containers, and returning +Container instances. + +See COPYING for license information. +""" + +import socket +from urllib import quote +from httplib import HTTPSConnection, HTTPConnection, HTTPException +from container import Container, ContainerResults +from utils import parse_url +from errors import ResponseError, NoSuchContainer, ContainerNotEmpty, \ + InvalidContainerName, CDNNotEnabled +from Queue import Queue, Empty, Full +from time import time +import consts +from authentication import Authentication +from fjson import json_loads + +# Because HTTPResponse objects *have* to have read() called on them +# before they can be used again ... +# pylint: disable-msg=W0612 + +class Connection(object): + """ + Manages the connection to the storage system and serves as a factory + for Container instances. + + @undocumented: cdn_connect + @undocumented: http_connect + @undocumented: cdn_request + @undocumented: make_request + @undocumented: _check_container_name + """ + def __init__(self, username=None, api_key=None, **kwargs): + """ + Accepts keyword arguments for Mosso username and api key. + Optionally, you can omit these keywords and supply an + Authentication object using the auth keyword. + + @type username: str + @param username: a Mosso username + @type api_key: str + @param api_key: a Mosso API key + """ + self.cdn_enabled = False + self.cdn_args = None + self.connection_args = None + self.cdn_connection = None + self.connection = None + self.token = None + self.debuglevel = int(kwargs.get('debuglevel', 0)) + socket.setdefaulttimeout = int(kwargs.get('timeout', 5)) + self.auth = kwargs.has_key('auth') and kwargs['auth'] or None + + if not self.auth: + authurl = kwargs.get('authurl', consts.default_authurl) + if username and api_key and authurl: + self.auth = Authentication(username, api_key, authurl) + else: + raise TypeError("Incorrect or invalid arguments supplied") + + self._authenticate() + + def _authenticate(self): + """ + Authenticate and setup this instance with the values returned. + """ + (url, self.cdn_url, self.token) = self.auth.authenticate() + self.connection_args = parse_url(url) + self.conn_class = self.connection_args[3] and HTTPSConnection or \ + HTTPConnection + self.http_connect() + if self.cdn_url: + self.cdn_connect() + + def cdn_connect(self): + """ + Setup the http connection instance for the CDN service. + """ + (host, port, cdn_uri, is_ssl) = parse_url(self.cdn_url) + conn_class = is_ssl and HTTPSConnection or HTTPConnection + self.cdn_connection = conn_class(host, port) + self.cdn_enabled = True + + def http_connect(self): + """ + Setup the http connection instance. + """ + (host, port, self.uri, is_ssl) = self.connection_args + self.connection = self.conn_class(host, port=port) + self.connection.set_debuglevel(self.debuglevel) + + def cdn_request(self, method, path=[], data='', hdrs=None): + """ + Given a method (i.e. GET, PUT, POST, etc), a path, data, header and + metadata dicts, performs an http request against the CDN service. + """ + if not self.cdn_enabled: + raise CDNNotEnabled() + + path = '/%s/%s' % \ + (self.uri.rstrip('/'), '/'.join([quote(i) for i in path])) + headers = {'Content-Length': len(data), 'User-Agent': consts.user_agent, + 'X-Auth-Token': self.token} + if isinstance(hdrs, dict): + headers.update(hdrs) + + # Send the request + self.cdn_connection.request(method, path, data, headers) + + def retry_request(): + '''Re-connect and re-try a failed request once''' + self.cdn_connect() + self.cdn_connection.request(method, path, data, headers) + return self.cdn_connection.getresponse() + + try: + response = self.cdn_connection.getresponse() + except HTTPException: + response = retry_request() + + if response.status == 401: + self._authenticate() + response = retry_request() + + return response + + + def make_request(self, method, path=[], data='', hdrs=None, parms=None): + """ + Given a method (i.e. GET, PUT, POST, etc), a path, data, header and + metadata dicts, and an optional dictionary of query parameters, + performs an http request. + """ + path = '/%s/%s' % \ + (self.uri.rstrip('/'), '/'.join([quote(i) for i in path])) + + if isinstance(parms, dict) and parms: + query_args = \ + ['%s=%s' % (quote(x),quote(str(y))) for (x,y) in parms.items()] + path = '%s?%s' % (path, '&'.join(query_args)) + + headers = {'Content-Length': len(data), 'User-Agent': consts.user_agent, + 'X-Auth-Token': self.token} + isinstance(hdrs, dict) and headers.update(hdrs) + + def retry_request(): + '''Re-connect and re-try a failed request once''' + self.http_connect() + self.connection.request(method, path, data, headers) + return self.connection.getresponse() + + try: + self.connection.request(method, path, data, headers) + response = self.connection.getresponse() + except HTTPException: + response = retry_request() + + if response.status == 401: + self._authenticate() + response = retry_request() + + return response + + def get_info(self): + """ + Return tuple for number of containers and total bytes in the account + + >>> connection.get_info() + (5, 2309749) + + @rtype: tuple + @return: a tuple containing the number of containers and total bytes + used by the account + """ + response = self.make_request('HEAD') + count = size = None + for hdr in response.getheaders(): + if hdr[0].lower() == 'x-account-container-count': + try: + count = int(hdr[1]) + except ValueError: + count = 0 + if hdr[0].lower() == 'x-account-bytes-used': + try: + size = int(hdr[1]) + except ValueError: + size = 0 + buff = response.read() + if (response.status < 200) or (response.status > 299): + raise ResponseError(response.status, response.reason) + return (count, size) + + def _check_container_name(self, container_name): + if not container_name or \ + '/' in container_name or \ + len(container_name) > consts.container_name_limit: + raise InvalidContainerName(container_name) + + def create_container(self, container_name): + """ + Given a container name, returns a L{Container} item, creating a new + Container if one does not already exist. + + >>> connection.create_container('new_container') + + + @param container_name: name of the container to create + @type container_name: str + @rtype: L{Container} + @return: an object representing the newly created container + """ + self._check_container_name(container_name) + + response = self.make_request('PUT', [container_name]) + buff = response.read() + if (response.status < 200) or (response.status > 299): + raise ResponseError(response.status, response.reason) + return Container(self, container_name) + + def delete_container(self, container_name): + """ + Given a container name, delete it. + + >>> connection.delete_container('old_container') + + @param container_name: name of the container to delete + @type container_name: str + """ + if isinstance(container_name, Container): + container_name = container_name.name + self._check_container_name(container_name) + + response = self.make_request('DELETE', [container_name]) + buff = response.read() + + if (response.status == 409): + raise ContainerNotEmpty(container_name) + elif (response.status < 200) or (response.status > 299): + raise ResponseError(response.status, response.reason) + + if self.cdn_enabled: + response = self.cdn_request('POST', [container_name], + hdrs={'X-CDN-Enabled': 'False'}) + + def get_all_containers(self, limit=None, marker=None, **parms): + """ + Returns a Container item result set. + + >>> connection.get_all_containers() + ContainerResults: 4 containers + >>> print ', '.join([container.name for container in + connection.get_all_containers()]) + new_container, old_container, pictures, music + + @rtype: L{ContainerResults} + @return: an iterable set of objects representing all containers on the + account + @param limit: number of results to return, up to 10,000 + @type limit: int + @param marker: return only results whose name is greater than "marker" + @type marker: str + """ + if limit: + parms['limit'] = limit + if marker: + parms['marker'] = marker + return ContainerResults(self, self.list_containers_info(**parms)) + + def get_container(self, container_name): + """ + Return a single Container item for the given Container. + + >>> connection.get_container('old_container') + + >>> container = connection.get_container('old_container') + >>> container.size_used + 23074 + + @param container_name: name of the container to create + @type container_name: str + @rtype: L{Container} + @return: an object representing the container + """ + self._check_container_name(container_name) + + response = self.make_request('HEAD', [container_name]) + count = size = None + for hdr in response.getheaders(): + if hdr[0].lower() == 'x-container-object-count': + try: + count = int(hdr[1]) + except ValueError: + count = 0 + if hdr[0].lower() == 'x-container-bytes-used': + try: + size = int(hdr[1]) + except ValueError: + size = 0 + buff = response.read() + if response.status == 404: + raise NoSuchContainer(container_name) + if (response.status < 200) or (response.status > 299): + raise ResponseError(response.status, response.reason) + return Container(self, container_name, count, size) + + def list_public_containers(self): + """ + Returns a list of containers that have been published to the CDN. + + >>> connection.list_public_containers() + ['container1', 'container2', 'container3'] + + @rtype: list(str) + @return: a list of all CDN-enabled container names as strings + """ + response = self.cdn_request('GET', ['']) + if (response.status < 200) or (response.status > 299): + buff = response.read() + raise ResponseError(response.status, response.reason) + return response.read().splitlines() + + def list_containers_info(self, limit=None, marker=None, **parms): + """ + Returns a list of Containers, including object count and size. + + >>> connection.list_containers_info() + [{u'count': 510, u'bytes': 2081717, u'name': u'new_container'}, + {u'count': 12, u'bytes': 23074, u'name': u'old_container'}, + {u'count': 0, u'bytes': 0, u'name': u'container1'}, + {u'count': 0, u'bytes': 0, u'name': u'container2'}, + {u'count': 0, u'bytes': 0, u'name': u'container3'}, + {u'count': 3, u'bytes': 2306, u'name': u'test'}] + + @rtype: list({"name":"...", "count":..., "bytes":...}) + @return: a list of all container info as dictionaries with the + keys "name", "count", and "bytes" + @param limit: number of results to return, up to 10,000 + @type limit: int + @param marker: return only results whose name is greater than "marker" + @type marker: str + """ + if limit: + parms['limit'] = limit + if marker: + parms['marker'] = marker + parms['format'] = 'json' + response = self.make_request('GET', [''], parms=parms) + if (response.status < 200) or (response.status > 299): + buff = response.read() + raise ResponseError(response.status, response.reason) + return json_loads(response.read()) + + def list_containers(self, limit=None, marker=None, **parms): + """ + Returns a list of Containers. + + >>> connection.list_containers() + ['new_container', + 'old_container', + 'container1', + 'container2', + 'container3', + 'test'] + + @rtype: list(str) + @return: a list of all containers names as strings + @param limit: number of results to return, up to 10,000 + @type limit: int + @param marker: return only results whose name is greater than "marker" + @type marker: str + """ + if limit: + parms['limit'] = limit + if marker: + parms['marker'] = marker + response = self.make_request('GET', [''], parms=parms) + if (response.status < 200) or (response.status > 299): + buff = response.read() + raise ResponseError(response.status, response.reason) + return response.read().splitlines() + + def __getitem__(self, key): + """ + Container objects can be grabbed from a connection using index + syntax. + + >>> container = conn['old_container'] + >>> container.size_used + 23074 + + @rtype: L{Container} + @return: an object representing the container + """ + return self.get_container(key) + +class ConnectionPool(Queue): + """ + A thread-safe connection pool object. + + This component isn't required when using the cloudfiles library, but it may + be useful when building threaded applications. + """ + def __init__(self, username=None, api_key=None, **kwargs): + auth = kwargs.get('auth', None) + self.timeout = kwargs.get('timeout', 5) + self.connargs = {'username': username, 'api_key': api_key} + poolsize = kwargs.get('poolsize', 10) + Queue.__init__(self, poolsize) + + def get(self): + """ + Return a cloudfiles connection object. + + @rtype: L{Connection} + @return: a cloudfiles connection object + """ + try: + (create, connobj) = Queue.get(self, block=0) + except Empty: + connobj = Connection(**self.connargs) + return connobj + + def put(self, connobj): + """ + Place a cloudfiles connection object back into the pool. + + @param connobj: a cloudfiles connection object + @type connobj: L{Connection} + """ + try: + Queue.put(self, (time(), connobj), block=0) + except Full: + del connobj + +# vim:set ai sw=4 ts=4 tw=0 expandtab: diff --git a/code/daemon/dependencies/cloudfiles/connection.pyc b/code/daemon/dependencies/cloudfiles/connection.pyc new file mode 100644 index 0000000..3bcd47f Binary files /dev/null and b/code/daemon/dependencies/cloudfiles/connection.pyc differ diff --git a/code/daemon/dependencies/cloudfiles/consts.py b/code/daemon/dependencies/cloudfiles/consts.py new file mode 100644 index 0000000..ff6637a --- /dev/null +++ b/code/daemon/dependencies/cloudfiles/consts.py @@ -0,0 +1,12 @@ +""" See COPYING for license information. """ + +__version__ = "1.4.0" +user_agent = "python-cloudfiles/%s" % __version__ +default_authurl = 'https://api.mosso.com/auth' +default_cdn_ttl = 86400 +cdn_log_retention = False + +meta_name_limit = 128 +meta_value_limit = 256 +object_name_limit = 1024 +container_name_limit = 256 diff --git a/code/daemon/dependencies/cloudfiles/consts.pyc b/code/daemon/dependencies/cloudfiles/consts.pyc new file mode 100644 index 0000000..195da12 Binary files /dev/null and b/code/daemon/dependencies/cloudfiles/consts.pyc differ diff --git a/code/daemon/dependencies/cloudfiles/container.py b/code/daemon/dependencies/cloudfiles/container.py new file mode 100644 index 0000000..d11453e --- /dev/null +++ b/code/daemon/dependencies/cloudfiles/container.py @@ -0,0 +1,419 @@ +""" +container operations + +Containers are storage compartments where you put your data (objects). +A container is similar to a directory or folder on a conventional filesystem +with the exception that they exist in a flat namespace, you can not create +containers inside of containers. + +See COPYING for license information. +""" + +from storage_object import Object, ObjectResults +from errors import ResponseError, InvalidContainerName, InvalidObjectName, \ + ContainerNotPublic, CDNNotEnabled +from utils import requires_name +import consts +from fjson import json_loads + +# Because HTTPResponse objects *have* to have read() called on them +# before they can be used again ... +# pylint: disable-msg=W0612 + +class Container(object): + """ + Container object and Object instance factory. + + If your account has the feature enabled, containers can be publically + shared over a global content delivery network. + + @ivar name: the container's name (generally treated as read-only) + @type name: str + @ivar object_count: the number of objects in this container (cached) + @type object_count: number + @ivar size_used: the sum of the sizes of all objects in this container + (cached) + @type size_used: number + @ivar cdn_ttl: the time-to-live of the CDN's public cache of this container + (cached, use make_public to alter) + @type cdn_ttl: number + @ivar cdn_log_retention: retention of the logs in the container. + @type cdn_log_retention: bool + + @undocumented: _fetch_cdn_data + @undocumented: _list_objects_raw + """ + def __set_name(self, name): + # slashes make for invalid names + if isinstance(name, (str, unicode)) and \ + ('/' in name or len(name) > consts.container_name_limit): + raise InvalidContainerName(name) + self._name = name + + name = property(fget=lambda self: self._name, fset=__set_name, + doc="the name of the container (read-only)") + + def __init__(self, connection=None, name=None, count=None, size=None): + """ + Containers will rarely if ever need to be instantiated directly by the + user. + + Instead, use the L{create_container}, + L{get_container}, + L{list_containers} and + other methods on a valid Connection object. + """ + self._name = None + self.name = name + self.conn = connection + self.object_count = count + self.size_used = size + self.cdn_uri = None + self.cdn_ttl = None + self.cdn_log_retention = None + if connection.cdn_enabled: + self._fetch_cdn_data() + + @requires_name(InvalidContainerName) + def _fetch_cdn_data(self): + """ + Fetch the object's CDN data from the CDN service + """ + response = self.conn.cdn_request('HEAD', [self.name]) + if (response.status >= 200) and (response.status < 300): + for hdr in response.getheaders(): + if hdr[0].lower() == 'x-cdn-uri': + self.cdn_uri = hdr[1] + if hdr[0].lower() == 'x-ttl': + self.cdn_ttl = int(hdr[1]) + if hdr[0].lower() == 'x-log-retention': + self.cdn_log_retention = hdr[1] == "True" and True or False + + @requires_name(InvalidContainerName) + def make_public(self, ttl=consts.default_cdn_ttl): + """ + Either publishes the current container to the CDN or updates its + CDN attributes. Requires CDN be enabled on the account. + + >>> container.make_public(ttl=604800) # expire in 1 week + + @param ttl: cache duration in seconds of the CDN server + @type ttl: number + """ + if not self.conn.cdn_enabled: + raise CDNNotEnabled() + if self.cdn_uri: + request_method = 'POST' + else: + request_method = 'PUT' + hdrs = {'X-TTL': str(ttl), 'X-CDN-Enabled': 'True'} + response = self.conn.cdn_request(request_method, [self.name], hdrs=hdrs) + if (response.status < 200) or (response.status >= 300): + raise ResponseError(response.status, response.reason) + self.cdn_ttl = ttl + for hdr in response.getheaders(): + if hdr[0].lower() == 'x-cdn-uri': + self.cdn_uri = hdr[1] + + @requires_name(InvalidContainerName) + def make_private(self): + """ + Disables CDN access to this container. + It may continue to be available until its TTL expires. + + >>> container.make_private() + """ + if not self.conn.cdn_enabled: + raise CDNNotEnabled() + hdrs = {'X-CDN-Enabled': 'False'} + self.cdn_uri = None + response = self.conn.cdn_request('POST', [self.name], hdrs=hdrs) + if (response.status < 200) or (response.status >= 300): + raise ResponseError(response.status, response.reason) + + @requires_name(InvalidContainerName) + def log_retention(self, log_retention=consts.cdn_log_retention): + """ + Enable CDN log retention on the container. If enabled logs will be + periodically (at unpredictable intervals) compressed and uploaded to + a ".CDN_ACCESS_LOGS" container in the form of + "container_name.YYYYMMDDHH-XXXX.gz". Requires CDN be enabled on the + account. + + >>> container.log_retention(True) + + @param log_retention: Enable or disable logs retention. + @type log_retention: bool + """ + if not self.conn.cdn_enabled: + raise CDNNotEnabled() + + hdrs = {'X-Log-Retention': log_retention} + response = self.conn.cdn_request('POST', [self.name], hdrs=hdrs) + if (response.status < 200) or (response.status >= 300): + raise ResponseError(response.status, response.reason) + + self.cdn_log_retention = log_retention + + def is_public(self): + """ + Returns a boolean indicating whether or not this container is + publically accessible via the CDN. + + >>> container.is_public() + False + >>> container.make_public() + >>> container.is_public() + True + + @rtype: bool + @return: whether or not this container is published to the CDN + """ + if not self.conn.cdn_enabled: + raise CDNNotEnabled() + return self.cdn_uri is not None + + @requires_name(InvalidContainerName) + def public_uri(self): + """ + Return the URI for this container, if it is publically + accessible via the CDN. + + >>> connection['container1'].public_uri() + 'http://c00061.cdn.cloudfiles.rackspacecloud.com' + + @rtype: str + @return: the public URI for this container + """ + if not self.is_public(): + raise ContainerNotPublic() + return self.cdn_uri + + @requires_name(InvalidContainerName) + def create_object(self, object_name): + """ + Return an L{Object} instance, creating it if necessary. + + When passed the name of an existing object, this method will + return an instance of that object, otherwise it will create a + new one. + + >>> container.create_object('new_object') + + >>> obj = container.create_object('new_object') + >>> obj.name + 'new_object' + + @type object_name: str + @param object_name: the name of the object to create + @rtype: L{Object} + @return: an object representing the newly created storage object + """ + return Object(self, object_name) + + @requires_name(InvalidContainerName) + def get_objects(self, prefix=None, limit=None, marker=None, + path=None, **parms): + """ + Return a result set of all Objects in the Container. + + Keyword arguments are treated as HTTP query parameters and can + be used to limit the result set (see the API documentation). + + >>> container.get_objects(limit=2) + ObjectResults: 2 objects + >>> for obj in container.get_objects(): + ... print obj.name + new_object + old_object + + @param prefix: filter the results using this prefix + @type prefix: str + @param limit: return the first "limit" objects found + @type limit: int + @param marker: return objects whose names are greater than "marker" + @type marker: str + @param path: return all objects in "path" + @type path: str + + @rtype: L{ObjectResults} + @return: an iterable collection of all storage objects in the container + """ + return ObjectResults(self, self.list_objects_info( + prefix, limit, marker, path, **parms)) + + @requires_name(InvalidContainerName) + def get_object(self, object_name): + """ + Return an L{Object} instance for an existing storage object. + + If an object with a name matching object_name does not exist + then a L{NoSuchObject} exception is raised. + + >>> obj = container.get_object('old_object') + >>> obj.name + 'old_object' + + @param object_name: the name of the object to retrieve + @type object_name: str + @rtype: L{Object} + @return: an Object representing the storage object requested + """ + return Object(self, object_name, force_exists=True) + + @requires_name(InvalidContainerName) + def list_objects_info(self, prefix=None, limit=None, marker=None, + path=None, **parms): + """ + Return information about all objects in the Container. + + Keyword arguments are treated as HTTP query parameters and can + be used limit the result set (see the API documentation). + + >>> conn['container1'].list_objects_info(limit=2) + [{u'bytes': 4820, + u'content_type': u'application/octet-stream', + u'hash': u'db8b55400b91ce34d800e126e37886f8', + u'last_modified': u'2008-11-05T00:56:00.406565', + u'name': u'new_object'}, + {u'bytes': 1896, + u'content_type': u'application/octet-stream', + u'hash': u'1b49df63db7bc97cd2a10e391e102d4b', + u'last_modified': u'2008-11-05T00:56:27.508729', + u'name': u'old_object'}] + + @param prefix: filter the results using this prefix + @type prefix: str + @param limit: return the first "limit" objects found + @type limit: int + @param marker: return objects with names greater than "marker" + @type marker: str + @param path: return all objects in "path" + @type path: str + + @rtype: list({"name":"...", "hash":..., "size":..., "type":...}) + @return: a list of all container info as dictionaries with the + keys "name", "hash", "size", and "type" + """ + parms['format'] = 'json' + resp = self._list_objects_raw( + prefix, limit, marker, path, **parms) + return json_loads(resp) + + @requires_name(InvalidContainerName) + def list_objects(self, prefix=None, limit=None, marker=None, + path=None, **parms): + """ + Return names of all L{Object}s in the L{Container}. + + Keyword arguments are treated as HTTP query parameters and can + be used to limit the result set (see the API documentation). + + >>> container.list_objects() + ['new_object', 'old_object'] + + @param prefix: filter the results using this prefix + @type prefix: str + @param limit: return the first "limit" objects found + @type limit: int + @param marker: return objects with names greater than "marker" + @type marker: str + @param path: return all objects in "path" + @type path: str + + @rtype: list(str) + @return: a list of all container names + """ + resp = self._list_objects_raw(prefix=prefix, limit=limit, + marker=marker, path=path, **parms) + return resp.splitlines() + + @requires_name(InvalidContainerName) + def _list_objects_raw(self, prefix=None, limit=None, marker=None, + path=None, **parms): + """ + Returns a chunk list of storage object info. + """ + if prefix: parms['prefix'] = prefix + if limit: parms['limit'] = limit + if marker: parms['marker'] = marker + if not path is None: parms['path'] = path # empty strings are valid + response = self.conn.make_request('GET', [self.name], parms=parms) + if (response.status < 200) or (response.status > 299): + buff = response.read() + raise ResponseError(response.status, response.reason) + return response.read() + + def __getitem__(self, key): + return self.get_object(key) + + def __str__(self): + return self.name + + @requires_name(InvalidContainerName) + def delete_object(self, object_name): + """ + Permanently remove a storage object. + + >>> container.list_objects() + ['new_object', 'old_object'] + >>> container.delete_object('old_object') + >>> container.list_objects() + ['new_object'] + + @param object_name: the name of the object to retrieve + @type object_name: str + """ + if isinstance(object_name, Object): + object_name = object_name.name + if not object_name: + raise InvalidObjectName(object_name) + response = self.conn.make_request('DELETE', [self.name, object_name]) + if (response.status < 200) or (response.status > 299): + buff = response.read() + raise ResponseError(response.status, response.reason) + buff = response.read() + +class ContainerResults(object): + """ + An iterable results set object for Containers. + + This class implements dictionary- and list-like interfaces. + """ + def __init__(self, conn, containers=list()): + self._containers = containers + self._names = [k['name'] for k in containers] + self.conn = conn + + def __getitem__(self, key): + return Container(self.conn, + self._containers[key]['name'], + self._containers[key]['count'], + self._containers[key]['bytes']) + + def __getslice__(self, i, j): + return [Container(self.conn, k['name'], k['count'], k['size']) for k in self._containers[i:j] ] + + def __contains__(self, item): + return item in self._names + + def __repr__(self): + return 'ContainerResults: %s containers' % len(self._containers) + __str__ = __repr__ + + def __len__(self): + return len(self._containers) + + def index(self, value, *args): + """ + returns an integer for the first index of value + """ + return self._names.index(value, *args) + + def count(self, value): + """ + returns the number of occurrences of value + """ + return self._names.count(value) + +# vim:set ai sw=4 ts=4 tw=0 expandtab: diff --git a/code/daemon/dependencies/cloudfiles/container.pyc b/code/daemon/dependencies/cloudfiles/container.pyc new file mode 100644 index 0000000..282b487 Binary files /dev/null and b/code/daemon/dependencies/cloudfiles/container.pyc differ diff --git a/code/daemon/dependencies/cloudfiles/errors.py b/code/daemon/dependencies/cloudfiles/errors.py new file mode 100644 index 0000000..48f4efd --- /dev/null +++ b/code/daemon/dependencies/cloudfiles/errors.py @@ -0,0 +1,112 @@ +""" +exception classes + +See COPYING for license information. +""" + +class ResponseError(Exception): + """ + Raised when the remote service returns an error. + """ + def __init__(self, status, reason): + self.status = status + self.reason = reason + Exception.__init__(self) + + def __str__(self): + return '%d: %s' % (self.status, self.reason) + + def __repr__(self): + return '%d: %s' % (self.status, self.reason) + +class NoSuchContainer(Exception): + """ + Raised on a non-existent Container. + """ + pass + +class NoSuchObject(Exception): + """ + Raised on a non-existent Object. + """ + pass + +class ContainerNotEmpty(Exception): + """ + Raised when attempting to delete a Container that still contains Objects. + """ + def __init__(self, container_name): + self.container_name = container_name + + def __str__(self): + return "Cannot delete non-empty Container %s" % self.container_name + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, self.container_name) + +class InvalidContainerName(Exception): + """ + Raised for invalid storage container names. + """ + pass + +class InvalidObjectName(Exception): + """ + Raised for invalid storage object names. + """ + pass + +class InvalidMetaName(Exception): + """ + Raised for invalid metadata names. + """ + pass + +class InvalidMetaValue(Exception): + """ + Raised for invalid metadata value. + """ + pass + +class InvalidUrl(Exception): + """ + Not a valid url for use with this software. + """ + pass + +class InvalidObjectSize(Exception): + """ + Not a valid storage_object size attribute. + """ + pass + +class IncompleteSend(Exception): + """ + Raised when there is a insufficient amount of data to send. + """ + pass + +class ContainerNotPublic(Exception): + """ + Raised when public features of a non-public container are accessed. + """ + pass + +class CDNNotEnabled(Exception): + """ + CDN is not enabled for this account. + """ + pass + +class AuthenticationFailed(Exception): + """ + Raised on a failure to authenticate. + """ + pass + +class AuthenticationError(Exception): + """ + Raised when an unspecified authentication error has occurred. + """ + pass + diff --git a/code/daemon/dependencies/cloudfiles/errors.pyc b/code/daemon/dependencies/cloudfiles/errors.pyc new file mode 100644 index 0000000..3fcd2ac Binary files /dev/null and b/code/daemon/dependencies/cloudfiles/errors.pyc differ diff --git a/code/daemon/dependencies/cloudfiles/fjson.py b/code/daemon/dependencies/cloudfiles/fjson.py new file mode 100644 index 0000000..aed662a --- /dev/null +++ b/code/daemon/dependencies/cloudfiles/fjson.py @@ -0,0 +1,43 @@ +from tokenize import generate_tokens, STRING, NAME, OP +from cStringIO import StringIO +from re import compile, DOTALL + +comments = compile(r'/\*.*\*/|//[^\r\n]*', DOTALL) + +def _loads(string): + ''' + Fairly competent json parser exploiting the python tokenizer and eval() + + _loads(serialized_json) -> object + ''' + try: + res = [] + consts = {'true': True, 'false': False, 'null': None} + string = '(' + comments.sub('', string) + ')' + for type, val, _, _, _ in generate_tokens(StringIO(string).readline): + if (type == OP and val not in '[]{}:,()-') or \ + (type == NAME and val not in consts): + raise AttributeError() + elif type == STRING: + res.append('u') + res.append(val.replace('\\/', '/')) + else: + res.append(val) + return eval(''.join(res), {}, consts) + except: + raise AttributeError() + +# look for a real json parser first +try: + # 2.6 will have a json module in the stdlib + from json import loads as json_loads +except ImportError: + try: + # simplejson is popular and pretty good + from simplejson import loads as json_loads + # fall back on local parser otherwise + except ImportError: + json_loads = _loads + +__all__ = ['json_loads'] + diff --git a/code/daemon/dependencies/cloudfiles/fjson.pyc b/code/daemon/dependencies/cloudfiles/fjson.pyc new file mode 100644 index 0000000..bd86cd5 Binary files /dev/null and b/code/daemon/dependencies/cloudfiles/fjson.pyc differ diff --git a/code/daemon/dependencies/cloudfiles/storage_object.py b/code/daemon/dependencies/cloudfiles/storage_object.py new file mode 100644 index 0000000..92691a9 --- /dev/null +++ b/code/daemon/dependencies/cloudfiles/storage_object.py @@ -0,0 +1,579 @@ +""" +Object operations + +An Object is analogous to a file on a conventional filesystem. You can +read data from, or write data to your Objects. You can also associate +arbitrary metadata with them. + +See COPYING for license information. +""" + +try: + from hashlib import md5 +except ImportError: + from md5 import md5 +import StringIO +import mimetypes +import os +import tempfile + +from urllib import quote +from errors import ResponseError, NoSuchObject, \ + InvalidObjectName, InvalidObjectSize, \ + InvalidMetaName, InvalidMetaValue, \ + IncompleteSend +from socket import timeout +import consts +from utils import requires_name + +# Because HTTPResponse objects *have* to have read() called on them +# before they can be used again ... +# pylint: disable-msg=W0612 + +class Object(object): + """ + Storage data representing an object, (metadata and data). + + @undocumented: _make_headers + @undocumented: _name_check + @undocumented: _initialize + @undocumented: compute_md5sum + @undocumented: __get_conn_for_write + @ivar name: the object's name (generally treat as read-only) + @type name: str + @ivar content_type: the object's content-type (set or read) + @type content_type: str + @ivar metadata: metadata associated with the object (set or read) + @type metadata: dict + @ivar size: the object's size (cached) + @type size: number + @ivar last_modified: date and time of last file modification (cached) + @type last_modified: str + @ivar container: the object's container (generally treat as read-only) + @type container: L{Container} + """ + # R/O support of the legacy objsum attr. + objsum = property(lambda self: self._etag) + + def __set_etag(self, value): + self._etag = value + self._etag_override = True + + etag = property(lambda self: self._etag, __set_etag) + + def __init__(self, container, name=None, force_exists=False, object_record=None): + """ + Storage objects rarely if ever need to be instantiated directly by the + user. + + Instead, use the L{create_object}, + L{get_object}, + L{list_objects} and other + methods on its parent L{Container} object. + """ + self.container = container + self.last_modified = None + self.metadata = {} + if object_record: + self.name = object_record['name'] + self.content_type = object_record['content_type'] + self.size = object_record['bytes'] + self.last_modified = object_record['last_modified'] + self._etag = object_record['hash'] + self._etag_override = False + else: + self.name = name + self.content_type = None + self.size = None + self._etag = None + self._etag_override = False + if not self._initialize() and force_exists: + raise NoSuchObject(self.name) + + @requires_name(InvalidObjectName) + def read(self, size=-1, offset=0, hdrs=None, buffer=None, callback=None): + """ + Read the content from the remote storage object. + + By default this method will buffer the response in memory and + return it as a string. However, if a file-like object is passed + in using the buffer keyword, the response will be written to it + instead. + + A callback can be passed in for reporting on the progress of + the download. The callback should accept two integers, the first + will be for the amount of data written so far, the second for + the total size of the transfer. Note: This option is only + applicable when used in conjunction with the buffer option. + + >>> test_object.write('hello') + >>> test_object.read() + 'hello' + + @param size: combined with offset, defines the length of data to be read + @type size: number + @param offset: combined with size, defines the start location to be read + @type offset: number + @param hdrs: an optional dict of headers to send with the request + @type hdrs: dictionary + @param buffer: an optional file-like object to write the content to + @type buffer: file-like object + @param callback: function to be used as a progress callback + @type callback: callable(transferred, size) + @rtype: str or None + @return: a string of all data in the object, or None if a buffer is used + """ + self._name_check() + if size > 0: + range = 'bytes=%d-%d' % (offset, (offset + size) - 1) + if hdrs: + hdrs['Range'] = range + else: + hdrs = {'Range': range} + response = self.container.conn.make_request('GET', + path = [self.container.name, self.name], hdrs = hdrs) + if (response.status < 200) or (response.status > 299): + buff = response.read() + raise ResponseError(response.status, response.reason) + + if hasattr(buffer, 'write'): + scratch = response.read(8192) + transferred = 0 + + while len(scratch) > 0: + buffer.write(scratch) + transferred += len(scratch) + if callable(callback): + callback(transferred, self.size) + scratch = response.read(8192) + return None + else: + return response.read() + + def save_to_filename(self, filename, callback=None): + """ + Save the contents of the object to filename. + + >>> container = connection['container1'] + >>> obj = container.get_object('backup_file') + >>> obj.save_to_filename('./backup_file') + + @param filename: name of the file + @type filename: str + @param callback: function to be used as a progress callback + @type callback: callable(transferred, size) + """ + try: + fobj = open(filename, 'wb') + self.read(buffer=fobj, callback=callback) + finally: + fobj.close() + + @requires_name(InvalidObjectName) + def stream(self, chunksize=8192, hdrs=None): + """ + Return a generator of the remote storage object's data. + + Warning: The HTTP response is only complete after this generator + has raised a StopIteration. No other methods can be called until + this has occurred. + + >>> test_object.write('hello') + >>> test_object.stream() + + >>> '-'.join(test_object.stream(chunksize=1)) + 'h-e-l-l-o' + + @param chunksize: size in bytes yielded by the generator + @type chunksize: number + @param hdrs: an optional dict of headers to send in the request + @type hdrs: dict + @rtype: str generator + @return: a generator which yields strings as the object is downloaded + """ + self._name_check() + response = self.container.conn.make_request('GET', + path = [self.container.name, self.name], hdrs = hdrs) + if response.status < 200 or response.status > 299: + buff = response.read() + raise ResponseError(response.status, response.reason) + buff = response.read(chunksize) + while len(buff) > 0: + yield buff + buff = response.read(chunksize) + # I hate you httplib + buff = response.read() + + @requires_name(InvalidObjectName) + def sync_metadata(self): + """ + Commits the metadata to the remote storage system. + + >>> test_object = container['paradise_lost.pdf'] + >>> test_object.metadata = {'author': 'John Milton'} + >>> test_object.sync_metadata() + + Object metadata can be set and retrieved through the object's + .metadata attribute. + """ + self._name_check() + if self.metadata: + headers = self._make_headers() + headers['Content-Length'] = 0 + response = self.container.conn.make_request( + 'POST', [self.container.name, self.name], hdrs=headers, data='' + ) + buff = response.read() + if response.status != 202: + raise ResponseError(response.status, response.reason) + + def __get_conn_for_write(self): + headers = self._make_headers() + + headers['X-Auth-Token'] = self.container.conn.token + + path = "/%s/%s/%s" % (self.container.conn.uri.rstrip('/'), \ + quote(self.container.name), quote(self.name)) + + # Requests are handled a little differently for writes ... + http = self.container.conn.connection + + # TODO: more/better exception handling please + http.putrequest('PUT', path) + for hdr in headers: + http.putheader(hdr, headers[hdr]) + http.putheader('User-Agent', consts.user_agent) + http.endheaders() + return http + + # pylint: disable-msg=W0622 + @requires_name(InvalidObjectName) + def write(self, data='', verify=True, callback=None): + """ + Write data to the remote storage system. + + By default, server-side verification is enabled, (verify=True), and + end-to-end verification is performed using an md5 checksum. When + verification is disabled, (verify=False), the etag attribute will + be set to the value returned by the server, not one calculated + locally. When disabling verification, there is no guarantee that + what you think was uploaded matches what was actually stored. Use + this optional carefully. You have been warned. + + A callback can be passed in for reporting on the progress of + the upload. The callback should accept two integers, the first + will be for the amount of data written so far, the second for + the total size of the transfer. + + >>> test_object = container.create_object('file.txt') + >>> test_object.content_type = 'text/plain' + >>> fp = open('./file.txt') + >>> test_object.write(fp) + + @param data: the data to be written + @type data: str or file + @param verify: enable/disable server-side checksum verification + @type verify: boolean + @param callback: function to be used as a progress callback + @type callback: callable(transferred, size) + """ + self._name_check() + if isinstance(data, file): + # pylint: disable-msg=E1101 + try: + data.flush() + except IOError: + pass # If the file descriptor is read-only this will fail + self.size = int(os.fstat(data.fileno())[6]) + else: + data = StringIO.StringIO(data) + self.size = data.len + + # If override is set (and _etag is not None), then the etag has + # been manually assigned and we will not calculate our own. + + if not self._etag_override: + self._etag = None + + if not self.content_type: + # pylint: disable-msg=E1101 + type = None + if hasattr(data, 'name'): + type = mimetypes.guess_type(data.name)[0] + self.content_type = type and type or 'application/octet-stream' + + http = self.__get_conn_for_write() + + response = None + transfered = 0 + running_checksum = md5() + + buff = data.read(4096) + try: + while len(buff) > 0: + http.send(buff) + if verify and not self._etag_override: + running_checksum.update(buff) + buff = data.read(4096) + transfered += len(buff) + if callable(callback): + callback(transfered, self.size) + response = http.getresponse() + buff = response.read() + except timeout, err: + if response: + # pylint: disable-msg=E1101 + buff = response.read() + raise err + else: + if verify and not self._etag_override: + self._etag = running_checksum.hexdigest() + + # ---------------------------------------------------------------- + + if (response.status < 200) or (response.status > 299): + raise ResponseError(response.status, response.reason) + + # If verification has been disabled for this write, then set the + # instances etag attribute to what the server returns to us. + if not verify: + for hdr in response.getheaders(): + if hdr[0].lower() == 'etag': + self._etag = hdr[1] + + @requires_name(InvalidObjectName) + def send(self, iterable): + """ + Write potentially transient data to the remote storage system using a + generator or stream. + + If the object's size is not set, chunked transfer encoding will be + used to upload the file. + + If the object's size attribute is set, it will be used as the + Content-Length. If the generator raises StopIteration prior to yielding + the right number of bytes, an IncompleteSend exception is raised. + + If the content_type attribute is not set then a value of + application/octet-stream will be used. + + Server-side verification will be performed if an md5 checksum is + assigned to the etag property before calling this method, + otherwise no verification will be performed, (verification + can be performed afterward though by using the etag attribute + which is set to the value returned by the server). + + >>> test_object = container.create_object('backup.tar.gz') + >>> pfd = os.popen('tar -czvf - ./data/', 'r') + >>> test_object.send(pfd) + + @param iterable: stream or generator which yields the content to upload + @type iterable: generator or stream + """ + self._name_check() + + if hasattr(iterable, 'read'): + def file_iterator(file): + chunk = file.read(4095) + while chunk: + yield chunk + chunk = file.read(4095) + raise StopIteration() + iterable = file_iterator(iterable) + + # This method implicitly diables verification + if not self._etag_override: + self._etag = None + + if not self.content_type: + self.content_type = 'application/octet-stream' + + path = "/%s/%s/%s" % (self.container.conn.uri.rstrip('/'), \ + quote(self.container.name), quote(self.name)) + headers = self._make_headers() + if self.size is None: + del headers['Content-Length'] + headers['Transfer-Encoding'] = 'chunked' + headers['X-Auth-Token'] = self.container.conn.token + headers['User-Agent'] = consts.user_agent + http = self.container.conn.connection + http.putrequest('PUT', path) + for key, value in headers.iteritems(): + http.putheader(key, value) + http.endheaders() + + response = None + transferred = 0 + try: + for chunk in iterable: + if self.size is None: + http.send("%X\r\n" % len(chunk)) + http.send(chunk) + http.send("\r\n") + else: + http.send(chunk) + transferred += len(chunk) + if self.size is None: + http.send("0\r\n\r\n") + # If the generator didn't yield enough data, stop, drop, and roll. + elif transferred < self.size: + raise IncompleteSend() + response = http.getresponse() + buff = response.read() + except timeout, err: + if response: + # pylint: disable-msg=E1101 + buff = response.read() + raise err + + if (response.status < 200) or (response.status > 299): + raise ResponseError(response.status, response.reason) + + for hdr in response.getheaders(): + if hdr[0].lower() == 'etag': + self._etag = hdr[1] + + def load_from_filename(self, filename, verify=True, callback=None): + """ + Put the contents of the named file into remote storage. + + >>> test_object = container.create_object('file.txt') + >>> test_object.content_type = 'text/plain' + >>> test_object.load_from_filename('./my_file.txt') + + @param filename: path to the file + @type filename: str + @param verify: enable/disable server-side checksum verification + @type verify: boolean + @param callback: function to be used as a progress callback + @type callback: callable(transferred, size) + """ + fobj = open(filename, 'rb') + self.write(fobj, verify=verify, callback=callback) + fobj.close() + + def _initialize(self): + """ + Initialize the Object with values from the remote service (if any). + """ + if not self.name: + return False + + response = self.container.conn.make_request( + 'HEAD', [self.container.name, self.name] + ) + buff = response.read() + if response.status == 404: + return False + if (response.status < 200) or (response.status > 299): + raise ResponseError(response.status, response.reason) + for hdr in response.getheaders(): + if hdr[0].lower() == 'content-type': + self.content_type = hdr[1] + if hdr[0].lower().startswith('x-object-meta-'): + self.metadata[hdr[0][14:]] = hdr[1] + if hdr[0].lower() == 'etag': + self._etag = hdr[1] + self._etag_override = False + if hdr[0].lower() == 'content-length': + self.size = int(hdr[1]) + if hdr[0].lower() == 'last-modified': + self.last_modified = hdr[1] + return True + + def __str__(self): + return self.name + + def _name_check(self): + if len(self.name) > consts.object_name_limit: + raise InvalidObjectName(self.name) + + def _make_headers(self): + """ + Returns a dictionary representing http headers based on the + respective instance attributes. + """ + headers = {} + headers['Content-Length'] = self.size and self.size or 0 + if self._etag: headers['ETag'] = self._etag + + if self.content_type: headers['Content-Type'] = self.content_type + else: headers['Content-Type'] = 'application/octet-stream' + + for key in self.metadata: + if len(key) > consts.meta_name_limit: + raise(InvalidMetaName(key)) + if len(self.metadata[key]) > consts.meta_value_limit: + raise(InvalidMetaValue(self.metadata[key])) + headers['X-Object-Meta-'+key] = self.metadata[key] + return headers + + @classmethod + def compute_md5sum(cls, fobj): + """ + Given an open file object, returns the md5 hexdigest of the data. + """ + checksum = md5() + buff = fobj.read(4096) + while buff: + checksum.update(buff) + buff = fobj.read(4096) + fobj.seek(0) + return checksum.hexdigest() + + def public_uri(self): + """ + Retrieve the URI for this object, if its container is public. + + >>> container1 = connection['container1'] + >>> container1.make_public() + >>> container1.create_object('file.txt').write('testing') + >>> container1['file.txt'].public_uri() + 'http://c00061.cdn.cloudfiles.rackspacecloud.com/file.txt' + + @return: the public URI for this object + @rtype: str + """ + return "%s/%s" % (self.container.public_uri().rstrip('/'), + quote(self.name)) + +class ObjectResults(object): + """ + An iterable results set object for Objects. + + This class implements dictionary- and list-like interfaces. + """ + def __init__(self, container, objects=None): + self._objects = objects and objects or list() + self._names = [obj['name'] for obj in self._objects] + self.container = container + + def __getitem__(self, key): + return Object(self.container, object_record=self._objects[key]) + + def __getslice__(self, i, j): + return [Object(self.container, object_record=k) for k in self._objects[i:j]] + + def __contains__(self, item): + return item in self._objects + + def __len__(self): + return len(self._objects) + + def __repr__(self): + return 'ObjectResults: %s objects' % len(self._objects) + __str__ = __repr__ + + def index(self, value, *args): + """ + returns an integer for the first index of value + """ + return self._names.index(value, *args) + + def count(self, value): + """ + returns the number of occurrences of value + """ + return self._names.count(value) + +# vim:set ai sw=4 ts=4 tw=0 expandtab: diff --git a/code/daemon/dependencies/cloudfiles/storage_object.pyc b/code/daemon/dependencies/cloudfiles/storage_object.pyc new file mode 100644 index 0000000..f0cb0a1 Binary files /dev/null and b/code/daemon/dependencies/cloudfiles/storage_object.pyc differ diff --git a/code/daemon/dependencies/cloudfiles/utils.py b/code/daemon/dependencies/cloudfiles/utils.py new file mode 100644 index 0000000..619b719 --- /dev/null +++ b/code/daemon/dependencies/cloudfiles/utils.py @@ -0,0 +1,47 @@ +""" See COPYING for license information. """ + +import re +from urlparse import urlparse +from errors import InvalidUrl +from consts import object_name_limit + +def parse_url(url): + """ + Given a URL, returns a 4-tuple containing the hostname, port, + a path relative to root (if any), and a boolean representing + whether the connection should use SSL or not. + """ + (scheme, netloc, path, params, query, frag) = urlparse(url) + + # We only support web services + if not scheme in ('http', 'https'): + raise InvalidUrl('Scheme must be one of http or https') + + is_ssl = scheme == 'https' and True or False + + # Verify hostnames are valid and parse a port spec (if any) + match = re.match('([a-zA-Z0-9\-\.]+):?([0-9]{2,5})?', netloc) + + if match: + (host, port) = match.groups() + if not port: + port = is_ssl and '443' or '80' + else: + raise InvalidUrl('Invalid host and/or port: %s' % netloc) + + return (host, int(port), path.strip('/'), is_ssl) + +def requires_name(exc_class): + """Decorator to guard against invalid or unset names.""" + def wrapper(f): + def decorator(*args, **kwargs): + if not hasattr(args[0], 'name'): + raise exc_class('') + if not args[0].name: + raise exc_class(args[0].name) + return f(*args, **kwargs) + decorator.__name__ = f.__name__ + decorator.__doc__ = f.__doc__ + decorator.parent_func = f + return decorator + return wrapper diff --git a/code/daemon/dependencies/cloudfiles/utils.pyc b/code/daemon/dependencies/cloudfiles/utils.pyc new file mode 100644 index 0000000..8de58ea Binary files /dev/null and b/code/daemon/dependencies/cloudfiles/utils.pyc differ diff --git a/code/daemon/transporters/transporter_mosso.py b/code/daemon/transporters/transporter_mosso.py new file mode 100644 index 0000000..5a97025 --- /dev/null +++ b/code/daemon/transporters/transporter_mosso.py @@ -0,0 +1,39 @@ +from transporter import * +from storages.mosso import * + + +TRANSPORTER_CLASS = "TransporterMosso" + + +class TransporterMosso (Transporter): + + + name = 'mosso' + valid_settings = ImmutableSet(["username", "api_key", "container"]) + required_settings = ImmutableSet(["username", "api_key", "container"]) + + + def __init__(self, settings, callback, error_callback, parent_logger=None): + Transporter.__init__(self, settings, callback, error_callback, parent_logger) + + # Raise exception when required settings have not been configured. + configured_settings = Set(self.settings.keys()) + if not "username" in configured_settings: + raise ImpropertlyConfigured, "username not set" + if not "api_key" in configured_settings: + raise ImpropertlyConfigured, "api_key not set" + if not "container" in configured_settings: + raise ImpropertlyConfigured, "container not set" + + # Map the settings to the format expected by S3Storage. + try: + self.storage = CloudFilesStorage( + self.settings["username"], + self.settings["api_key"], + self.settings["container"] + ) + except Exception, e: + if e.__class__ == cloudfiles.errors.AuthenticationFailed: + raise ConnectionError, "Authentication failed" + else: + raise ConnectionError(e)