From e72f0878235ffdf4846ddb5471da4d5fb28f5685 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Sun, 16 Feb 2014 20:10:29 +0100 Subject: [PATCH] This has been moved to http://github.com/cloudfs/ftp-cloudfs --- ChangeLog | 268 --------- README.rst | 216 +------- bin/ftpcloudfs | 18 - ftpcloudfs.conf.example | 69 --- ftpcloudfs/.gitignore | 4 - ftpcloudfs/__init__.py | 0 ftpcloudfs/chunkobject.py | 65 --- ftpcloudfs/constants.py | 11 - ftpcloudfs/errors.py | 21 - ftpcloudfs/fs.py | 1015 ---------------------------------- ftpcloudfs/main.py | 329 ----------- ftpcloudfs/monkeypatching.py | 117 ---- ftpcloudfs/server.py | 78 --- ftpcloudfs/utils.py | 49 -- setup.py | 37 -- tests/README | 10 - tests/test_fs.py | 638 --------------------- tests/test_ftpcloudfs.py | 368 ------------ 18 files changed, 2 insertions(+), 3311 deletions(-) delete mode 100644 ChangeLog delete mode 100755 bin/ftpcloudfs delete mode 100644 ftpcloudfs.conf.example delete mode 100644 ftpcloudfs/.gitignore delete mode 100644 ftpcloudfs/__init__.py delete mode 100644 ftpcloudfs/chunkobject.py delete mode 100644 ftpcloudfs/constants.py delete mode 100644 ftpcloudfs/errors.py delete mode 100644 ftpcloudfs/fs.py delete mode 100644 ftpcloudfs/main.py delete mode 100644 ftpcloudfs/monkeypatching.py delete mode 100644 ftpcloudfs/server.py delete mode 100644 ftpcloudfs/utils.py delete mode 100755 setup.py delete mode 100644 tests/README delete mode 100644 tests/test_fs.py delete mode 100644 tests/test_ftpcloudfs.py diff --git a/ChangeLog b/ChangeLog deleted file mode 100644 index e2b00f6..0000000 --- a/ChangeLog +++ /dev/null @@ -1,268 +0,0 @@ -2013-12-01 Juan J. Martinez - - * 0.25.1, 0.25.2 - - Fixed a bug in ObjectStorageFS when used in "delayed authentication" - mode that resulted in a information leak vulnerability in sftp-cloudfs. - - Under certain conditions it was possible to serve a cached directory - listing from a different user. - -2013-11-15 Juan J. Martinez - - * 0.25 - - Large file support added. The server will split big files in parts - to go over the object size limit of Swift. See 'split-large-files' - configuration token. - - For further info please read: - - http://docs.openstack.org/developer/swift/overview_large_objects.html - -2013-11-08 Juan J. Martinez - - * 0.24.3 - - Updated requirements to pyftpdlib >= 1.3.0. - -2013-10-17 Juan J. Martinez - - * 0.24.2 - - Fixed a bug in token cache code. The server could get stuck with - a cached invalid token not requesting a new one. - -2013-10-12 Juan J. Martinez - - * 0.24.1 - - Fixed a small bug that prevented users with no privileges to list - the root directory to access containers with the appropriate ACL. - This was affecting sftp-cloudfs mainly as some SFTP clients will - perform 'stat' over a directory before 'cd'. The problem was - introduced in 0.23. - -2013-10-11 Juan J. Martinez - - * 0.24 - - Introduced an auth token cache when memcache is available. - Thanks to Vil Surkin for the RFE. - -2013-10-02 Juan J. Martinez - - * 0.23.4 - - Fixed a bug listing a directory with more than 10k objects that - included a virtual directory ('subdir') as last object of a 10k batch. - -2013-09-29 Juan J. Martinez - - * 0.23.3 - - Fixed a bug that was preventing non privileged users to log in. That - was introduced in 0.23.2. - -2013-09-29 Juan J. Martinez - - * 0.23.2 - - Fixed a bug that was raising an exception at the first swift operation - after authentication. - -2013-09-08 Juan J. Martinez - - * 0.23.1 - - Fixed a bug that prevented Auth 2.0 (keystone) to work. - Thanks to Dmitry (cyberflow) for the report. - -2013-08-29 Juan J. Martinez - - * 0.23 - ftpcloudfs ported from python-cloudfiles to python-swiftclient - - Using python-swiftfiles instead of the abandoned python-cloudfiles - - The code has been tidied up for better maintainability and there are - some performance improvements. The functionality should be the same, - but please test this release before upgrading your production systems! - - From now on the project will focus on OpenStack Object Storage (swift); - although Rackspace Cloud Files is still compatible. - - If you want to keep using python-cloudfiles, please use ftp-cloudfs 0.22. - -2013-07-11 Juan J. Martinez - - * 0.22 - Cache performance improvements - - Cache management has been improved limiting the calls to memcache - - Cache entries are now serialized using JSON before being stored in memcache - (memcache uses pickle by default) - - Enabled cache compression - - IMPORTANT: this is the last release supporting python-cloudfiles. - -2013-06-19 Juan J. Martinez - - * 0.21.1 - Minor fixes and cache regression - - Improved logging for socket errors (timeouts mainly) - - Fixed cache regression that abused cache flushes when using Memcache - - Proper handling of no user/password logins - -2013-06-12 Juan J. Martinez - - * 0.21 - Port to pyftpdlib 1.2.0 - - Using pyftpdlib's MultiprocessFTPServer instead of our pre-fork model. - - 'workers' configuration token has been removed (it had no effect in new pyftpdlib). - - fully converted to use unicode (pyftpdlib requirement). - -2013-03-15 Juan J. Martinez - - * 0.20.1 - - This will be the last release supporting pyftpdlib <= 0.7.0. - pyftpdlib 1.0.1 will work with a warning. - -2013-02-07 Juan J. Martinez - - * 0.20 - - Auth 2.0 (OpenStack Identity Service 2.0, aka Keystone) support, thanks to - schuermannkl for his help testing the feature - -2013-02-05 Juan J. Martinez - - * 0.19.1 - - Minor release to fix Debian packaging support - - Copyright year bump - -2012-10-30 Juan J. Martinez - * 0.19 - - Support for manifest file (big files), showing real size/hash - - Seek support for read operations (FTP REST + RETR support, AKA resume download) - -2012-09-12 Juan J. Martinez - * 0.18 - - Bug fixes in the code to account the max number of connections per IP, - thanks to Konstantin vz'One Enchant - -2012-09-11 Juan J. Martinez - * 0.17.1 - - Bug fix, code cleaning - - when the retry code on SSLError fails, an exception was raised making that - worker crash; thanks to Konstantin vz'One Enchant for the patch - -2012-09-07 Juan J. Martinez - * 0.17 - - Bug fixes from Maxim Mitroshin - - Fixed a race condition when checking the max connections per IP - - More efficient shared cache by limiting the number of Memcache connections - -2012-09-04 Juan J. Martinez - * 0.16 - - Improved logging, including formatting, log in, log out and most important commands - -2012-08-31 Juan J. Martinez - * 0.15 - - Bug fixes - - Catch SSLErrors on Connection.make_request and retry - - Added X-Forwarded-For header to PUT requests - -2012-06-22 Juan J. Martinez - * 0.14 - Support for directory listings with more than 10000 objects - -2012-06-07 Juan J. Martinez - * 0.13 - Shared cache bug fixes and improvements - - Bug fixes - - the cache failed to invalidate after write operations on the root directory - - improved cache invalidation - - tests were modified to run properly with a shared cache enabled server - -2012-03-28 Juan J. Martinez - * 0.12.2 - Minor fix (unrequired cache flush) - -2012-03-28 Juan J. Martinez - * 0.12.1 - Minor fixes (README.rst and version number) - -2012-03-27 Juan J. Martinez - * 0.12 - Small fixes and performance improvements - - New features - - Cache code revisited, including support for a shared cache with Memcache - -2012-01-03 Juan J. Martinez - * 0.11 - Fixes and performance improvements - - Bug fixes - - use cloudfiles_api_timeout - - performance improvements on downloads - - improved logging - - handle SSL errors - -2011-11-24 Juan J. Martinez - * 0.10 - Fixes, better packaging support and some new features - - Bug fixes - - Fix pypi tarball - - Fix debian packaging - - New features - - MD5 FTP extension - - Max connections per IP limit - - X-Forwarded-For header support in API calls - -2011-08-17 Nick Craig-Wood - * 0.9 - Mostly bug fixes - - Bug fixes - - Fix licence inconsistency - - Improve cache management to avoid inconsistencies between workers - - Don't allow DELE on directories - - Fix masquerade option - - Fix user and group - - Fix allowing access to a container even if the root is inaccessible - -2011-06-17 Nick Craig-Wood - * 0.8 - Lots of fixes and features after extensive load testing - - Bug fixes - - Fix cat file bug - - Fix exceptions on closing files crashing the daemon - - Fix excessive open files by running garbage collector periodically - - Fix time parsing (timezone error (times wrong) and floating point parsing) - - Fix crash on logging unicode strings - - Fix timeout problems under heavy load - - New features - - Daemon improvements: logging, pidfile, drop privileges, syslog, multiple workers - - Verbose switch - - Configuration file support - - Internal reorganisation to make re-usable and testable ftpcloudfs.fs interface - -2011-03-18 Chmouel Boudjnah - * 0.7 - A lot of improvement and fixes from Nick Craig-Wood nick@craig-wood.com - highlights are (see commmit for details) : - - - Pseudo-hierachical folders/directories supported - - Containers show size in listings - - Logging in server module - - Attempt to catch and translate all cloudfiles errors with correct error numbers - - Rename support - - Lots of tests - -2011-02-19 Chmouel Boudjnah - * 0.6 - - Daemonize ftp-cloudfs with python-daemon. - -2011-02-12 Chmouel Boudjnah - - * 0.5: - A lot of improvement and fixes from Nick Craig-Wood nick@craig-wood.com - highlights are : - - - Make compatible with pyftpd >= 0.6 - - Implement file modification times in RackspaceCloudFilesFS.stat - - Fix crash if user doesn't have permissions to list containers - - Remove custom directory list routines and replace with a listdir cache - to improve compatibility - - Allow to specify a custom auth url (ie: for OpenStack or UK CloudFiles). - -2009-11-02 Chmouel Boudjnah - - * server.py (RackspaceCloudAuthorizer.validate_authentication): - Add servicenet support. - diff --git a/README.rst b/README.rst index cac54d0..1ca63df 100644 --- a/README.rst +++ b/README.rst @@ -1,215 +1,3 @@ -================================================= -FTP Interface to OpenStack Object Storage (Swift) -================================================= - -:Homepage: https://pypi.python.org/pypi/ftp-cloudfs/ -:Credits: Copyright 2009--2013 Chmouel Boudjnah -:Licence: MIT - - -DESCRIPTION -=========== - -ftp-cloudfs is a ftp server acting as a proxy to `OpenStack Object Storage (swift)`_. -It allow you to connect via any FTP client to do upload/download or create containers. - -By default the server will bind to port 2021 which allow to be run as a non -root/administrator user. - -.. _OpenStack Object Storage (Swift): http://launchpad.net/swift - -It supports pseudo-hierarchical folders/directories as described in the `OpenStack Object Storage API`_. - -.. _OpenStack Object Storage API: http://docs.openstack.org/openstack-object-storage/developer/content/ - - -REQUIREMENTS -============ - -- Python 2 >= 2.6 -- python-swiftclient >= 1.6.0 - https://github.com/openstack/python-swiftclient/ -- pyftpdlib >= 1.3.0 - http://code.google.com/p/pyftpdlib/ -- python-daemon >= 1.5.5 - http://pypi.python.org/pypi/python-daemon/ -- python-memcache >= 1.45 - http://www.tummy.com/Community/software/python-memcached/ - -IMPORTANT: pyftpdlib 1.2.0 has a couple of known issues (memory leak, file descriptor leak) and it shouldn't -be used in production systems. - - -Operating Systems -================= - -fpt-cloudfs is developed and tested in Ubuntu and Debian Linux distributions but it should work on any -Unix-like (including Mac OS X) as long as you install the requirements listed above. - - -INSTALL -======= - -Use standard setup.py directives ie.:: - - python setup.py install - -Or if you have `pip`_ installed you can just run:: - - pip install ftp-cloudfs - -which will install ftp-cloudfs with all the dependencies needed. - -ftp-cloudfs has been `included in Debian Jessie`_. - -.. _`pip`: http://pip.openplans.org/ -.. _included in Debian Jessie: http://packages.debian.org/jessie/ftp-cloudfs - - -USAGE -====== - -The install should have created a /usr/bin/ftpcloudfs (or whatever -prefix defined in your python distribution or command line arguments) -which can be used like this: - -Usage: ftpcloudfs [options] - -Options: - --version show program's version number and exit - -h, --help show this help message and exit - -p PORT, --port=PORT Port to bind the server (default: 2021) - -b BIND_ADDRESS, --bind-address=BIND_ADDRESS - Address to bind (default: 127.0.0.1) - -a AUTHURL, --auth-url=AUTHURL - Authentication URL (required) - --memcache=MEMCACHE Memcache server(s) to be used for cache (ip:port) - -v, --verbose Be verbose on logging - -f, --foreground Do not attempt to daemonize but run in foreground - -l LOG_FILE, --log-file=LOG_FILE - Log File: Default stdout when in foreground - --syslog Enable logging to the system logger (daemon facility) - --pid-file=PID_FILE Pid file location when in daemon mode - --uid=UID UID to drop the privilige to when in daemon mode - --gid=GID GID to drop the privilige to when in daemon mode - --keystone-auth Use auth 2.0 (Keystone, requires keystoneclient) - --keystone-region-name=REGION_NAME - Region name to be used in auth 2.0 - --keystone-tenant-separator=TENANT_SEPARATOR - Character used to separate tenant_name/username in - auth 2.0 (default: TENANT.USERNAME) - --keystone-service-type=SERVICE_TYPE - Service type to be used in auth 2.0 (default: object- - store) - --keystone-endpoint-type=ENDPOINT_TYPE - Endpoint type to be used in auth 2.0 (default: - publicURL) - -The defaults can be changed using a configuration file (by default in -/etc/ftpcloudfs.conf). Check the example file included in the package. - - -CACHE MANAGEMENT -================ - -`OpenStack Object Storage (Swift)`_ is an object storage and not a real file system. -This proxy simulates enough file system functionality to be used over FTP, but it -has a performance impact. - -To improve the performance a cache is used. It can be local or external (with -Memcache). By default a local cache is used, unless one or more Memcache servers -are configured. - -If you're using just one client the local cache may be fine, but if you're using -several connections, configuring an external cache is highly recommended. - -If an external cache is available it will be used to cache authentication tokens too -so any Memcache server must be secured to prevent unauthorized access as it could be -possible to associate a token with a specific user (not trivial) or even use the -cache key (MD5 hash) to brute-force the user password. - - -AUTH 2.0 -======== - -By default ftp-cloudfs will use Swift auth 1.0, that is compatible with `OpenStack Object Storage` -using `swauth`_ auth middleware and Swift implementations such as `Rackspace Cloud Files` or -`Memset's Memstore Cloud Storage`. - -Optionally `OpenStack Identity Service 2.0`_ can be used. Currently python-keystoneclient (0.3.2+ -recommended) is required to use auth 2.0 and it can be enabled with ``keystone-auth`` option. - -You can provide a tenant name in the FTP login user with TENANT.USERNAME (using a dot as -separator). Please check the example configuration file for further details. - -.. _swauth: https://github.com/gholt/swauth -.. _OpenStack Identity Service 2.0: http://docs.openstack.org/api/openstack-identity-service/2.0/content/index.html -.. _RackSpace Cloud Files: http://www.rackspace.com/cloud/cloud_hosting_products/files/ -.. _Memset's Memstore Cloud Storage: https://www.memset.com/cloud/storage/ - - -LARGE FILE SUPPORT -================== - -The object storage has a limit on the size of a single uploaded object (by default this is 5GB). -Files larger than that can be split in parts and merged back on the fly using a manifest file. - -ftp-cloudfs supports this transparently with the *split-large-files* configuration token, setting -it to the number of megabytes wanted to use for each part (disabled by default). - -When a *FILE* is larger than the specified amount of MB, a *FILE.part* directory will be created and -*n* parts will be created splitting the file automatically. The original file name will be used to -store the manifest. If the original file is downloaded, the parts will be served as it was a single file. - - -SUPPORT -======= - -The project website is at: - -https://github.com/cloudfs/ftp-cloudfs/issues - -There you can file bug reports, ask for help or contribute patches. There's additional information at: - -https://github.com/cloudfs/ftp-cloudfs/wiki - -LICENSE -======= - -Unless otherwise noted, all files are released under the `MIT`_ license, -exceptions contain licensing information in them. - -.. _`MIT`: http://en.wikipedia.org/wiki/MIT_License - - Copyright (C) 2009-2013 Chmouel Boudjnah - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. - - -Authors -======= - -- Chmouel Boudjnah -- Nick Craig-Wood -- Juan J. Martinez - - -Contributors -============ - -- Christophe Le Guern -- Konstantin vz'One Enchant -- Maxim Mitroshin +This has been moved to : +http://github.com/cloudfs/ftp-cloudfs diff --git a/bin/ftpcloudfs b/bin/ftpcloudfs deleted file mode 100755 index 43a3d3d..0000000 --- a/bin/ftpcloudfs +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -""" -FTP Cloud Files - A FTP Proxy to Cloud FIles. -""" -import sys -import os - -sys.path.append( - os.path.abspath( - os.path.join( - os.path.dirname( - sys.argv[0]), ".."))) - -from ftpcloudfs.main import Main - -if __name__ == '__main__': - main = Main() - main.main() diff --git a/ftpcloudfs.conf.example b/ftpcloudfs.conf.example deleted file mode 100644 index 6528a30..0000000 --- a/ftpcloudfs.conf.example +++ /dev/null @@ -1,69 +0,0 @@ -# ftpcloudfs example configuration file -# - -[ftpcloudfs] - -# FTP banner (%v version, %f ftp handler version) -# banner = ftp-cloudfs %v using pyftpdlib %f ready. - -# Port to bind. -# port = 2021 - -# Address to bind. -# bind-address = 127.0.0.1 - -# Authentication URL (required) -# auth-url = (empty) - -# DEPRECATED: Number of workers to use (no effect) -# workers = (empty) - -# Memcache server(s) for external cache (eg 127.0.0.1:11211) -# Can be a comma-separated list. -# memcache = (empty) - -# Maximum number of client connections per IP -# default is 0 (no limit) -# max-cons-per-ip = 0 - -# Large file support. -# Specify a size in MB to split large files. -# split-large-files = (empty) - -# Be verbose on logging. -# verbose = no - -# Enable logging to the system logger. -# syslog = no - -# Log file location. -# log-file = (empty) - -# Pid file location when in daemon mode. -# pid-file = (empty) - -# UID to drop privileges when in daemon mode. -# uid = (empty) - -# GID to drop priviliges when in daemon mode. -# gid = (empty) - -# Masquerade IP address in case your server is behind a firewall or NATed. -# masquerade-firewall = (empty) - -# Auth 2.0 (Keystone), requires keystoneclient -# keystone-auth = no - -# Region name to be used with Auth 2.0 (optional) -# keystone-region-name = (empty) - -# Tenant separator to be used with Auth 2.0 (eg. TENANT.USERNAME) -# keystone-tenant-separator = '.' - -# Service type to be used with Auth 2.0. -# keystone-service-type = 'object-store' - -# Endpoint type to be used with Auth 2.0. -# keystone-endpoint-type = 'publicURL' - -# EOF diff --git a/ftpcloudfs/.gitignore b/ftpcloudfs/.gitignore deleted file mode 100644 index ac0b399..0000000 --- a/ftpcloudfs/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*~ -\#* -.#* -*.pyc diff --git a/ftpcloudfs/__init__.py b/ftpcloudfs/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ftpcloudfs/chunkobject.py b/ftpcloudfs/chunkobject.py deleted file mode 100644 index db12237..0000000 --- a/ftpcloudfs/chunkobject.py +++ /dev/null @@ -1,65 +0,0 @@ - -import logging -from urllib import quote -from socket import timeout -from ssl import SSLError -from swiftclient.client import ClientException, http_connection - -from ftpcloudfs.utils import smart_str - -class ChunkObject(object): - - def __init__(self, conn, container, name, content_type=None): - # FIXME - # self._name_check() - - parsed, self.chunkable_http = http_connection(conn.url) - - logging.debug("ChunkObject: new connection open (%r, %r)" % (parsed, self.chunkable_http)) - - path = '%s/%s/%s' % (parsed.path.rstrip('/'), - quote(smart_str(container)), - quote(smart_str(name)), - ) - headers = { 'X-Auth-Token': conn.token, - 'Content-Type': content_type or 'application/octet-stream', - 'Transfer-Encoding': 'chunked', - # User-Agent ? - } - if conn.real_ip: - headers['X-Forwarded-For'] = conn.real_ip - self.chunkable_http.putrequest('PUT', path) - for key, value in headers.iteritems(): - self.chunkable_http.putheader(key, value) - self.chunkable_http.endheaders() - logging.debug("ChunkedObject: path=%r, headers=%r" % (path, headers)) - - def send_chunk(self, chunk): - logging.debug("ChunkObject: sending %s bytes" % len(chunk)) - try: - self.chunkable_http.send("%X\r\n" % len(chunk)) - self.chunkable_http.send(chunk) - self.chunkable_http.send("\r\n") - except (timeout, SSLError), err: - raise ClientException(err.message) - - def finish_chunk(self): - logging.debug("ChunkObject: finish_chunk") - try: - self.chunkable_http.send("0\r\n\r\n") - response = self.chunkable_http.getresponse() - except (timeout, SSLError), err: - raise ClientException(err.message) - - try: - response.read() - except (timeout, SSLError): - # this is not relevant, keep going - pass - - if response.status // 100 != 2: - raise ClientException(response.reason, - http_status=response.status, - http_reason=response.reason, - ) - diff --git a/ftpcloudfs/constants.py b/ftpcloudfs/constants.py deleted file mode 100644 index 68659c7..0000000 --- a/ftpcloudfs/constants.py +++ /dev/null @@ -1,11 +0,0 @@ -version = '0.25.2' - -default_banner = "ftp-cloudfs %v using pyftpdlib %f ready." -default_config_file = '/etc/ftpcloudfs.conf' -default_address = '127.0.0.1' -default_port = 2021 - -# keystone defaults -default_ks_tenant_separator = '.' -default_ks_service_type = 'object-store' -default_ks_endpoint_type = 'publicURL' diff --git a/ftpcloudfs/errors.py b/ftpcloudfs/errors.py deleted file mode 100644 index 1df2f2d..0000000 --- a/ftpcloudfs/errors.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -Errors for ObjectStorageFS -""" - -class IOSError(OSError, IOError): - """ - Subclass of OSError and IOError. - - This is needed because pyftpdlib catches either OSError, or - IOError depending on which operation it is performing, which is - perfectly correct, but makes our life more difficult. - - However our operations don't map to simple functions, and have - common infrastructure. These common infrastructure functions can - be called from either context and so don't know which error to - raise. - - Using this combined type everywhere fixes the problem at very - small cost (multiple inheritance!). - """ - diff --git a/ftpcloudfs/fs.py b/ftpcloudfs/fs.py deleted file mode 100644 index a58f631..0000000 --- a/ftpcloudfs/fs.py +++ /dev/null @@ -1,1015 +0,0 @@ -""" - A filesystem like interface to an object storage. - -Authors: Chmouel Boudjnah - Nick Craig-Wood - Juan J. Martinez -""" - -import os -import sys -import time -import mimetypes -import stat -import logging -from errno import EPERM, ENOENT, EACCES, EIO, ENOTDIR, ENOTEMPTY -from swiftclient.client import Connection, ClientException -from chunkobject import ChunkObject -from errors import IOSError -import posixpath -from utils import smart_str -from functools import wraps -import memcache -import multiprocessing -try: - from hashlib import md5 -except ImportError: - from md5 import md5 -try: - import json -except ImportError: - import simplejson as json - -__all__ = ['ObjectStorageFS'] - -class ProxyConnection(Connection): - """ - Add X-Forwarded-For header to all requests. - - Optionally if `range_from` is available it will be used to add a Range header - starting from it. - """ - - # max time to cache auth tokens (seconds), based on swift defaults - TOKEN_TTL = 86400 - - def __init__(self, memcache, *args, **kwargs): - self.memcache = memcache - self.real_ip = None - self.range_from = None - self.ignore_auth_cache = False - super(ProxyConnection, self).__init__(*args, **kwargs) - - def http_connection(self): - def request_wrapper(fn): - @wraps(fn) - def request_x_forwarded_for(method, url, body=None, headers=None): - if headers is None: - headers = {} - if self.real_ip: - headers['X-Forwarded-For'] = self.real_ip - if self.range_from: - headers['Range'] = "bytes=%s-" % self.range_from - # only for one request - self.range_from = None - - if 'body' in fn.func_code.co_varnames: - fn(method, url, body=body, headers=headers) - else: # swiftclient 2.0, ported to Requests - fn(method, url, data=body, headers=headers) - return request_x_forwarded_for - - parsed, conn = super(ProxyConnection, self).http_connection() - conn.request = request_wrapper(conn.request) - - return parsed, conn - - def get_auth(self): - """Perform the authentication using a token cache if memcache is available""" - if self.memcache: - key = "tk%s" % md5("%s%s%s" % (self.authurl, self.user, self.key)).hexdigest() - cache = self.memcache.get(key) - if not cache or self.ignore_auth_cache: - logging.debug("token cache miss, key=%s" % key) - cache = super(ProxyConnection, self).get_auth() - self.memcache.set(key, cache, self.TOKEN_TTL) - self.ignore_auth_cache = False - else: - logging.debug("token cache hit, key=%s" % key) - self.ignore_auth_cache = True - return cache - # no memcache - return super(ProxyConnection, self).get_auth() - -def translate_objectstorage_error(fn): - """ - Decorator to catch Object Storage errors and translating them into IOSError. - - Other exceptions are not caught. - """ - @wraps(fn) - def wrapper(*args,**kwargs): - name = getattr(fn, "func_name", "unknown") - log = lambda msg: logging.debug("At %s: %s" % (name, msg)) - try: - return fn(*args, **kwargs) - except ClientException, e: - # some errno mapping - if e.http_status == 404: - err = ENOENT - elif e.http_status == 400: - err = EPERM - elif e.http_status == 403: - err = EACCES - else: - err = EIO - - msg = "%s: %s" % (smart_str(e.msg), smart_str(e.http_reason)) - log(msg) - raise IOSError(err, msg) - return wrapper - -def parse_fspath(path): - """ - Returns a (container, path) tuple. - - For shorter paths replaces not provided values with empty strings. - May raise IOSError for invalid paths. - """ - if not path.startswith('/'): - logging.warning('parse_fspath: You have to provide an absolute path: %r' % path) - raise IOSError(ENOENT, 'Absolute path needed') - parts = path.split('/', 2)[1:] - while len(parts) < 2: - parts.append('') - return tuple(parts) - -class ObjectStorageFD(object): - """File alike object attached to the Object Storage.""" - - split_size = 0 - - def __init__(self, cffs, container, obj, mode): - self.cffs = cffs - self.container = container - self.name = obj - self.mode = mode - self.closed = False - self.total_size = 0 - self.part_size = 0 - self.part = 0 - self.headers = dict() - self.content_type = mimetypes.guess_type(self.name)[0] - self.pending_copy_task = None - - self.obj = None - - # this is only used by `seek`, so we delay the HEAD request until is required - self.size = None - - if not all([container, obj]): - self.closed = True - raise IOSError(EPERM, 'Container and object required') - - logging.debug("ObjectStorageFD object: %r (mode: %r)" % (obj, mode)) - - if 'r' in self.mode: - logging.debug("read fd %r" % self.name) - else: # write - logging.debug("write fd %r" % self.name) - self.obj = ChunkObject(self.conn, self.container, self.name, content_type=self.content_type) - - @property - def part_base_name(self): - return u"%s.part" % self.name - - @property - def part_name(self): - return u"%s/%.6d" % (self.part_base_name, self.part) - - @property - def conn(self): - """Connection to the storage.""" - return self.cffs.conn - - def _start_copy_task(self): - """ - Copy the first part of a multi-part file to its final location and create - the manifest file. - - This happens in the background, pending_copy_task must be cleaned up at - the end. - """ - def copy_task(conn, container, name, part_name, part_base_name): - # open a new connection - conn = ProxyConnection(None, preauthurl=conn.url, preauthtoken=conn.token) - headers = { 'x-copy-from': "/%s/%s" % (container, name) } - logging.debug("copying first part %r/%r, %r" % (container, part_name, headers)) - try: - conn.put_object(container, part_name, headers=headers, contents=None) - except ClientException as ex: - logging.error("Failed to copy %s: %s" % (name, ex.http_reason)) - sys.exit(1) - # setup the manifest - headers = { 'x-object-manifest': "%s/%s" % (container, part_base_name) } - logging.debug("creating manifest %r/%r, %r" % (container, name, headers)) - try: - conn.put_object(container, name, headers=headers, contents=None) - except ClientException as ex: - logging.error("Failed to store the manifest %s: %s" % (name, ex.http_reason)) - sys.exit(1) - logging.debug("copy task done") - self.pending_copy_task = multiprocessing.Process(target=copy_task, - args=(self.conn, - self.container, - self.name, - self.part_name, - self.part_base_name, - ), - ) - self.pending_copy_task.start() - - @translate_objectstorage_error - def write(self, data): - """Write data to the object.""" - if 'r' in self.mode: - raise IOSError(EPERM, "File is opened for read") - - # large file support - if self.split_size: - # data can be of any size, so we need to split it in split_size chunks - offs = 0 - while offs < len(data): - if self.part_size + len(data) - offs > self.split_size: - current_size = self.split_size-self.part_size - logging.debug("data is to large (%r), using %s" % (len(data), current_size)) - else: - current_size = len(data)-offs - self.part_size += current_size - if not self.obj: - self.obj = ChunkObject(self.conn, self.container, self.part_name, content_type=self.content_type) - self.obj.send_chunk(data[offs:offs+current_size]) - offs += current_size - if self.part_size == self.split_size: - logging.debug("current size is %r, split_file is %r" % (self.part_size, self.split_size)) - self.obj.finish_chunk() - # this obj is not valid anymore, will create a new one if a new part is required - self.obj = None - # make it the first part - if self.part == 0: - self._start_copy_task() - self.part_size = 0 - self.part += 1 - else: - self.obj.send_chunk(data) - - @translate_objectstorage_error - def close(self): - """Close the object and finish the data transfer.""" - if 'r' in self.mode: - return - if self.pending_copy_task: - logging.debug("waiting for a pending copy task...") - self.pending_copy_task.join() - logging.debug("wait is over") - if self.pending_copy_task.exitcode != 0: - raise IOSError(EIO, 'Failed to store the file') - if self.obj is not None: - self.obj.finish_chunk() - - def read(self, size=65536): - """ - Read data from the object. - - We can use just one request because 'seek' is not fully supported. - - NB: It uses the size passed into the first call for all subsequent calls. - """ - if self.obj is None: - if self.total_size > 0: - self.conn.range_from = self.total_size - # we need to open a new connection to inject the `Range` header - if self.conn.http_conn: - self.conn.http_conn[1].close() - self.conn.http_conn = None - _, self.obj = self.conn.get_object(self.container, self.name, resp_chunk_size=size) - - logging.debug("read size=%r, total_size=%r (range_from: %s)" % (size, - self.total_size, self.conn.range_from)) - - try: - buff = self.obj.next() - self.total_size += len(buff) - except StopIteration: - return "" - else: - return buff - - def seek(self, offset, whence=None): - """ - Seek in the object. - - It's supported only for read operations because of object storage limitations. - """ - logging.debug("seek offset=%s, whence=%s" % (str(offset), str(whence))) - - if 'r' in self.mode: - - if self.size is None: - meta = self.conn.head_object(self.container, self.name) - try: - self.size = int(meta["content-length"]) - except ValueError: - raise IOSError(EPERM, "Invalid file size") - - if not whence: - offs = offset - elif whence == 1: - offs = self.total_size + offset - elif whence == 2: - offs = self.size - offset - else: - raise IOSError(EPERM, "Invalid file offset") - - if offs < 0 or offs > self.size: - raise IOSError(EPERM, "Invalid file offset") - - # we need to start over after a seek call - if self.obj is not None: - del self.obj # GC the generator - self.obj = None - self.total_size = offs - else: - raise IOSError(EPERM, "Seek not available for write operations") - -class CacheEncoder(json.JSONEncoder): - """JSONEncoder to encode the os.stat_result values into a list.""" - def default(self, obj): - if isinstance(obj, os.stat_result): - return tuple(obj) - return json.JSONEncoder.default(self, obj) - -def serialize(obj): - """Serialize a cache dict into a JSON object.""" - return json.dumps(obj, cls=CacheEncoder) - -def unserialize(js): - """Unserialize a JSON object into a cache dict.""" - return dict(((smart_str(key), os.stat_result(value)) for key, value in json.loads(js).iteritems())) - -class ListDirCache(object): - """ - Cache for listdir. - - This is to cache the very common case when we call listdir and then - immediately call stat() on all the objects. - - In the OS this would be cached in the VFS but we have to make our - own caching here to avoid the stat calls each making a connection. - """ - MAX_CACHE_TIME = 10 # seconds to cache the listdir for - MIN_COMPRESS_LEN = 4096 # min length in bytes to compress cache entries - memcache = None - - def __init__(self, cffs): - self.cffs = cffs - self.path = None - self.cache = {} - self.when = time.time() - - if self.cffs.memcache_hosts and ListDirCache.memcache is None: - logging.debug("connecting to memcache %r" % self.cffs.memcache_hosts) - ListDirCache.memcache = memcache.Client(self.cffs.memcache_hosts) - - @property - def conn(self): - """Connection to the storage.""" - return self.cffs.conn - - def key(self, index): - """Returns a key for a user distributed cache.""" - logging.debug("cache key for %r" % [self.cffs.authurl, self.cffs.username, index]) - if not hasattr(self, "_key_base"): - self._key_base = md5("%s%s" % (self.cffs.authurl, self.cffs.username)).hexdigest() - return "%s-%s" % (self._key_base, md5(smart_str(index)).hexdigest()) - - def flush(self, path=None): - """Flush the listdir cache.""" - logging.debug("cache flush, current path: %s request: %s" % (self.path, path)) - if self.memcache: - if path is not None: - logging.debug("flushing memcache for %r" % path) - self.memcache.delete(self.key(path)) - if self.path == path: - self.cache = None - elif self.path is not None: - logging.debug("flushing memcache for %r" % self.path) - self.memcache.delete(self.key(path)) - self.cache = None - else: - self.cache = None - - def _make_stat(self, last_modified=None, content_type="application/directory", count=1, bytes=0, **kwargs): - """Make a stat object from the parameters passed in from""" - if last_modified: - if "." in last_modified: - last_modified, microseconds = last_modified.rsplit(".", 1) - if microseconds.endswith("Z"): - microseconds = microseconds[:-1] - microseconds = float("0."+microseconds) - else: - microseconds = 0.0 - mtime_tuple = list(time.strptime(last_modified, "%Y-%m-%dT%H:%M:%S")) - mtime_tuple[8] = 0 # Use GMT - mtime = time.mktime(mtime_tuple) + microseconds - else: - mtime = time.time() - if content_type == "application/directory": - mode = 0755|stat.S_IFDIR - else: - mode = 0644|stat.S_IFREG - #(mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) - return os.stat_result((mode, 0L, 0L, count, 0, 0, bytes, mtime, mtime, mtime)) - - def listdir_container(self, cache, container, path=""): - """Fills cache with the list dir of the container""" - container = smart_str(container) - path = smart_str(path) - logging.debug("listdir container %r path %r" % (container, path)) - if path: - prefix = path.rstrip("/")+"/" - else: - prefix = None - _, objects = self.conn.get_container(container, prefix=prefix, delimiter="/") - - # override 10000 objects limit with markers - nbobjects = len(objects) - while nbobjects >= 10000: - # get last object as a marker - lastobject = objects[-1] - if 'subdir' in lastobject: - # {u'subdir': 'dirname'} - lastobjectname = lastobject['subdir'].rstrip("/") - else: - lastobjectname = lastobject['name'] - # get a new list with the marker - _, newobjects = self.conn.get_container(container, prefix=prefix, delimiter="/", marker=lastobjectname) - # get the new list length - nbobjects = len(newobjects) - logging.debug("number of objects after marker %s: %s" % (lastobjectname, nbobjects)) - # add the new list to current list - objects.extend(newobjects) - logging.debug("total number of objects %s:" % len(objects)) - - for obj in objects: - # {u'bytes': 4820, u'content_type': '...', u'hash': u'...', u'last_modified': u'2008-11-05T00:56:00.406565', u'name': u'new_object'}, - if 'subdir' in obj: - # {u'subdir': 'dirname'} - obj['name'] = obj['subdir'].rstrip("/") - elif obj.get('bytes') == 0 and obj.get('hash') and obj.get('content_type') != 'application/directory': - # if it's a 0 byte file, has a hash and is not a directory, we make an extra call - # to check if it's a manifest file and retrieve the real size / hash - manifest_obj = self.conn.head_object(container, obj['name']) - logging.debug("possible manifest file: %r" % manifest_obj) - if 'x-object-manifest' in manifest_obj: - logging.debug("manifest found: %s" % manifest_obj['x-object-manifest']) - obj['hash'] = manifest_obj['etag'] - obj['bytes'] = int(manifest_obj['content-length']) - obj['count'] = 1 - # Keep all names in utf-8, just like the filesystem - name = posixpath.basename(obj['name']).encode("utf-8") - cache[name] = self._make_stat(**obj) - - def listdir_root(self, cache): - """Fills cache with the list of containers""" - logging.debug("listdir root") - try: - _, objects = self.conn.get_account() - except ClientException: - # when implementing contaniners' ACL, getting the containers - # list can raise a ResponseError, but still access to the - # the containers we have permissions to access to - return - for obj in objects: - # {u'count': 0, u'bytes': 0, u'name': u'container1'}, - # Keep all names in utf-8, just like the filesystem - name = obj['name'].encode("utf-8") - cache[name] = self._make_stat(**obj) - - def listdir(self, path): - """Return the directory list of the path, filling the cache in the process""" - path = path.rstrip("/") or "/" - logging.debug("listdir %r" % path) - cache = None - if self.memcache: - cache = self.memcache.get(self.key(path)) - if cache: - cache = unserialize(cache) - logging.debug("memcache hit %r" % self.key(path)) - else: - logging.debug("memcache miss %r" % self.key(path)) - if not cache: - cache = {} - if path == "/": - self.listdir_root(cache) - else: - container, obj = parse_fspath(path) - self.listdir_container(cache, container, obj) - if self.memcache: - if self.memcache.set(self.key(path), serialize(cache), self.MAX_CACHE_TIME, min_compress_len=self.MIN_COMPRESS_LEN): - logging.debug("memcache stored %r" % self.key(path)) - else: - logging.warning("Failed to store the cache") - self.cache = cache - self.path = path - self.when = time.time() - leaves = sorted(self.cache.keys()) - logging.debug(".. %r" % leaves) - return leaves - - def listdir_with_stat(self, path): - """ - Return the directory list of the path with stat objects. - - The cache will be filled in in the process, as a list of tuples - (leafname, stat_result). - """ - self.listdir(path) - return sorted(self.cache.iteritems()) - - def valid(self, path): - """Check the cache is valid for the container and directory path""" - if not self.cache or self.path != path: - if self.memcache: - cache = self.memcache.get(self.key(path)) - if cache: - cache = unserialize(cache) - logging.debug("memcache hit %r" % self.key(path)) - self.cache = cache - self.path = path - return True - return False - age = time.time() - self.when - return age < self.MAX_CACHE_TIME - - def stat(self, path): - """ - Returns an os.stat_result for path or raises IOSError. - - Returns the information from the cache if possible. - """ - path = path.rstrip("/") or "/" - logging.debug("stat path %r" % (path)) - directory, leaf = posixpath.split(path) - # Refresh the cache it if is old, or wrong - if not self.valid(directory): - logging.debug("invalid cache for %r (path: %r)" % (directory, self.path)) - self.listdir(directory) - if path != "/": - try: - stat_info = self.cache[smart_str(leaf)] - except KeyError: - logging.debug("Didn't find %r in directory listing" % leaf) - # it can be a container and the user doesn't have - # permissions to list the root - if directory == '/' and leaf: - try: - container = self.conn.head_container(leaf) - except ClientException: - raise IOSError(ENOENT, 'No such file or directory %s' % leaf) - - logging.debug("Accessing %r container without root listing" % leaf) - stat_info = self._make_stat(count=int(container["x-container-object-count"]), - bytes=int(container["x-container-bytes-used"]), - ) - else: - raise IOSError(ENOENT, 'No such file or directory %s' % leaf) - else: - # Root directory size is sum of containers, count is containers - bytes = sum(stat_info.st_size for stat_info in self.cache.values()) - count = len(self.cache) - stat_info = self._make_stat(count=count, bytes=bytes) - logging.debug("stat path: %r" % stat_info) - return stat_info - -class ObjectStorageFS(object): - """ - Object Storage File System emulation. - - All the methods on this class emulate os.* or os.path.* functions - of the same name. - """ - memcache_hosts = None - - @translate_objectstorage_error - def __init__(self, username, api_key, authurl, keystone=None): - """ - Create the Object Storage connection. - - username - if None then don't make the connection (delayed auth) - api_key - authurl - keystone - optional for auth 2.0 (keystone) - """ - self.conn = None - self.authurl = authurl - self.keystone = keystone - # A cache to hold the information from the last listdir - self._listdir_cache = ListDirCache(self) - self._cwd = '/' - if username is not None: - self.authenticate(username, api_key) - - @translate_objectstorage_error - def authenticate(self, username, api_key): - """Authenticates and opens the connection""" - if not username or not api_key: - raise ClientException("username/password required", http_status=401) - - kwargs = dict(authurl=self.authurl, auth_version="1.0") - - if self.keystone: - if self.keystone['tenant_separator'] in username: - tenant_name, username = username.split(self.keystone['tenant_separator'], 1) - else: - tenant_name = None - - logging.debug("keystone authurl=%r username=%r tenant_name=%r conf=%r" % (self.authurl, username, tenant_name, self.keystone)) - - ks = self.keystone - kwargs["auth_version"] = "2.0" - kwargs["tenant_name"] = tenant_name - kwargs["os_options"] = dict(service_type=ks['service_type'], - endpoint_type=ks['endpoint_type'], - region_name=ks['region_name'], - ) - - self.conn = ProxyConnection(self._listdir_cache.memcache, - user=username, - key=api_key, - **kwargs - ) - # force authentication - self.conn.url, self.conn.token = self.conn.get_auth() - self.conn.http_conn = None - # now we are authenticated and we have an username - self.username = username - - def close(self): - """Dummy function which does nothing - no need to close""" - pass - - def isabs(self, path): - """Test whether a path is absolute""" - return posixpath.isabs(path) - - def normpath(self, path): - """Normalize path, eliminating double slashes, etc""" - return posixpath.normpath(path) - - def abspath(self, path): - """Return an absolute path""" - if not self.isabs(path): - path = posixpath.join(self.getcwd(), path) - return self.normpath(path) - - def mkstemp(self, suffix='', prefix='', dir=None, mode='wb'): - """ - A wrapper around tempfile.mkstemp creating a file with a unique name. - - Unlike mkstemp it returns an object with a file-like interface. - """ - e = "mkstemp suffix=%r prefix=%r, dir=%r mode=%r - not implemented" % (suffix, prefix, dir, mode) - logging.debug(e) - raise IOSError(EPERM, 'Operation not permitted: %s' % e) - - @translate_objectstorage_error - def open(self, path, mode): - """Open path with mode, raise IOError on error""" - path = self.abspath(path) - logging.debug("open %r mode %r" % (path, mode)) - self._listdir_cache.flush(posixpath.dirname(path)) - container, obj = parse_fspath(path) - return ObjectStorageFD(self, container, obj, mode) - - def chdir(self, path): - """Change current directory, raise OSError on error""" - path = self.abspath(path) - logging.debug("chdir %r" % path) - if not path.startswith("/"): - raise IOSError(ENOENT, 'Failed to change directory.') - container, obj = parse_fspath(path) - if not container: - logging.debug("cd to /") - else: - logging.debug("cd to container %r directory %r" % (container, obj)) - if not self.isdir(path): - raise IOSError(ENOTDIR, "Can't cd to a file") - self._cwd = path - - def getcwd(self): - """Returns the current working directory""" - return self._cwd - - def _container_exists(self, container): - # verify the container exsists - try: - self.conn.head_container(container) - except ClientException, e: - if e.http_status == 404: - raise IOSError(ENOTDIR, "Container not found") - raise - return True - - @translate_objectstorage_error - def mkdir(self, path): - """ - Make a directory. - - Raises OSError on error. - """ - path = self.abspath(path) - logging.debug("mkdir %r" % path) - container, obj = parse_fspath(path) - if obj: - self._listdir_cache.flush(posixpath.dirname(path)) - logging.debug("Making directory %r in %r" % (obj, container)) - self._container_exists(container) - self.conn.put_object(container, obj, contents=None, content_type="application/directory") - else: - self._listdir_cache.flush("/") - logging.debug("Making container %r" % (container,)) - self.conn.put_container(container) - - @translate_objectstorage_error - def listdir(self, path): - """ - List a directory. - - Raises OSError on error. - """ - path = self.abspath(path) - logging.debug("listdir %r" % path) - list_dir = map(lambda x: unicode(x, 'utf-8'), self._listdir_cache.listdir(path)) - return list_dir - - @translate_objectstorage_error - def listdir_with_stat(self, path): - """ - Return the directory list of the path with stat objects. - - The the cache is filled in the process, as a list of tuples (leafname, stat_result). - """ - path = self.abspath(path) - logging.debug("listdir_with_stat %r" % path) - return [(unicode(name, 'utf-8)'), stat) for name, stat in self._listdir_cache.listdir_with_stat(path)] - - @translate_objectstorage_error - def rmdir(self, path): - """ - Remove a directory. - - Eaise OSError on error. - """ - path = self.abspath(path) - logging.debug("rmdir %r" % path) - container, obj = parse_fspath(path) - - if not self.isdir(path): - if self.isfile(path): - raise IOSError(ENOTDIR, "Not a directory") - raise IOSError(ENOENT, 'No such file or directory') - - if self.listdir(path): - raise IOSError(ENOTEMPTY, "Directory not empty: %s" % path) - - if obj: - self._listdir_cache.flush(posixpath.dirname(path)) - logging.debug("Removing directory %r in %r" % (obj, container)) - self.conn.delete_object(container, obj) - else: - self._listdir_cache.flush("/") - logging.debug("Removing container %r" % (container,)) - self.conn.delete_container(container) - - @translate_objectstorage_error - def remove(self, path): - """ - Remove a file. - - Raises OSError on error. - """ - path = self.abspath(path) - logging.debug("remove %r" % path) - container, name = parse_fspath(path) - - if not name: - raise IOSError(EACCES, "Can't remove a container") - - if self.isdir(path): - raise IOSError(EACCES, "Can't remove a directory (use rmdir instead)") - - self.conn.delete_object(container, name) - self._listdir_cache.flush(posixpath.dirname(path)) - return not name - - @translate_objectstorage_error - def _rename_container(self, src_container_name, dst_container_name): - """Rename src_container_name into dst_container_name""" - logging.debug("rename container %r -> %r" % (src_container_name, dst_container_name)) - # Delete the old container first, raising error if not empty - self.conn.delete_container(src_container_name) - self.conn.put_container(dst_container_name) - self._listdir_cache.flush("/") - - @translate_objectstorage_error - def rename(self, src, dst): - """ - Rename a file/directory from src to dst. - - Raises OSError on error. - """ - src = self.abspath(src) - dst = self.abspath(dst) - logging.debug("rename %r -> %r" % (src, dst)) - self._listdir_cache.flush() - # Check not renaming to itself - if src == dst: - logging.debug("Renaming %r to itself - doing nothing" % src) - return - # If dst is an existing directory, copy src inside it - if self.isdir(dst): - if dst: - dst += "/" - dst += posixpath.basename(src) - # Check constraints for renaming a directory - if self.isdir(src): - if self.listdir(src): - raise IOSError(ENOTEMPTY, "Can't rename non-empty directory: %s" % src) - if self.isfile(dst): - raise IOSError(ENOTDIR, "Can't rename directory to file") - # Check not renaming to itself - if src == dst: - logging.debug("Renaming %r to itself - doing nothing" % src) - return - # Parse the paths now - src_container_name, src_path = parse_fspath(src) - dst_container_name, dst_path = parse_fspath(dst) - logging.debug("`.. %r/%r -> %r/%r" % (src_container_name, src_path, dst_container_name, dst_path)) - # Check if we are renaming containers - if not src_path and not dst_path and src_container_name and dst_container_name: - return self._rename_container(src_container_name, dst_container_name) - # ...otherwise can't deal with root stuff - if not src_container_name or not src_path or not dst_container_name or not dst_path: - raise IOSError(EACCES, "Can't rename to / from root") - # Check destination directory exists - if not self.isdir(posixpath.split(dst)[0]): - raise IOSError(ENOENT, "Can't copy %r to %r, destination directory doesn't exist" % (src, dst)) - - # check src/dst containers - self._container_exists(src_container_name) - self._container_exists(dst_container_name) - - # Do the rename of the file/dir - headers = { 'x-copy-from': "/%s/%s" % (src_container_name, src_path) } - self.conn.put_object(dst_container_name, dst_path, headers=headers, contents=None) - # Delete src - self.conn.delete_object(src_container_name, src_path) - self._listdir_cache.flush(posixpath.dirname(src)) - self._listdir_cache.flush(posixpath.dirname(dst)) - - def chmod(self, path, mode): - """Change file/directory mode""" - e = "chmod %03o %r - not implemented" % (mode, path) - logging.debug(e) - raise IOSError(EPERM, 'Operation not permitted: %s' % e) - - def isfile(self, path): - """ - Is this path a file. - - Shouldn't raise an error if not found like os.path.isfile. - """ - logging.debug("isfile %r" % path) - try: - return stat.S_ISREG(self.stat(path).st_mode) - except EnvironmentError: - return False - - def islink(self, path): - """ - Is this path a link. - - Shouldn't raise an error if not found like os.path.islink. - """ - logging.debug("islink %r" % path) - return False - - def isdir(self, path): - """ - Is this path a directory. - - Shouldn't raise an error if not found like os.path.isdir. - """ - logging.debug("isdir %r" % path) - try: - return stat.S_ISDIR(self.stat(path).st_mode) - except EnvironmentError: - return False - - def getsize(self, path): - """ - Return the size of path. - - Raises OSError on error. - """ - logging.debug("getsize %r" % path) - return self.stat(path).st_size - - def getmtime(self, path): - """ - Return the modification time of path. - - Raises OSError on error. - """ - logging.debug("getmtime %r" % path) - return self.stat(path).st_mtime - - def realpath(self, path): - """Return the canonical path of the specified path""" - return self.abspath(path) - - def lexists(self, path): - """ - Test whether a path exists. - - Returns True for broken symbolic links. - """ - logging.debug("lexists %r" % path) - try: - self.stat(path) - return True - except EnvironmentError: - return False - - @translate_objectstorage_error - def stat(self, path): - """ - Return os.stat_result object for path. - - Raises OSError on error. - """ - path = self.abspath(path) - logging.debug("stat %r" % path) - return self._listdir_cache.stat(path) - - exists = lexists - lstat = stat - - def validpath(self, path): - """Check whether the path belongs to user's home directory""" - return True - - def flush(self): - """Flush cache""" - if self._listdir_cache: - self._listdir_cache.flush() - - def get_user_by_uid(self, uid): - """ - Return the username associated with user id. - - If this can't be determined return raw uid instead. - """ - return self.username - - def get_group_by_gid(self, gid): - """ - Return the groupname associated with group id. - - If this can't be determined return raw gid instead. - On Windows just return "group". - """ - return self.username - - def readlink(self, path): - """ - Return a string representing the path to which a symbolic link points. - - We never return that we have a symlink in stat, so this should - never be called. - """ - e = "readlink %r - not implemented" % path - logging.debug(e) - raise IOSError(EPERM, 'Operation not permitted: %s' % e) - - @translate_objectstorage_error - def md5(self, path): - """ - Return the object MD5 for path. - - Raise OSError on error. - """ - path = self.abspath(path) - logging.debug("md5 %r" % path) - container, name = parse_fspath(path) - - if not name: - raise IOSError(EACCES, "Can't return the MD5 of a container") - - if self.isdir(path): - # this is only 100% accurate for virtual directories - raise IOSError(EACCES, "Can't return the MD5 of a directory") - - meta = self.conn.head_object(container, name) - return meta["etag"] diff --git a/ftpcloudfs/main.py b/ftpcloudfs/main.py deleted file mode 100644 index fdad863..0000000 --- a/ftpcloudfs/main.py +++ /dev/null @@ -1,329 +0,0 @@ -# -*- encoding: utf-8 -*- -__author__ = "Chmouel Boudjnah " - -import sys -import os -import signal -import socket -from ConfigParser import RawConfigParser -import logging -from logging.handlers import SysLogHandler - -from optparse import OptionParser -import pyftpdlib.servers - -from server import ObjectStorageFtpFS -from fs import ObjectStorageFD -from constants import version, default_address, default_port, \ - default_config_file, default_banner, \ - default_ks_tenant_separator, default_ks_service_type, default_ks_endpoint_type -from monkeypatching import MyFTPHandler -from multiprocessing import Manager - -def modify_supported_ftp_commands(): - """Remove the FTP commands we don't / can't support, and add the extensions.""" - unsupported = ( - 'SITE CHMOD', - ) - for cmd in unsupported: - if cmd in pyftpdlib.handlers.proto_cmds: - del pyftpdlib.handlers.proto_cmds[cmd] - # add the MD5 command, FTP extension according to IETF Draft: - # http://tools.ietf.org/html/draft-twine-ftpmd5-00 - pyftpdlib.handlers.proto_cmds.update({ - 'MD5': dict(perm=None, - auth=True, - arg=True, - help=u'Syntax: MD5 file-name (get MD5 of file)') - }) - -class Main(object): - """ftp-cloudfs: A FTP Proxy Interface to OpenStack Object Storage (Swift).""" - - def __init__(self): - self.options = None - - def setup_log(self): - """Setup Logging.""" - - if self.options.log_level: - self.options.log_level = logging.DEBUG - else: - self.options.log_level = logging.INFO - - if self.options.syslog: - logger = logging.getLogger() - try: - handler = SysLogHandler(address='/dev/log', - facility=SysLogHandler.LOG_DAEMON) - except IOError: - # fall back to UDP - handler = SysLogHandler(facility=SysLogHandler.LOG_DAEMON) - finally: - prefix = "%s[%s]: " % (__package__, self.pid) - formatter = logging.Formatter(prefix + "%(message)s") - handler.setFormatter(formatter) - logger.addHandler(handler) - logger.setLevel(self.options.log_level) - else: - log_format = '%(asctime)-15s - %(levelname)s - %(message)s' - logging.basicConfig(filename=self.options.log_file, - format=log_format, - level=self.options.log_level) - - # warnings - if self.config.get("ftpcloudfs", "workers") is not None: - logging.warning("workers configuration token has been deprecated and has no effect") - if self.config.get("ftpcloudfs", "service-net") is not None: - logging.warning("service-net configuration token has been deprecated and has no effect (see ChangeLog)") - - def parse_configuration(self, config_file=default_config_file): - """Parse the configuration file""" - config = RawConfigParser({'banner': default_banner, - 'port': default_port, - 'bind-address': default_address, - 'workers': None, - 'memcache': None, - 'max-cons-per-ip': '0', - 'auth-url': None, - 'service-net': None, - 'verbose': 'no', - 'syslog': 'no', - 'log-file': None, - 'pid-file': None, - 'uid': None, - 'gid': None, - 'masquerade-firewall': None, - 'split-large-files': '0', - # keystone auth 2.0 support - 'keystone-auth': False, - 'keystone-region-name': None, - 'keystone-tenant-separator': default_ks_tenant_separator, - 'keystone-service-type': default_ks_service_type, - 'keystone-endpoint-type': default_ks_endpoint_type, - }) - config.read(default_config_file) - if not config.has_section('ftpcloudfs'): - config.add_section('ftpcloudfs') - - self.config = config - - def parse_arguments(self): - """Parse command line options""" - parser = OptionParser(version="%prog " + version) - parser.add_option('-p', '--port', - type="int", - dest="port", - default=self.config.get('ftpcloudfs', 'port'), - help="Port to bind the server (default: %d)" % \ - (default_port)) - - parser.add_option('-b', '--bind-address', - type="str", - dest="bind_address", - default=self.config.get('ftpcloudfs', 'bind-address'), - help="Address to bind (default: %s)" % \ - (default_address)) - - parser.add_option('-a', '--auth-url', - type="str", - dest="authurl", - default=self.config.get('ftpcloudfs', 'auth-url'), - help="Authentication URL (required)") - - memcache = self.config.get('ftpcloudfs', 'memcache') - if memcache: - memcache = [x.strip() for x in memcache.split(',')] - parser.add_option('--memcache', - type="str", - dest="memcache", - action="append", - default=memcache, - help="Memcache server(s) to be used for cache (ip:port)") - - parser.add_option('-v', '--verbose', - action="store_true", - dest="log_level", - default=self.config.getboolean('ftpcloudfs', 'verbose'), - help="Be verbose on logging") - - parser.add_option('-f', '--foreground', - action="store_true", - dest="foreground", - default=False, - help="Do not attempt to daemonize but run in foreground") - - parser.add_option('-l', '--log-file', - type="str", - dest="log_file", - default=self.config.get('ftpcloudfs', 'log-file'), - help="Log File: Default stdout when in foreground") - - parser.add_option('--syslog', - action="store_true", - dest="syslog", - default=self.config.getboolean('ftpcloudfs', 'syslog'), - help="Enable logging to the system logger " + \ - "(daemon facility)") - - parser.add_option('--pid-file', - type="str", - dest="pid_file", - default=self.config.get('ftpcloudfs', 'pid-file'), - help="Pid file location when in daemon mode") - - parser.add_option('--uid', - type="int", - dest="uid", - default=self.config.get('ftpcloudfs', 'uid'), - help="UID to drop the privilige to when in daemon mode") - - parser.add_option('--gid', - type="int", - dest="gid", - default=self.config.get('ftpcloudfs', 'gid'), - help="GID to drop the privilige to when in daemon mode") - - parser.add_option('--keystone-auth', - action="store_true", - dest="keystone", - default=self.config.get('ftpcloudfs', 'keystone-auth'), - help="Use auth 2.0 (Keystone, requires keystoneclient)") - - parser.add_option('--keystone-region-name', - type="str", - dest="region_name", - default=self.config.get('ftpcloudfs', 'keystone-region-name'), - help="Region name to be used in auth 2.0") - - parser.add_option('--keystone-tenant-separator', - type="str", - dest="tenant_separator", - default=self.config.get('ftpcloudfs', 'keystone-tenant-separator'), - help="Character used to separate tenant_name/username in auth 2.0" + \ - " (default: TENANT%sUSERNAME)" % default_ks_tenant_separator) - - parser.add_option('--keystone-service-type', - type="str", - dest="service_type", - default=self.config.get('ftpcloudfs', 'keystone-service-type'), - help="Service type to be used in auth 2.0 (default: %s)" % default_ks_service_type) - - parser.add_option('--keystone-endpoint-type', - type="str", - dest="endpoint_type", - default=self.config.get('ftpcloudfs', 'keystone-endpoint-type'), - help="Endpoint type to be used in auth 2.0 (default: %s)" % default_ks_endpoint_type) - - (options, _) = parser.parse_args() - - if options.keystone: - try: - from keystoneclient.v2_0 import client as _test_ksclient - except ImportError: - parser.error("Auth 2.0 (keystone) requires python-keystoneclient.") - keystone_keys = ('region_name', 'tenant_separator', 'service_type', 'endpoint_type') - options.keystone = dict((key, getattr(options, key)) for key in keystone_keys) - - if not options.authurl: - parser.error("An authentication URL is required and it wasn't provided") - - self.options = options - - def setup_server(self): - """Run the main ftp server loop.""" - banner = self.config.get('ftpcloudfs', 'banner').replace('%v', version) - banner = banner.replace('%f', pyftpdlib.__ver__) - - MyFTPHandler.banner = banner - ObjectStorageFtpFS.authurl = self.options.authurl - ObjectStorageFtpFS.keystone = self.options.keystone - ObjectStorageFtpFS.memcache_hosts = self.options.memcache - - try: - # store bytes - ObjectStorageFD.split_size = int(self.config.get('ftpcloudfs', 'split-large-files'))*10**6 - except ValueError, errmsg: - sys.exit('Split large files error: %s' % errmsg) - - masquerade = self.config.get('ftpcloudfs', 'masquerade-firewall') - if masquerade: - try: - MyFTPHandler.masquerade_address = socket.gethostbyname(masquerade) - except socket.gaierror, (_, errmsg): - sys.exit('Masquerade address error: %s' % errmsg) - - try: - max_cons_per_ip = int(self.config.get('ftpcloudfs', 'max-cons-per-ip')) - except ValueError, errmsg: - sys.exit('Max connections per IP error: %s' % errmsg) - - ftpd = pyftpdlib.servers.MultiprocessFTPServer((self.options.bind_address, - self.options.port), - MyFTPHandler, - ) - - # set it to unlimited, we use our own checks with a shared dict - ftpd.max_cons_per_ip = 0 - ftpd.handler.max_cons_per_ip = max_cons_per_ip - - return ftpd - - def setup_daemon(self, preserve=None): - """Setup the daemon context for the server.""" - import daemon - from utils import PidFile - import tempfile - - daemonContext = daemon.DaemonContext() - - if not self.options.pid_file: - self.options.pid_file = "%s/ftpcloudfs.pid" % \ - (tempfile.gettempdir()) - - self.pidfile = PidFile(self.options.pid_file) - daemonContext.pidfile = self.pidfile - if self.options.uid: - daemonContext.uid = self.options.uid - - if self.options.gid: - daemonContext.gid = self.options.gid - - if preserve: - daemonContext.files_preserve = preserve - - return daemonContext - - def signal_handler(self, signal, frame): - """Catch signals and propagate them to child processes.""" - if self.shm_manager: - self.shm_manager.shutdown() - self.shm_manager = None - self.old_signal_handler(signal, frame) - - def main(self): - """Main entry point.""" - self.pid = os.getpid() - self.parse_configuration() - self.parse_arguments() - modify_supported_ftp_commands() - - ftpd = self.setup_server() - - if self.options.foreground: - MyFTPHandler.shared_ip_map = None - self.setup_log() - ftpd.serve_forever() - return - - daemonContext = self.setup_daemon([ftpd.socket.fileno(), ftpd.ioloop.fileno(),]) - with daemonContext: - self.old_signal_handler = signal.signal(signal.SIGTERM, self.signal_handler) - - self.shm_manager = Manager() - MyFTPHandler.shared_ip_map = self.shm_manager.dict() - MyFTPHandler.shared_lock = self.shm_manager.Lock() - - self.setup_log() - ftpd.serve_forever() - diff --git a/ftpcloudfs/monkeypatching.py b/ftpcloudfs/monkeypatching.py deleted file mode 100644 index 59f47e7..0000000 --- a/ftpcloudfs/monkeypatching.py +++ /dev/null @@ -1,117 +0,0 @@ -import sys -import socket -from pyftpdlib.handlers import DTPHandler, FTPHandler, _strerror -from ftpcloudfs.utils import smart_str -from server import ObjectStorageAuthorizer -from multiprocessing.managers import RemoteError - -class MyDTPHandler(DTPHandler): - def send(self, data): - data = smart_str(data) - return DTPHandler.send(self, data) - - def close(self): - if self.file_obj is not None and not self.file_obj.closed: - try: - self.file_obj.close() - except Exception, e: - msg = u"Data connection error (%s)" % e - self.cmd_channel.log(msg) - self.cmd_channel.respond(u"421 " + msg) - finally: - self.file_obj = None - - DTPHandler.close(self) - -class MyFTPHandler(FTPHandler): - # don't kick off client in long time transactions - timeout = 0 - dtp_handler = MyDTPHandler - authorizer = ObjectStorageAuthorizer() - max_cons_per_ip = 0 - use_sendfile = False - - @staticmethod - def abstracted_fs(root, cmd_channel): - """Get an AbstractedFs for the user logged in on the cmd_channel.""" - cffs = cmd_channel.authorizer.get_abstracted_fs(cmd_channel.username) - cffs.init_abstracted_fs(root, cmd_channel) - return cffs - - def process_command(self, cmd, *args, **kwargs): - """ - Flush the FS cache with every new FTP command (non-shared cache). - - Also track the remote ip to set the X-Forwarded-For header. - """ - if self.fs: - if self.fs.memcache_hosts is None: - self.fs.flush() - self.fs.conn.real_ip = self.remote_ip - FTPHandler.process_command(self, cmd, *args, **kwargs) - - def ftp_MD5(self, path): - line = self.fs.fs2ftp(path) - try: - md5_checksum = self.run_as_current_user(self.fs.md5, path) - except OSError, err: - why = _strerror(err) - self.respond('550 %s.' % why) - else: - msg = md5_checksum.upper() - self.respond('251 "%s" %s' % (line.replace('"', '""'), msg)) - - def handle(self): - """Track the ip and check max cons per ip (if needed).""" - if self.max_cons_per_ip and self.remote_ip and self.shared_ip_map != None: - count = 0 - try: - self.shared_lock.acquire() - count = self.shared_ip_map.get(self.remote_ip, 0) + 1 - self.shared_ip_map[self.remote_ip] = count - self.logline("Connected, shared ip map: %s" % self.shared_ip_map) - except RemoteError, e: - self.logerror("Connection tracking failed: %s" % e) - finally: - self.shared_lock.release() - - self.logline("Connection track: %s -> %s" % (self.remote_ip, count)) - - if count > self.max_cons_per_ip: - self.handle_max_cons_per_ip() - return - - FTPHandler.handle(self) - - def handle_error(self): - """Catch some 'expected' exceptions not processed by FTPHandler/AsyncChat.""" - # this is aesthetic only - t, v, _ = sys.exc_info() - if t == socket.error: - self.log("Connection error: %s" % v) - self.handle_close() - return - - FTPHandler.handle_error(self) - - def close(self): - """Remove the ip from the shared map before calling close.""" - if not self._closed and self.max_cons_per_ip and self.shared_ip_map != None: - try: - self.shared_lock.acquire() - if self.remote_ip in self.shared_ip_map: - self.shared_ip_map[self.remote_ip] -= 1 - if self.shared_ip_map[self.remote_ip] <= 0: - del self.shared_ip_map[self.remote_ip] - self.logline("Disconnected, shared ip map: %s" % self.shared_ip_map) - except RemoteError, e: - self.logerror("Connection tracking cleanup failed: %s" % e) - finally: - self.shared_lock.release() - - - FTPHandler.close(self) - - # We want to log more commands. - log_cmds_list = ["ABOR", "APPE", "DELE", "RMD", "RNFR", "RNTO", "RETR", "STOR", "MKD",] - diff --git a/ftpcloudfs/server.py b/ftpcloudfs/server.py deleted file mode 100644 index 353440f..0000000 --- a/ftpcloudfs/server.py +++ /dev/null @@ -1,78 +0,0 @@ -#/usr/bin/env python -# -# Authors: Chmouel Boudjnah -# Juan J. Martinez -# -import os - -from pyftpdlib.filesystems import AbstractedFS -from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed - -from fs import ObjectStorageFS - -class ObjectStorageFtpFS(ObjectStorageFS, AbstractedFS): - """Object Storage File system emulation for a FTP server.""" - servicenet = False - authurl = None - keystone = None - - def __init__(self, username, api_key, authurl=None, keystone=None): - ObjectStorageFS.__init__(self, - username, - api_key, - authurl=authurl or self.authurl, - keystone=keystone or self.keystone, - ) - - def init_abstracted_fs(self, root, cmd_channel): - AbstractedFS.__init__(self, root, cmd_channel) - -class ObjectStorageAuthorizer(DummyAuthorizer): - """ - FTP server authorizer. - - Logs the users into the object storage and keeps track of them. - """ - users = {} - abstracted_fs_for_user = {} - - def validate_authentication(self, username, password, handler): - """ - Validates the username and passwords. - - This creates the AbstractedFS at the same time and caches it under the username for retrieval with get_abstracted_fs. - """ - try: - cffs = ObjectStorageFtpFS(username, password) - except EnvironmentError, e: - msg = "Failed to authenticate user %s: %s" % (username, e) - handler.logerror(msg) - raise AuthenticationFailed(msg) - self.abstracted_fs_for_user[username] = cffs - handler.log("Authentication validated for user %s" % username) - - def get_abstracted_fs(self, username): - """ - Gets an AbstractedFs object for the user. - - Raises KeyError if username isn't found. - """ - return self.abstracted_fs_for_user.pop(username) - - def has_user(self, username): - return username != 'anonymous' - - def has_perm(self, username, perm, path=None): - return True - - def get_perms(self, username): - return u'lrdw' - - def get_home_dir(self, username): - return unicode(os.sep) - - def get_msg_login(self, username): - return u'Welcome %s' % username - - def get_msg_quit(self, username): - return u'Goodbye %s' % username diff --git a/ftpcloudfs/utils.py b/ftpcloudfs/utils.py deleted file mode 100644 index ac69959..0000000 --- a/ftpcloudfs/utils.py +++ /dev/null @@ -1,49 +0,0 @@ -import types -import fcntl -import os - -class PidFile(object): - """Context manager that locks a pid file.""" - def __init__(self, path): - self.path = path - self.pidfile = None - - def close(self): - pidfile = self.pidfile - self.pidfile = None - pidfile.close() - - def __enter__(self): - self.pidfile = open(self.path, "a+") - fcntl.flock(self.pidfile.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - self.pidfile.seek(0) - self.pidfile.truncate() - self.pidfile.write(str(os.getpid())) - self.pidfile.flush() - self.pidfile.seek(0) - return self.pidfile - - def __exit__(self, exc_type=None, exc_value=None, exc_tb=None): - if self.pidfile: - self.pidfile.close() - os.remove(self.path) - -#from django.utils -def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): - if strings_only and isinstance(s, (types.NoneType, int)): - return s - elif not isinstance(s, basestring): - try: - return str(s) - except UnicodeEncodeError: - if isinstance(s, Exception): - return ' '.join([smart_str(arg, encoding, strings_only, - errors) for arg in s]) - return unicode(s).encode(encoding, errors) - elif isinstance(s, unicode): - return s.encode(encoding, errors) - elif s and encoding != 'utf-8': - return s.decode('utf-8', errors).encode(encoding, errors) - else: - return s - diff --git a/setup.py b/setup.py deleted file mode 100755 index ebade52..0000000 --- a/setup.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -import os -from setuptools import setup, find_packages -from ftpcloudfs.constants import version - -def read(fname): - full_path = os.path.join(os.path.dirname(__file__), fname) - if os.path.exists(fname): - return open(full_path).read() - else: - return "" - -setup(name='ftp-cloudfs', - version=version, - download_url="http://pypi.python.org/packages/source/f/ftp-cloudfs/ftp-cloudfs-%s.tar.gz" % (version), - description='FTP interface to OpenStack Object Storage (Swift)', - author='Chmouel Boudjnah', - author_email='chmouel@chmouel.com', - url='https://pypi.python.org/pypi/ftp-cloudfs/', - long_description = read('README.rst'), - license='MIT', - include_package_data=True, - zip_safe=False, - install_requires=['pyftpdlib>=1.3.0', 'python-swiftclient>=1.6.0', 'python-daemon>=1.5.5', 'python-memcached'], - scripts=['bin/ftpcloudfs'], - packages = find_packages(exclude=['tests',]), - tests_require = ["nose"], - classifiers = [ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Programming Language :: Python', - 'Operating System :: OS Independent', - 'Environment :: No Input/Output (Daemon)', - 'License :: OSI Approved :: MIT License', - ], - test_suite = "nose.collector", - ) diff --git a/tests/README b/tests/README deleted file mode 100644 index 2afd03d..0000000 --- a/tests/README +++ /dev/null @@ -1,10 +0,0 @@ -To run the tests you'll need access to an Openstack Object Storage server. - -Set these environment variables before running the tests - - export OS_API_USER='user' - export OS_API_KEY='key' - export OS_AUTH_URL='https://url.of.auth.server/v1.0' - -Once your test env is correct, all tests must pass! - diff --git a/tests/test_fs.py b/tests/test_fs.py deleted file mode 100644 index bb76e55..0000000 --- a/tests/test_fs.py +++ /dev/null @@ -1,638 +0,0 @@ -#!/usr/bin/python -import unittest -import os -import sys -from datetime import datetime -from swiftclient import client -from ftpcloudfs.fs import ObjectStorageFS, ListDirCache -from ftpcloudfs.errors import IOSError - -#import logging -#logging.getLogger().setLevel(logging.DEBUG) -#logging.basicConfig(level=logging.DEBUG) - -class ObjectStorageFSTest(unittest.TestCase): - '''ObjectStorageFS Tests''' - - def setUp(self): - if not hasattr(self, 'username'): - cls = self.__class__ - if not all(['OS_API_KEY' in os.environ, - 'OS_API_USER' in os.environ, - 'OS_AUTH_URL' in os.environ, - ]): - print "env OS_API_USER/OS_API_KEY/OS_AUTH_URL not found." - sys.exit(1) - cls.username = os.environ['OS_API_USER'] - cls.api_key = os.environ['OS_API_KEY'] - cls.auth_url = os.environ.get('OS_AUTH_URL') - cls.cnx = ObjectStorageFS(self.username, self.api_key, self.auth_url) - cls.conn = client.Connection(user=self.username, key=self.api_key, authurl=self.auth_url) - self.container = "ftpcloudfs_testing" - self.cnx.mkdir("/%s" % self.container) - self.cnx.chdir("/%s" % self.container) - - def create_file(self, path, contents): - '''Create path with contents''' - fd = self.cnx.open(path, "wb") - fd.write(contents) - fd.close() - - def read_file(self, path): - fd = self.cnx.open(path, "rb") - contents = '' - while True: - chunk = fd.read() - if not chunk: - break - contents += chunk - fd.close() - return contents - - def test_mkdir_chdir_rmdir(self): - ''' mkdir/chdir/rmdir directory ''' - directory = "/foobarrandom" - self.cnx.mkdir(directory) - self.cnx.chdir(directory) - self.assertEqual(self.cnx.getcwd(), directory) - self.assertEqual(self.cnx.listdir(directory), []) - self.cnx.rmdir(directory) - - def test_mkdir_chdir_mkdir_rmdir_subdir(self): - ''' mkdir/chdir/rmdir sub directory ''' - directory = "/foobarrandom" - self.cnx.mkdir(directory) - self.cnx.chdir(directory) - subdirectory = "potato" - subdirpath = directory + "/" + subdirectory - self.cnx.mkdir(subdirectory) - # Can't delete a directory with stuff in - self.assertRaises(EnvironmentError, self.cnx.rmdir, directory) - self.cnx.chdir(subdirectory) - self.cnx.chdir("..") - self.assertEqual(self.cnx.getcwd(), directory) - self.cnx.rmdir(subdirectory) - self.cnx.chdir("..") - self.cnx.rmdir(directory) - - def test_write_open_delete(self): - ''' write/open/delete file ''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.assertEquals(self.cnx.getsize("testfile.txt"), len(content_string)) - contents = self.read_file("testfile.txt") - self.assertEqual(contents, content_string) - self.cnx.remove("testfile.txt") - - def test_write_open_delete_subdir(self): - ''' write/open/delete file in a subdirectory''' - self.cnx.mkdir("potato") - self.cnx.chdir("potato") - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.assertEquals(self.cnx.getsize("testfile.txt"), len(content_string)) - content = self.read_file("/%s/potato/testfile.txt" % self.container) - self.assertEqual(content, content_string) - self.cnx.remove("testfile.txt") - self.cnx.chdir("..") - self.cnx.rmdir("potato") - - def test_write_to_slash(self): - ''' write to slash should not be permitted ''' - self.cnx.chdir("/") - content_string = "Hello Moto" - self.assertRaises(EnvironmentError, self.create_file, "testfile.txt", content_string) - - def test_chdir_to_a_file(self): - ''' chdir to a file ''' - self.create_file("testfile.txt", "Hello Moto") - self.assertRaises(EnvironmentError, self.cnx.chdir, "/%s/testfile.txt" % self.container) - self.cnx.remove("testfile.txt") - - def test_chdir_to_slash(self): - ''' chdir to slash ''' - self.cnx.chdir("/") - - def test_chdir_to_nonexistent_container(self): - ''' chdir to non existent container''' - self.assertRaises(EnvironmentError, self.cnx.chdir, "/i_dont_exist") - - def test_chdir_to_nonexistent_directory(self): - ''' chdir to nonexistend directory''' - self.assertRaises(EnvironmentError, self.cnx.chdir, "i_dont_exist") - self.assertRaises(EnvironmentError, self.cnx.chdir, "/%s/i_dont_exist" % self.container) - - def test_listdir_root(self): - ''' list root directory ''' - self.cnx.chdir("/") - dt = abs(datetime.utcfromtimestamp(self.cnx.getmtime("/")) - datetime.utcnow()) - self.assertTrue(dt.seconds < 60) - ls = self.cnx.listdir(".") - self.assertTrue(self.container in ls) - dt = abs(datetime.utcfromtimestamp(self.cnx.getmtime(self.container)) - datetime.utcnow()) - self.assertTrue(dt.seconds < 60) - self.assertTrue('potato' not in ls) - self.cnx.mkdir("potato") - ls = self.cnx.listdir(".") - self.assertTrue(self.container in ls) - self.assertTrue('potato' in ls) - self.cnx.rmdir("potato") - - def test_listdir(self): - ''' list directory ''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - dt = abs(datetime.utcfromtimestamp(self.cnx.getmtime("testfile.txt")) - datetime.utcnow()) - self.assertTrue(dt.seconds < 60) - self.assertEqual(self.cnx.listdir("."), ["testfile.txt"]) - self.cnx.remove("testfile.txt") - - def test_listdir_subdir(self): - ''' list a sub directory''' - content_string = "Hello Moto" - self.create_file("1.txt", content_string) - self.create_file("2.txt", content_string) - self.cnx.mkdir("potato") - self.create_file("potato/3.txt", content_string) - self.create_file("potato/4.txt", content_string) - self.assertEqual(self.cnx.listdir("."), ["1.txt", "2.txt", "potato"]) - self.cnx.chdir("potato") - self.assertEqual(self.cnx.listdir("."), ["3.txt", "4.txt"]) - self.cnx.remove("3.txt") - self.cnx.remove("4.txt") - self.assertEqual(self.cnx.listdir("."), []) - self.cnx.chdir("..") - self.cnx.remove("1.txt") - self.cnx.remove("2.txt") - self.assertEqual(self.cnx.listdir("."), ["potato"]) - dt = abs(datetime.utcfromtimestamp(self.cnx.getmtime("potato")) - datetime.utcnow()) - self.assertTrue(dt.seconds < 60) - self.cnx.rmdir("potato") - self.assertEqual(self.cnx.listdir("."), []) - - def test_rename_file(self): - '''rename a file''' - content_string = "Hello Moto" * 100 - self.create_file("testfile.txt", content_string) - self.assertEquals(self.cnx.getsize("testfile.txt"), len(content_string)) - self.assertRaises(EnvironmentError, self.cnx.getsize, "testfile2.txt") - self.cnx.rename("testfile.txt", "testfile2.txt") - self.assertEquals(self.cnx.getsize("testfile2.txt"), len(content_string)) - self.assertRaises(EnvironmentError, self.cnx.getsize, "testfile.txt") - self.cnx.remove("testfile2.txt") - - def test_rename_file_into_subdir1(self): - '''rename a file into a subdirectory 1''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.cnx.mkdir("potato") - self.assertEquals(self.cnx.getsize("testfile.txt"), len(content_string)) - self.assertRaises(EnvironmentError, self.cnx.getsize, "potato/testfile3.txt") - self.cnx.rename("testfile.txt", "potato/testfile3.txt") - self.assertEquals(self.cnx.getsize("potato/testfile3.txt"), len(content_string)) - self.assertRaises(EnvironmentError, self.cnx.getsize, "testfile.txt") - self.cnx.remove("potato/testfile3.txt") - self.cnx.rmdir("potato") - - def test_rename_file_into_subdir2(self): - '''rename a file into a subdirectory without specifying dest leaf''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.cnx.mkdir("potato") - self.assertEquals(self.cnx.getsize("testfile.txt"), len(content_string)) - self.assertRaises(EnvironmentError, self.cnx.getsize, "potato/testfile.txt") - self.cnx.rename("testfile.txt", "potato") - self.assertEquals(self.cnx.getsize("potato/testfile.txt"), len(content_string)) - self.assertRaises(EnvironmentError, self.cnx.getsize, "testfile.txt") - self.cnx.remove("potato/testfile.txt") - self.cnx.rmdir("potato") - - def test_rename_file_into_root(self): - '''rename a file into a subdirectory without specifying dest leaf''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.assertRaises(EnvironmentError, self.cnx.rename, "testfile.txt", "/testfile.txt") - self.cnx.remove("testfile.txt") - - def test_rename_directory_into_file(self): - '''rename a directory into a file - shouldn't work''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.assertRaises(EnvironmentError, self.cnx.rename, "/%s" % self.container, "testfile.txt") - self.cnx.remove("testfile.txt") - - def test_rename_directory_into_directory(self): - '''rename a directory into a directory''' - self.cnx.mkdir("potato") - self.assertEquals(self.cnx.listdir("potato"), []) - self.cnx.rename("potato", "potato2") - self.assertEquals(self.cnx.listdir("potato2"), []) - self.cnx.rmdir("potato2") - - def test_rename_directory_into_existing_directory(self): - '''rename a directory into an existing directory''' - self.cnx.mkdir("potato") - self.cnx.mkdir("potato2") - self.assertEquals(self.cnx.listdir("potato"), []) - self.assertEquals(self.cnx.listdir("potato2"), []) - self.cnx.rename("potato", "potato2") - self.assertEquals(self.cnx.listdir("potato2"), ["potato"]) - self.assertEquals(self.cnx.listdir("potato2/potato"), []) - self.cnx.rmdir("potato2/potato") - self.cnx.rmdir("potato2") - - def test_rename_directory_into_self(self): - '''rename a directory into itself''' - self.cnx.mkdir("potato") - self.assertEquals(self.cnx.listdir("potato"), []) - self.cnx.rename("potato", "/%s" % self.container) - self.assertEquals(self.cnx.listdir("potato"), []) - self.cnx.rename("potato", "/%s/potato" % self.container) - self.assertEquals(self.cnx.listdir("potato"), []) - self.cnx.rename("potato", "potato") - self.assertEquals(self.cnx.listdir("potato"), []) - self.cnx.rename("/%s/potato" % self.container, ".") - self.assertEquals(self.cnx.listdir("potato"), []) - self.cnx.rmdir("potato") - - def test_rename_full_directory(self): - '''rename a directory into a directory''' - self.cnx.mkdir("potato") - self.create_file("potato/something.txt", "p") - try: - self.assertEquals(self.cnx.listdir("potato"), ["something.txt"]) - self.assertRaises(EnvironmentError, self.cnx.rename, "potato", "potato2") - finally: - self.cnx.remove("potato/something.txt") - self.cnx.rmdir("potato") - - def test_rename_container(self): - '''rename an empty container''' - self.cnx.mkdir("/potato") - self.assertEquals(self.cnx.listdir("/potato"), []) - self.assertRaises(EnvironmentError, self.cnx.listdir, "/potato2") - self.cnx.rename("/potato", "/potato2") - self.assertRaises(EnvironmentError, self.cnx.listdir, "/potato") - self.assertEquals(self.cnx.listdir("/potato2"), []) - self.cnx.rmdir("/potato2") - - def test_rename_full_container(self): - '''rename a full container''' - self.cnx.mkdir("/potato") - self.create_file("/potato/test.txt", "onion") - self.assertEquals(self.cnx.listdir("/potato"), ["test.txt"]) - self.assertRaises(EnvironmentError, self.cnx.rename, "/potato", "/potato2") - self.cnx.remove("/potato/test.txt") - self.cnx.rmdir("/potato") - - def test_unicode_file(self): - '''Test unicode file creation''' - # File names use a utf-8 interface - file_name = u"Smiley\u263a.txt".encode("utf-8") - self.create_file(file_name, "Hello Moto") - self.assertEqual(self.cnx.listdir("."), [unicode(file_name, "utf-8")]) - self.cnx.remove(file_name) - - def test_unicode_directory(self): - '''Test unicode directory creation''' - # File names use a utf-8 interface - dir_name = u"Smiley\u263aDir".encode("utf-8") - self.cnx.mkdir(dir_name) - self.assertEqual(self.cnx.listdir("."), [unicode(dir_name, "utf-8")]) - self.cnx.rmdir(dir_name) - - def test_mkdir_container_unicode(self): - ''' mkdir/chdir/rmdir directory ''' - directory = u"/Smiley\u263aContainer".encode("utf-8") - self.cnx.mkdir(directory) - self.cnx.chdir(directory) - self.cnx.rmdir(directory) - - def test_fakedir(self): - '''Make some fake directories and test''' - - objs = [ "test1.txt", "potato/test2.txt", "potato/sausage/test3.txt", "potato/sausage/test4.txt", ] - for obj in objs: - self.conn.put_object(self.container, obj, content_type="text/plain", contents="Hello Moto") - - self.assertEqual(self.cnx.listdir("."), ["potato", "test1.txt"]) - self.assertEqual(self.cnx.listdir("potato"), ["sausage","test2.txt"]) - self.assertEqual(self.cnx.listdir("potato/sausage"), ["test3.txt", "test4.txt"]) - - self.cnx.chdir("potato") - - self.assertEqual(self.cnx.listdir("."), ["sausage","test2.txt"]) - self.assertEqual(self.cnx.listdir("sausage"), ["test3.txt", "test4.txt"]) - - self.cnx.chdir("sausage") - - self.assertEqual(self.cnx.listdir("."), ["test3.txt", "test4.txt"]) - - self.cnx.chdir("../..") - - for obj in objs: - self.conn.delete_object(self.container, obj) - - self.assertEqual(self.cnx.listdir("."), []) - - def test_md5(self): - self.conn.put_object(self.container, "test1.txt", content_type="text/plain", contents="Hello Moto") - self.assertEquals(self.cnx.md5("test1.txt"),"0d933ae488fd55cc6bdeafffbaabf0c4") - self.cnx.remove("test1.txt") - self.assertRaises(EnvironmentError, self.cnx.md5, "/") - self.assertRaises(EnvironmentError, self.cnx.md5, "/%s" % self.container) - self.cnx.mkdir("/%s/sausage" % self.container) - self.assertRaises(EnvironmentError, self.cnx.md5, "/%s/sausage" % self.container) - self.cnx.rmdir("/%s/sausage" % self.container) - - def test_listdir_manifest(self): - ''' list directory including a manifest file ''' - content_string = "0" * 1024 - for i in range(1, 5): - self.create_file("testfile.part/%d" % i, content_string) - self.conn.put_object(self.container, "testfile", contents=None, headers={ "x-object-manifest": '%s/testfile.part' % self.container }) - self.assertEqual(self.cnx.listdir("."), ["testfile", "testfile.part"]) - self.assertEqual(self.cnx.getsize("testfile"), 4096) - self.cnx.remove("testfile") - for i in range(1, 5): - self.cnx.remove("testfile.part/%d" % i) - - def test_seek_set_resume(self): - ''' seek/resume functionality (seek_set) ''' - content_string = "This is a chunk of data"*1024 - self.create_file("testfile.txt", content_string) - self.assertEquals(self.cnx.getsize("testfile.txt"), len(content_string)) - - fd = self.cnx.open("testfile.txt", "rb") - contents = fd.read(1024) - fd.close() - - fd = self.cnx.open("testfile.txt", "rb") - fd.seek(1024) - contents += fd.read(512) - fd.close() - - fd = self.cnx.open("testfile.txt", "rb") - fd.seek(1024+512) - contents += fd.read() - fd.close() - - self.assertEqual(contents, content_string) - self.cnx.remove("testfile.txt") - - def test_seek_end_resume(self): - ''' seek/resume functionality (seek_end) ''' - content_string = "This is another chunk of data"*1024 - self.create_file("testfile.txt", content_string) - self.assertEquals(self.cnx.getsize("testfile.txt"), len(content_string)) - - fd = self.cnx.open("testfile.txt", "rb") - contents = fd.read(len(content_string)-1024) - fd.close() - - fd = self.cnx.open("testfile.txt", "rb") - fd.seek(1024, 2) - contents += fd.read() - fd.close() - - self.assertEqual(contents, content_string) - self.cnx.remove("testfile.txt") - - def test_seek_cur_resume(self): - ''' seek/resume functionality (seek_cur) ''' - content_string = "This is another chunk of data"*1024 - self.create_file("testfile.txt", content_string) - self.assertEquals(self.cnx.getsize("testfile.txt"), len(content_string)) - - fd = self.cnx.open("testfile.txt", "rb") - contents = fd.read(len(content_string)-1024) - fd.close() - - fd = self.cnx.open("testfile.txt", "rb") - fd.seek(1024) - fd.read(512) - fd.seek(len(content_string)-1024-512-1024, 1) - contents += fd.read() - fd.close() - - self.assertEqual(contents, content_string) - self.cnx.remove("testfile.txt") - - def test_seek_invalid_offset(self): - ''' seek functionality, invalid offset ''' - content_string = "0"*1024 - self.create_file("testfile.txt", content_string) - self.assertEquals(self.cnx.getsize("testfile.txt"), len(content_string)) - - fd = self.cnx.open("testfile.txt", "rb") - self.assertRaises(IOSError, fd.seek, 1025) - fd.close() - - fd = self.cnx.open("testfile.txt", "rb") - self.assertRaises(IOSError, fd.seek, -1) - fd.close() - - fd = self.cnx.open("testfile.txt", "rb") - self.assertRaises(IOSError, fd.seek, -1, 2) - fd.close() - - fd = self.cnx.open("testfile.txt", "rb") - self.assertRaises(IOSError, fd.seek, 1025, 2) - fd.close() - - fd = self.cnx.open("testfile.txt", "rb") - fd.read(512) - self.assertRaises(IOSError, fd.seek, 513, 1) - self.assertRaises(IOSError, fd.seek, -513, 1) - fd.close() - - self.cnx.remove("testfile.txt") - - def test_large_file_support(self): - ''' auto-split of large files ''' - size = 1024**2 - part_size = 64*1024 - fd = self.cnx.open("bigfile.txt", "wb") - fd.split_size = part_size - content = '' - for part in xrange(size/4096): - content += chr(part)*4096 - fd.write(chr(part)*4096) - fd.close() - self.assertEqual(self.cnx.listdir("."), ["bigfile.txt", "bigfile.txt.part"]) - self.assertEqual(self.cnx.getsize("bigfile.txt"), size) - self.assertEqual(len(self.cnx.listdir("bigfile.txt.part/")), size/part_size) - self.assertEqual(self.cnx.getsize("bigfile.txt.part/000000"), part_size) - stored_content = self.read_file("/%s/bigfile.txt" % self.container) - self.assertEqual(stored_content, content) - self.cnx.remove("bigfile.txt") - for i in range(size/part_size): - self.cnx.remove("bigfile.txt.part/%.6d" % i) - - def test_large_file_support_big_chunk(self): - ''' auto-split of large files, writing a single big chunk ''' - size = 1024**2 - part_size = 64*1024 - fd = self.cnx.open("bigfile.txt", "wb") - fd.split_size = part_size - fd.write('0'*size) - fd.close() - self.assertEqual(self.cnx.listdir("."), ["bigfile.txt", "bigfile.txt.part"]) - self.assertEqual(self.cnx.getsize("bigfile.txt"), size) - self.assertEqual(len(self.cnx.listdir("bigfile.txt.part/")), size/part_size) - self.assertEqual(self.cnx.getsize("bigfile.txt.part/000000"), part_size) - self.cnx.remove("bigfile.txt") - for i in xrange(size/part_size): - self.cnx.remove("bigfile.txt.part/%.6d" % i) - - def test_large_file_support_content(self): - ''' auto-split of large files, reminder last part ''' - size = 1024**2 - part_size = 64*1000 # size % part_size != 0 - content = '' - fd = self.cnx.open("bigfile.txt", "wb") - fd.split_size = part_size - for part in xrange(size/4096): - content += chr(part)*4096 - fd.write(chr(part)*4096) - fd.close() - stored_content = self.read_file("/%s/bigfile.txt" % self.container) - self.assertEqual(len(stored_content), len(content)) - self.assertEqual(stored_content, content) - self.cnx.remove("bigfile.txt") - for i in xrange(1+(size/part_size)): - self.cnx.remove("bigfile.txt.part/%.6d" % i) - - def tearDown(self): - # Delete eveything from the container using the API - _, fails = self.conn.get_container(self.container) - for obj in fails: - self.conn.delete_object(self.container, obj["name"]) - self.cnx.rmdir("/%s" % self.container) - self.assertEquals(fails, [], "The test failed to clean up after itself leaving these objects: %r" % fails) - - -class MockupConnection(object): - '''Mockup object to simulate a CF connection.''' - def __init__(self, num_objects, objects): - self.num_objects = num_objects - self.objects = objects - - @staticmethod - def gen_object(name): - return dict(bytes=1024, content_type='text/plain', - hash='c644eacf6e9c21c7d2cca3ce8bb0ec13', - last_modified='2012-06-20T00:00:00.000000', - name=name) - - @staticmethod - def gen_subdir(name): - return dict(subdir=name) - - def list_containers_info(self): - return [dict(count=self.num_objects, bytes=1024*self.num_objects, name='container'),] - - def get_account(self): - return {}, [{ "name": "container", "count": self.num_objects, "bytes": self.num_objects*1024 },] - - def get_container(self, container, prefix=None, delimiter=None, marker=None, limit=10000): - if container != 'container': - raise client.ClientException("Not found", http_status=404) - - # test provided objects - if self.objects: - index = 0 - if marker: - while True: - name = self.objects[index].get('name', self.objects[index].get('subdir')) - if marker == name.rstrip("/"): - index += 1 - break - index += 1 - if index == self.num_objects: - # marker not found, so it's ignored - index = 0 - break - return {}, self.objects[index:index+10000] - - # generated - start = 0 - if marker: - while start <= self.num_objects: - if marker == 'object%s.txt' % start: - break - start += 1 - - end = self.num_objects-start - if end == 0: - # marker not found, so it's ingored (behaviour in OpenStack - # Object Storage) - start = 0 - end = self.num_objects - if end > limit: - end = limit - - return {}, [self.gen_object('object%s.txt' % i) for i in xrange(start, start+end)] - -class MockupOSFS(object): - '''Mockup object to simulate a CFFS.''' - memcache_hosts = None - auth_url = 'https://auth.service.fake/v1' - username = 'user' - - def __init__(self, num_objects, objects=None): - if objects and len(objects) != num_objects: - raise ValueError("objects provided but num_objects doesn't match") - - self.num_objects = num_objects - self.objects = objects - self.conn = MockupConnection(num_objects, objects) - - def _container_exists(self, container): - if container != 'container': - raise client.ClientException("Not found", http_status=404) - -class ListDirTest(unittest.TestCase): - ''' - ObjectStorageFS cache Tests. - - These tests use the Mockup* objects because some of the tests would require - creating/deleting too many objects to run the test over the real storage. - ''' - - def test_listdir(self): - """Test listdir, less than 10000 (limit) objects""" - lc = ListDirCache(MockupOSFS(100)) - - ld = lc.listdir('/') - self.assertEqual(len(ld), 1) - self.assertEqual(ld, ['container',]) - - ld = lc.listdir('/container') - self.assertEqual(len(ld), 100) - self.assertEqual(sorted(ld), sorted(['object%s.txt' % i for i in xrange(100)])) - - def test_listdir_marker(self): - """Test listdir, more than 10000 (limit) objects""" - lc = ListDirCache(MockupOSFS(10100)) - - ld = lc.listdir('/container') - self.assertEqual(len(ld), 10100) - self.assertEqual(sorted(ld), sorted(['object%s.txt' % i for i in xrange(10100)])) - - def test_listdir_marker_is_subdir(self): - """Test listdir, more than 10000 (limit) objects, marker will be a subdir""" - - objects = [MockupConnection.gen_object("object%s.txt" % i) for i in xrange(9999)] + \ - [MockupConnection.gen_subdir("00dir_name/")] + \ - [MockupConnection.gen_object("object%s.txt" % i) for i in xrange(9999, 10099)] - - lc = ListDirCache(MockupOSFS(10100, objects)) - - ld = sorted(lc.listdir('/container')) - self.assertEqual(len(ld), 10100) - self.assertEqual(ld[0], '00dir_name') - self.assertEqual(ld[1:], sorted(['object%s.txt' % i for i in xrange(10099)])) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_ftpcloudfs.py b/tests/test_ftpcloudfs.py deleted file mode 100644 index ef3bd02..0000000 --- a/tests/test_ftpcloudfs.py +++ /dev/null @@ -1,368 +0,0 @@ -#!/usr/bin/python -import unittest -import os -import sys -import ftplib -import StringIO -from datetime import datetime -from time import sleep -from swiftclient import client - -from ftpcloudfs.constants import default_address, default_port -from ftpcloudfs.fs import ListDirCache - -#import logging -#logging.getLogger().setLevel(logging.DEBUG) -#logging.basicConfig(level=logging.DEBUG) - -class FtpObjectStorageFSTest(unittest.TestCase): - ''' FTP Cloud FS main test ''' - - def setUp(self): - if not all(['OS_API_KEY' in os.environ, - 'OS_API_USER' in os.environ, - 'OS_AUTH_URL' in os.environ, - ]): - print "env OS_API_USER/OS_API_KEY/OS_AUTH_URL not found." - sys.exit(1) - - self.username = os.environ['OS_API_USER'] - self.api_key = os.environ['OS_API_KEY'] - self.auth_url = os.environ.get('OS_AUTH_URL') - self.cnx = ftplib.FTP() - self.cnx.host = default_address - self.cnx.port = default_port - self.cnx.connect() - self.cnx.login(self.username, self.api_key) - self.container = "ftpcloudfs_testing" - self.cnx.mkd("/%s" % self.container) - self.cnx.cwd("/%s" % self.container) - self.conn = client.Connection(user=self.username, key=self.api_key, authurl=self.auth_url) - - def create_file(self, path, contents): - '''Create path with contents''' - self.cnx.storbinary("STOR %s" % path, StringIO.StringIO(contents)) - - def test_mkdir_chdir_rmdir(self): - ''' mkdir/chdir/rmdir directory ''' - directory = "/foobarrandom" - self.assertEqual(self.cnx.mkd(directory), directory) - self.assertEqual(self.cnx.cwd(directory), - '250 "%s" is the current directory.' % (directory)) - self.assertEqual(self.cnx.rmd(directory), "250 Directory removed.") - - def test_mkdir_chdir_mkdir_rmdir_subdir(self): - ''' mkdir/chdir/rmdir sub directory ''' - directory = "/foobarrandom" - self.assertEqual(self.cnx.mkd(directory), directory) - self.assertEqual(self.cnx.cwd(directory), - '250 "%s" is the current directory.' % (directory)) - subdirectory = "potato" - subdirpath = directory + "/" + subdirectory - self.assertEqual(self.cnx.mkd(subdirectory), subdirpath) - # Can't delete a directory with stuff in - self.assertRaises(ftplib.error_perm, self.cnx.rmd, directory) - self.assertEqual(self.cnx.cwd(subdirectory), - '250 "%s" is the current directory.' % (subdirpath)) - self.assertEqual(self.cnx.cwd(".."), - '250 "%s" is the current directory.' % (directory)) - self.assertEqual(self.cnx.rmd(subdirectory), "250 Directory removed.") - self.assertEqual(self.cnx.cwd(".."), - '250 "/" is the current directory.') - self.assertEqual(self.cnx.rmd(directory), "250 Directory removed.") - - def test_write_open_delete(self): - ''' write/open/delete file ''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.assertEquals(self.cnx.size("testfile.txt"), len(content_string)) - store = StringIO.StringIO() - self.cnx.retrbinary("RETR testfile.txt", store.write) - self.assertEqual(store.getvalue(), content_string) - self.assertEqual(self.cnx.delete("testfile.txt"), "250 File removed.") - store.close() - - def test_write_open_delete_subdir(self): - ''' write/open/delete file in a subdirectory''' - self.cnx.mkd("potato") - self.cnx.cwd("potato") - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.assertEquals(self.cnx.size("testfile.txt"), len(content_string)) - store = StringIO.StringIO() - self.cnx.retrbinary("RETR /%s/potato/testfile.txt" % self.container, store.write) - self.assertEqual(store.getvalue(), content_string) - self.assertEqual(self.cnx.delete("testfile.txt"), "250 File removed.") - self.cnx.cwd("..") - self.cnx.rmd("potato") - store.close() - - def test_write_to_slash(self): - ''' write to slash should not be permitted ''' - self.cnx.cwd("/") - content_string = "Hello Moto" - self.assertRaises(ftplib.error_perm, self.create_file, "testfile.txt", content_string) - - def test_chdir_to_a_file(self): - ''' chdir to a file ''' - self.create_file("testfile.txt", "Hello Moto") - self.assertRaises(ftplib.error_perm, self.cnx.cwd, "/%s/testfile.txt" % self.container) - self.cnx.delete("testfile.txt") - - def test_chdir_to_slash(self): - ''' chdir to slash ''' - self.cnx.cwd("/") - - def test_chdir_to_nonexistent_container(self): - ''' chdir to non existent container''' - self.assertRaises(ftplib.error_perm, self.cnx.cwd, "/i_dont_exist") - - def test_chdir_to_nonexistent_directory(self): - ''' chdir to nonexistend directory''' - self.assertRaises(ftplib.error_perm, self.cnx.cwd, "i_dont_exist") - self.assertRaises(ftplib.error_perm, self.cnx.cwd, "/%s/i_dont_exist" % self.container) - - def test_listdir_root(self): - ''' list root directory ''' - self.cnx.cwd("/") - ls = self.cnx.nlst() - self.assertTrue(self.container in ls) - self.assertTrue('potato' not in ls) - self.cnx.mkd("potato") - ls = self.cnx.nlst() - self.assertTrue(self.container in ls) - self.assertTrue('potato' in ls) - self.cnx.rmd("potato") - - def test_listdir(self): - ''' list directory ''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.assertEqual(self.cnx.nlst(), ["testfile.txt"]) - lines = [] - self.assertEquals(self.cnx.retrlines('LIST', callback=lines.append), '226 Transfer complete.') - self.assertEquals(len(lines), 1) - line = lines[0] - expected = "-rw-r--r-- 1 "+self.username+" "+self.username+" 10 "+ datetime.utcnow().strftime("%b %d %H:") - self.assertTrue(line.startswith(expected), "line %r != expected %r" % (line, expected)) - self.assertTrue(line.endswith(" testfile.txt")) - self.cnx.delete("testfile.txt") - - def test_listdir_subdir(self): - ''' list a sub directory''' - content_string = "Hello Moto" - self.create_file("1.txt", content_string) - self.create_file("2.txt", content_string) - self.cnx.mkd("potato") - self.create_file("potato/3.txt", content_string) - self.create_file("potato/4.txt", content_string) - self.assertEqual(self.cnx.nlst(), ["1.txt", "2.txt", "potato"]) - self.cnx.cwd("potato") - self.assertEqual(self.cnx.nlst(), ["3.txt", "4.txt"]) - self.cnx.delete("3.txt") - self.cnx.delete("4.txt") - self.assertEqual(self.cnx.nlst(), []) - self.cnx.cwd("..") - self.cnx.delete("1.txt") - self.cnx.delete("2.txt") - self.assertEqual(self.cnx.nlst(), ["potato"]) - lines = [] - self.assertEquals(self.cnx.retrlines('LIST', callback=lines.append), '226 Transfer complete.') - self.assertEquals(len(lines), 1) - line = lines[0] - expected = "drwxr-xr-x 1 "+self.username+" "+self.username+" 0 "+ datetime.utcnow().strftime("%b %d %H:") - self.assertTrue(line.startswith(expected), "line %r != expected %r" % (line, expected)) - self.assertTrue(line.endswith(" potato")) - self.cnx.rmd("potato") - self.assertEqual(self.cnx.nlst(), []) - - def test_rename_file(self): - '''rename a file''' - content_string = "Hello Moto" * 100 - self.create_file("testfile.txt", content_string) - self.assertEquals(self.cnx.size("testfile.txt"), len(content_string)) - self.assertRaises(ftplib.error_perm, self.cnx.size, "testfile2.txt") - self.cnx.rename("testfile.txt", "testfile2.txt") - self.assertEquals(self.cnx.size("testfile2.txt"), len(content_string)) - self.assertRaises(ftplib.error_perm, self.cnx.size, "testfile.txt") - self.cnx.delete("testfile2.txt") - - def test_rename_file_into_subdir1(self): - '''rename a file into a subdirectory 1''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.cnx.mkd("potato") - self.assertEquals(self.cnx.size("testfile.txt"), len(content_string)) - self.assertRaises(ftplib.error_perm, self.cnx.size, "potato/testfile3.txt") - self.cnx.rename("testfile.txt", "potato/testfile3.txt") - self.assertEquals(self.cnx.size("potato/testfile3.txt"), len(content_string)) - self.assertRaises(ftplib.error_perm, self.cnx.size, "testfile.txt") - self.cnx.delete("potato/testfile3.txt") - self.cnx.rmd("potato") - - def test_rename_file_into_subdir2(self): - '''rename a file into a subdirectory without specifying dest leaf''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.cnx.mkd("potato") - self.assertEquals(self.cnx.size("testfile.txt"), len(content_string)) - self.assertRaises(ftplib.error_perm, self.cnx.size, "potato/testfile.txt") - self.cnx.rename("testfile.txt", "potato") - self.assertEquals(self.cnx.size("potato/testfile.txt"), len(content_string)) - self.assertRaises(ftplib.error_perm, self.cnx.size, "testfile.txt") - self.cnx.delete("potato/testfile.txt") - self.cnx.rmd("potato") - - def test_rename_file_into_root(self): - '''rename a file into a subdirectory without specifying dest leaf''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.assertRaises(ftplib.error_perm, self.cnx.rename, "testfile.txt", "/testfile.txt") - self.cnx.delete("testfile.txt") - - def test_rename_directory_into_file(self): - '''rename a directory into a file - shouldn't work''' - content_string = "Hello Moto" - self.create_file("testfile.txt", content_string) - self.assertRaises(ftplib.error_perm, self.cnx.rename, "/%s" % self.container, "testfile.txt") - self.cnx.delete("testfile.txt") - - def test_rename_directory_into_directory(self): - '''rename a directory into a directory''' - self.cnx.mkd("potato") - self.assertEquals(self.cnx.nlst("potato"), []) - self.cnx.rename("potato", "potato2") - self.assertEquals(self.cnx.nlst("potato2"), []) - self.cnx.rmd("potato2") - - def test_rename_directory_into_existing_directory(self): - '''rename a directory into an existing directory''' - self.cnx.mkd("potato") - self.cnx.mkd("potato2") - self.assertEquals(self.cnx.nlst("potato"), []) - self.assertEquals(self.cnx.nlst("potato2"), []) - self.cnx.rename("potato", "potato2") - self.assertEquals(self.cnx.nlst("potato2"), ["potato"]) - self.assertEquals(self.cnx.nlst("potato2/potato"), []) - self.cnx.rmd("potato2/potato") - self.cnx.rmd("potato2") - - def test_rename_directory_into_self(self): - '''rename a directory into itself''' - self.cnx.mkd("potato") - self.assertEquals(self.cnx.nlst("potato"), []) - self.cnx.rename("potato", "/%s" % self.container) - self.assertEquals(self.cnx.nlst("potato"), []) - self.cnx.rename("potato", "/%s/potato" % self.container) - self.assertEquals(self.cnx.nlst("potato"), []) - self.cnx.rename("potato", "potato") - self.assertEquals(self.cnx.nlst("potato"), []) - self.cnx.rename("/%s/potato" % self.container, ".") - self.assertEquals(self.cnx.nlst("potato"), []) - self.cnx.rmd("potato") - - def test_rename_full_directory(self): - '''rename a directory into a directory''' - self.cnx.mkd("potato") - self.create_file("potato/something.txt", "p") - try: - self.assertEquals(self.cnx.nlst("potato"), ["something.txt"]) - self.assertRaises(ftplib.error_perm, self.cnx.rename, "potato", "potato2") - finally: - self.cnx.delete("potato/something.txt") - self.cnx.rmd("potato") - - def test_rename_container(self): - '''rename an empty container''' - self.cnx.mkd("/potato") - self.assertEquals(self.cnx.nlst("/potato"), []) - self.assertRaises(ftplib.error_perm, self.cnx.nlst, "/potato2") - self.cnx.rename("/potato", "/potato2") - self.assertRaises(ftplib.error_perm, self.cnx.nlst, "/potato") - self.assertEquals(self.cnx.nlst("/potato2"), []) - self.cnx.rmd("/potato2") - - def test_rename_full_container(self): - '''rename a full container''' - self.cnx.mkd("/potato") - self.create_file("/potato/test.txt", "onion") - self.assertEquals(self.cnx.nlst("/potato"), ["test.txt"]) - self.assertRaises(ftplib.error_perm, self.cnx.rename, "/potato", "/potato2") - self.cnx.delete("/potato/test.txt") - self.cnx.rmd("/potato") - - def test_unicode_file(self): - '''Test unicode file creation''' - # File names use a utf-8 interface - file_name = u"Smiley\u263a.txt".encode("utf-8") - self.create_file(file_name, "Hello Moto") - self.assertEqual(self.cnx.nlst(), [file_name]) - self.cnx.delete(file_name) - - def test_unicode_directory(self): - '''Test unicode directory creation''' - # File names use a utf-8 interface - dir_name = u"Smiley\u263aDir".encode("utf-8") - self.cnx.mkd(dir_name) - self.assertEqual(self.cnx.nlst(), [dir_name]) - self.cnx.rmd(dir_name) - - def test_mkdir_container_unicode(self): - ''' mkdir/chdir/rmdir directory ''' - directory = u"/Smiley\u263aContainer".encode("utf-8") - self.assertEqual(self.cnx.mkd(directory), directory) - self.assertEqual(self.cnx.cwd(directory), - '250 "%s" is the current directory.' % (directory)) - self.assertEqual(self.cnx.rmd(directory), "250 Directory removed.") - - def test_fakedir(self): - '''Make some fake directories and test''' - - objs = ["test1.txt", "potato/test2.txt", "potato/sausage/test3.txt", - "potato/sausage/test4.txt",] - for obj in objs: - self.conn.put_object(self.container, obj, content_type="text/plain", contents="Hello Moto") - - self.assertEqual(self.cnx.nlst(), ["potato", "test1.txt"]) - self.assertEqual(self.cnx.nlst("potato"), ["sausage","test2.txt"]) - self.assertEqual(self.cnx.nlst("potato/sausage"), ["test3.txt", "test4.txt"]) - - self.cnx.cwd("potato") - - self.assertEqual(self.cnx.nlst(), ["sausage","test2.txt"]) - self.assertEqual(self.cnx.nlst("sausage"), ["test3.txt", "test4.txt"]) - - self.cnx.cwd("sausage") - - self.assertEqual(self.cnx.nlst(), ["test3.txt", "test4.txt"]) - - self.cnx.cwd("../..") - - for obj in objs: - self.conn.delete_object(self.container, obj) - - sleep(ListDirCache.MAX_CACHE_TIME) - - self.assertEqual(self.cnx.nlst(), []) - - def test_md5(self): - ''' MD5 extension''' - self.create_file("testfile.txt", "Hello Moto") - response = self.cnx.sendcmd("MD5 /%s/testfile.txt" % self.container) - self.cnx.delete("testfile.txt") - self.assertEqual(response, '251 "/%s/testfile.txt" 0D933AE488FD55CC6BDEAFFFBAABF0C4' % self.container) - self.assertRaises(ftplib.error_perm, self.cnx.sendcmd, "MD5 /%s" % self.container) - self.assertRaises(ftplib.error_perm, self.cnx.sendcmd, "MD5 /") - - def tearDown(self): - # Delete eveything from the container using the API - self.cnx.close() - _, fails = self.conn.get_container(self.container) - for obj in fails: - if "name" in obj: - self.conn.delete_object(self.container, obj["name"]) - self.conn.delete_container(self.container) - self.assertEquals(fails, [], "The test failed to clean up after itself leaving these objects: %r" % fails) - -if __name__ == '__main__': - unittest.main()