Added account_autocreate mode where authorized requests to accounts t…

…hat don't yet exist within Swift will cause those accounts to be automatically created. Also did a small refactor surrounding swift.common.utils.TRUE_VALUES.

doc/source/deployment_guide.rst

@@ -547,6 +547,10 @@ error_suppression_limit       10               Error count to consider a
                                                node error limited
 allow_account_management      false            Whether account PUTs and DELETEs
                                                are even callable
+account_autocreate            false            If set to 'true' authorized
+                                               accounts that do not yet exist
+                                               within the Swift cluster will
+                                               be automatically created.
 ============================  ===============  =============================
 
 [tempauth]

etc/proxy-server.conf-sample

@@ -40,6 +40,9 @@ use = egg:swift#proxy
 # If set to 'true' any authorized user may create and delete accounts; if
 # 'false' no one, even authorized, can.
 # allow_account_management = false
+# If set to 'true' authorized accounts that do not yet exist within the Swift
+# cluster will be automatically created.
+# account_autocreate = false
 
 [filter:tempauth]
 use = egg:swift#tempauth

swift/common/bench.py

@@ -43,7 +43,7 @@ def __init__(self, logger, conf, names):
         self.user = conf.user
         self.key = conf.key
         self.auth_url = conf.auth
-        self.use_proxy = conf.use_proxy in TRUE_VALUES
+        self.use_proxy = conf.use_proxy.lower() in TRUE_VALUES
         if self.use_proxy:
             url, token = client.get_auth(self.auth_url, self.user, self.key)
             self.token = token
@@ -125,7 +125,7 @@ def __init__(self, logger, conf):
         self.logger = logger
         self.conf = conf
         self.names = []
-        self.delete = conf.delete in TRUE_VALUES
+        self.delete = conf.delete.lower() in TRUE_VALUES
         self.gets = int(conf.num_gets)
 
     def run(self):

swift/common/daemon.py

@@ -75,7 +75,8 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
                           log_name=kwargs.get('log_name'))
 
     # once on command line (i.e. daemonize=false) will over-ride config
-    once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES
+    once = once or \
+            conf.get('daemonize', 'true').lower() not in utils.TRUE_VALUES
 
     # pre-configure logger
     if 'logger' in kwargs:

swift/common/middleware/staticweb.py

@@ -270,7 +270,7 @@ def _listing(self, env, start_response, prefix=None):
         :param start_response: The original WSGI start_response hook.
         :param prefix: Any prefix desired for the container listing.
         """
-        if self._listings not in TRUE_VALUES:
+        if self._listings.lower() not in TRUE_VALUES:
             resp = HTTPNotFound()(env, self._start_response)
             return self._error_response(resp, env, start_response)
         tmp_env = self._get_escalated_env(env)

swift/common/utils.py

@@ -72,7 +72,7 @@
         pass
 
 # Used when reading config values
-TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes', 'on', 'On', 't', 'y'))
+TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
 
 
 def validate_configuration():

swift/proxy/server.py

@@ -41,8 +41,8 @@
 from webob import Request, Response
 
 from swift.common.ring import Ring
-from swift.common.utils import get_logger, normalize_timestamp, split_path, \
-    cache_from_env, ContextPool
+from swift.common.utils import cache_from_env, ContextPool, get_logger, \
+    normalize_timestamp, split_path, TRUE_VALUES
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_metadata, check_object_creation, \
     check_utf8, CONTAINER_LISTING_LIMIT, MAX_ACCOUNT_NAME_LENGTH, \
@@ -338,7 +338,7 @@ def error_limit(self, node):
         node['errors'] = self.app.error_suppression_limit + 1
         node['last_error'] = time.time()
 
-    def account_info(self, account):
+    def account_info(self, account, autocreate=False):
         """
         Get account information, and also verify that the account exists.
 
@@ -353,7 +353,7 @@ def account_info(self, account):
             result_code = self.app.memcache.get(cache_key)
             if result_code == 200:
                 return partition, nodes
-            elif result_code == 404:
+            elif result_code == 404 and not autocreate:
                 return None, None
         result_code = 0
         attempts_left = self.app.account_ring.replica_count
@@ -386,6 +386,17 @@ def account_info(self, account):
             except (Exception, TimeoutError):
                 self.exception_occurred(node, _('Account'),
                     _('Trying to get account info for %s') % path)
+        if result_code == 404 and autocreate:
+            if len(account) > MAX_ACCOUNT_NAME_LENGTH:
+                return None, None
+            headers = {'X-Timestamp': normalize_timestamp(time.time()),
+                       'X-Trans-Id': self.trans_id}
+            resp = self.make_requests(Request.blank('/v1' + path),
+                self.app.account_ring, partition, 'PUT',
+                path, [headers] * len(nodes))
+            if resp.status_int // 100 != 2:
+                raise Exception('Could not autocreate account %r' % path)
+            result_code = 200
         if self.app.memcache and result_code in (200, 404):
             if result_code == 200:
                 cache_timeout = self.app.recheck_account_existence
@@ -397,7 +408,7 @@ def account_info(self, account):
             return partition, nodes
         return None, None
 
-    def container_info(self, account, container):
+    def container_info(self, account, container, account_autocreate=False):
         """
         Get container information and thusly verify container existance.
         This will also make a call to account_info to verify that the
@@ -423,7 +434,7 @@ def container_info(self, account, container):
                     return partition, nodes, read_acl, write_acl
                 elif status == 404:
                     return None, None, None, None
-        if not self.account_info(account)[1]:
+        if not self.account_info(account, autocreate=account_autocreate)[1]:
             return None, None, None, None
         result_code = 0
         read_acl = None
@@ -854,7 +865,8 @@ def POST(self, req):
         if error_response:
             return error_response
         container_partition, containers, _junk, req.acl = \
-            self.container_info(self.account_name, self.container_name)
+            self.container_info(self.account_name, self.container_name,
+                account_autocreate=self.app.account_autocreate)
         if 'swift.authorize' in req.environ:
             aresp = req.environ['swift.authorize'](req)
             if aresp:
@@ -911,7 +923,8 @@ def _connect_put_node(self, nodes, part, path, headers):
     def PUT(self, req):
         """HTTP PUT request handler."""
         container_partition, containers, _junk, req.acl = \
-            self.container_info(self.account_name, self.container_name)
+            self.container_info(self.account_name, self.container_name,
+                account_autocreate=self.app.account_autocreate)
         if 'swift.authorize' in req.environ:
             aresp = req.environ['swift.authorize'](req)
             if aresp:
@@ -1219,7 +1232,8 @@ def PUT(self, req):
             resp.body = 'Container name length of %d longer than %d' % \
                         (len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
             return resp
-        account_partition, accounts = self.account_info(self.account_name)
+        account_partition, accounts = self.account_info(self.account_name,
+            autocreate=self.app.account_autocreate)
         if not accounts:
             return HTTPNotFound(request=req)
         container_partition, containers = self.app.container_ring.get_nodes(
@@ -1249,7 +1263,8 @@ def POST(self, req):
             self.clean_acls(req) or check_metadata(req, 'container')
         if error_response:
             return error_response
-        account_partition, accounts = self.account_info(self.account_name)
+        account_partition, accounts = self.account_info(self.account_name,
+            autocreate=self.app.account_autocreate)
         if not accounts:
             return HTTPNotFound(request=req)
         container_partition, containers = self.app.container_ring.get_nodes(
@@ -1391,7 +1406,7 @@ def __init__(self, conf, memcache=None, logger=None, account_ring=None,
         self.put_queue_depth = int(conf.get('put_queue_depth', 10))
         self.object_chunk_size = int(conf.get('object_chunk_size', 65536))
         self.client_chunk_size = int(conf.get('client_chunk_size', 65536))
-        self.log_headers = conf.get('log_headers') == 'True'
+        self.log_headers = conf.get('log_headers', 'no').lower() in TRUE_VALUES
         self.error_suppression_interval = \
             int(conf.get('error_suppression_interval', 60))
         self.error_suppression_limit = \
@@ -1401,7 +1416,7 @@ def __init__(self, conf, memcache=None, logger=None, account_ring=None,
         self.recheck_account_existence = \
             int(conf.get('recheck_account_existence', 60))
         self.allow_account_management = \
-            conf.get('allow_account_management', 'false').lower() == 'true'
+            conf.get('allow_account_management', 'no').lower() in TRUE_VALUES
         self.resellers_conf = ConfigParser()
         self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf'))
         self.object_ring = object_ring or \
@@ -1413,6 +1428,8 @@ def __init__(self, conf, memcache=None, logger=None, account_ring=None,
         self.memcache = memcache
         mimetypes.init(mimetypes.knownfiles +
                        [os.path.join(swift_dir, 'mime.types')])
+        self.account_autocreate = \
+            conf.get('account_autocreate', 'no').lower() in TRUE_VALUES
 
     def get_controller(self, path):
         """

swift/stats/log_uploader.py

@@ -69,7 +69,7 @@ def __init__(self, uploader_conf, plugin_name, regex=None, cutoff=None):
         self.internal_proxy = InternalProxy(proxy_server_conf)
         self.new_log_cutoff = int(cutoff or
                                   uploader_conf.get('new_log_cutoff', '7200'))
-        self.unlink_log = uploader_conf.get('unlink_log', 'True').lower() in \
+        self.unlink_log = uploader_conf.get('unlink_log', 'true').lower() in \
                 utils.TRUE_VALUES
         self.filename_pattern = regex or \
             uploader_conf.get('source_filename_pattern',

test/unit/common/test_utils.py

@@ -768,6 +768,10 @@ def test_human_readable(self):
         self.assertEquals(utils.human_readable(1237940039285380274899124224),
                           '1024Yi')
 
+    def test_TRUE_VALUES(self):
+        for v in utils.TRUE_VALUES:
+            self.assertEquals(v, v.lower())
+
 
 if __name__ == '__main__':
     unittest.main()

test/unit/proxy/test_server.py

@@ -393,6 +393,48 @@ def test(*status_list):
             test(404, 507, 503)
             test(503, 503, 503)
 
+    def test_account_info_account_autocreate(self):
+        with save_globals():
+            self.memcache.store = {}
+            proxy_server.http_connect = \
+                fake_http_connect(404, 404, 404, 201, 201, 201)
+            partition, nodes = \
+                self.controller.account_info(self.account, autocreate=False)
+            self.check_account_info_return(partition, nodes, is_none=True)
+
+            self.memcache.store = {}
+            proxy_server.http_connect = \
+                fake_http_connect(404, 404, 404, 201, 201, 201)
+            partition, nodes = \
+                self.controller.account_info(self.account)
+            self.check_account_info_return(partition, nodes, is_none=True)
+
+            self.memcache.store = {}
+            proxy_server.http_connect = \
+                fake_http_connect(404, 404, 404, 201, 201, 201)
+            partition, nodes = \
+                self.controller.account_info(self.account, autocreate=True)
+            self.check_account_info_return(partition, nodes)
+
+            self.memcache.store = {}
+            proxy_server.http_connect = \
+                fake_http_connect(404, 404, 404, 503, 201, 201)
+            partition, nodes = \
+                self.controller.account_info(self.account, autocreate=True)
+            self.check_account_info_return(partition, nodes)
+
+            self.memcache.store = {}
+            proxy_server.http_connect = \
+                fake_http_connect(404, 404, 404, 503, 201, 503)
+            exc = None
+            try:
+                partition, nodes = \
+                    self.controller.account_info(self.account, autocreate=True)
+            except Exception, err:
+                exc = err
+            self.assertEquals(str(exc),
+                              "Could not autocreate account '/some_account'")
+
     def check_container_info_return(self, ret, is_none=False):
         if is_none:
             partition, nodes, read_acl, write_acl = None, None, None, None
@@ -406,7 +448,7 @@ def check_container_info_return(self, ret, is_none=False):
         self.assertEqual(write_acl, ret[3])
 
     def test_container_info_invalid_account(self):
-        def account_info(self, account):
+        def account_info(self, account, autocreate=False):
             return None, None
 
         with save_globals():
@@ -417,7 +459,7 @@ def account_info(self, account):
 
     # tests if 200 is cached and used
     def test_container_info_200(self):
-        def account_info(self, account):
+        def account_info(self, account, autocreate=False):
             return True, True
 
         with save_globals():
@@ -443,7 +485,7 @@ def account_info(self, account):
 
     # tests if 404 is cached and used
     def test_container_info_404(self):
-        def account_info(self, account):
+        def account_info(self, account, autocreate=False):
             return True, True
 
         with save_globals():