Merge master to feature/ec

Change-Id: I3e1d725fa85bf68c3d5fb0a062c9a053064a9e66
Implements: blueprint swift-ec

.mailmap

@@ -67,3 +67,4 @@ Mauro Stettler <mauro.stettler@gmail.com> <mauro.stettler@gmail.com>
 Pawel Palucki <pawel.palucki@gmail.com> <pawel.palucki@gmail.com>
 Guang Yee <guang.yee@hp.com> <guang.yee@hp.com>
 Jing Liuqing <jing.liuqing@99cloud.net> <jing.liuqing@99cloud.net>
+Lorcan Browne <lorcan.browne@hp.com> <lorcan.browne@hp.com>

AUTHORS

@@ -21,26 +21,32 @@ Joe Arnold (joe@swiftstack.com)
 Ionuț Arțăriși (iartarisi@suse.cz)
 Christian Berendt (berendt@b1-systems.de)
 Luis de Bethencourt (luis@debethencourt.com)
+Keshava Bharadwaj (kb.sankethi@gmail.com)
 Yummy Bian (yummy.bian@gmail.com)
 Darrell Bishop (darrell@swiftstack.com)
 James E. Blair (jeblair@openstack.org)
 Fabien Boucher (fabien.boucher@enovance.com)
 Chmouel Boudjnah (chmouel@enovance.com)
 Clark Boylan (clark.boylan@gmail.com)
 Pádraig Brady (pbrady@redhat.com)
+Lorcan Browne (lorcan.browne@hp.com)
 Russell Bryant (rbryant@redhat.com)
+Jay S. Bryant (jsbryant@us.ibm.com)
 Brian D. Burns (iosctr@gmail.com)
 Devin Carlen (devin.carlen@gmail.com)
 Thierry Carrez (thierry@openstack.org)
+Mahati Chamarthy (mahati.chamarthy@gmail.com)
 Zap Chang (zapchang@gmail.com)
 François Charlier (francois.charlier@enovance.com)
 Ray Chen (oldsharp@163.com)
 Brian Cline (bcline@softlayer.com)
 Alistair Coles (alistair.coles@hp.com)
 Brian Curtin (brian.curtin@rackspace.com)
+Thiago da Silva (thiago@redhat.com)
 Julien Danjou (julien@danjou.info)
 Ksenia Demina (kdemina@mirantis.com)
 Dan Dillinger (dan.dillinger@sonian.net)
+Gerry Drudy (gerry.drudy@hp.com)
 Morgan Fainberg (morgan.fainberg@gmail.com)
 ZhiQiang Fan (aji.zqfan@gmail.com)
 Flaper Fesp (flaper87@gmail.com)
@@ -55,6 +61,7 @@ David Goetz (david.goetz@rackspace.com)
 Jonathan Gonzalez V (jonathan.abdiel@gmail.com)
 Joe Gordon (jogo@cloudscaling.com)
 David Hadas (davidh@il.ibm.com)
+Andrew Hale (andy@wwwdata.eu)
 Soren Hansen (soren@linux2go.dk)
 Richard (Rick) Hawkins (richard.hawkins@rackspace.com)
 Doug Hellmann (doug.hellmann@dreamhost.com)
@@ -67,6 +74,7 @@ Kun Huang (gareth@unitedstack.com)
 Matthieu Huin (mhu@enovance.com)
 Hodong Hwang (hodong.hwang@kt.com)
 Motonobu Ichimura (motonobu@gmail.com)
+Andreas Jaeger (aj@suse.de)
 Shri Javadekar (shrinand@maginatics.com)
 Iryoung Jeong (iryoung@gmail.com)
 Paul Jimenez (pj@place.org)
@@ -80,6 +88,7 @@ Morita Kazutaka (morita.kazutaka@gmail.com)
 Josh Kearney (josh@jk0.org)
 Ilya Kharin (ikharin@mirantis.com)
 Dae S. Kim (dae@velatum.com)
+Nathan Kinder (nkinder@redhat.com)
 Eugene Kirpichov (ekirpichov@gmail.com)
 Leah Klearman (lklrmn@gmail.com)
 Steve Kowalik (steven@wedontsleep.org)
@@ -88,6 +97,7 @@ Sushil Kumar (sushil.kumar2@globallogic.com)
 Madhuri Kumari (madhuri.rai07@gmail.com)
 Steven Lang (Steven.Lang@hgst.com)
 Gonéri Le Bouder (goneri.lebouder@enovance.com)
+John Leach (john@johnleach.co.uk)
 Ed Leafe (ed.leafe@rackspace.com)
 Thomas Leaman (thomas.leaman@hp.com)
 Eohyung Lee (liquid@kt.com)
@@ -104,6 +114,7 @@ Dragos Manolescu (dragosm@hp.com)
 Steve Martinelli (stevemar@ca.ibm.com)
 Juan J. Martinez (juan@memset.com)
 Marcelo Martins (btorch@gmail.com)
+Dolph Mathews (dolph.mathews@gmail.com)
 Donagh McCabe (donagh.mccabe@hp.com)
 Andy McCrae (andy.mccrae@gmail.com)
 Paul McMillan (paul.mcmillan@nebula.com)
@@ -117,6 +128,7 @@ Maru Newby (mnewby@internap.com)
 Newptone (xingchao@unitedstack.com)
 Colin Nicholson (colin.nicholson@iomart.com)
 Zhenguo Niu (zhenguo@unitedstack.com)
+Timothy Okwii (tokwii@cisco.com)
 Matthew Oliver (matt@oliver.net.au)
 Eamonn O'Toole (eamonn.otoole@hp.com)
 James Page (james.page@ubuntu.com)
@@ -129,17 +141,19 @@ Dieter Plaetinck (dieter@vimeo.com)
 Peter Portante (peter.portante@redhat.com)
 Dan Prince (dprince@redhat.com)
 Felipe Reyes (freyes@tty.cl)
+Matt Riedemann (mriedem@us.ibm.com)
 Li Riqiang (lrqrun@gmail.com)
+Rafael Rivero (rafael@cloudscaling.com)
 Victor Rodionov (victor.rodionov@nexenta.com)
 Aaron Rosen (arosen@nicira.com)
 Brent Roskos (broskos@internap.com)
 Cristian A Sanchez (cristian.a.sanchez@intel.com)
+saranjan (saranjan@cisco.com)
 Christian Schwede (info@cschwede.de)
 Mark Seger (Mark.Seger@hp.com)
 Andrew Clay Shafer (acs@parvuscaptus.com)
 Chuck Short (chuck.short@canonical.com)
 Michael Shuler (mshuler@gmail.com)
-Thiago da Silva (thiago@redhat.com)
 David Moreau Simard (dmsimard@iweb.com)
 Scott Simpson (sasimpson@gmail.com)
 Liu Siqi (meizu647@gmail.com)

CHANGELOG

@@ -1,3 +1,57 @@
+swift (2.2.0)
+
+    * Added support for Keystone v3 auth.
+
+      Keystone v3 introduced the concept of "domains" and user names
+      are no longer unique across domains. Swift's Keystone integration
+      now requires that ACLs be set on IDs, which are unique across
+      domains, and further restricts setting new ACLs to only use IDs.
+
+      Please see http://swift.openstack.org/overview_auth.html for
+      more information on configuring Swift and Keystone together.
+
+    * Swift now supports server-side account-to-account copy. Server-
+      side copy in Swift requires the X-Copy-From header (on a PUT)
+      or the Destination header (on a COPY). To initiate an account-to-
+      account copy, the existing header value remains the same, but the
+      X-Copy-From-Account header (on a PUT) or the Destination-Account
+      (on a COPY) are used to indicate the proper account.
+
+    * Limit partition movement when adding a new placement tier.
+
+      When adding a new placement tier (server, zone, or region), Swift
+      previously attempted to move all placement partitions, regardless
+      of the space available on the new tier, to ensure the best possible
+      durability. Unfortunately, this could result in too many partitions
+      being moved all at once to a new tier. Swift's ring-builder now
+      ensures that only the correct number of placement partitions are
+      rebalanced, and thus makes adding capacity to the cluster more
+      efficient.
+
+    * Per storage policy container counts are now reported in an
+      account response headers.
+
+    * Swift will now reject, with a 4xx series response, GET requests
+      with more than 50 ranges, more than 3 overlapping ranges, or more
+      than 8 non-increasing ranges.
+
+    * The bind_port config setting is now required to be explicitly set.
+
+    * The object server can now use splice() for a zero-copy GET
+      response. This feature is enabled with the "splice" config variable
+      in the object server config and defaults to off. Also, this feature
+      only works on recent Linux kernels (AF_ALG sockets must be
+      supported). A zero-copy GET response can significantly reduce CPU
+      requirements for object servers.
+
+    * Added "--no-overlap" option to swift-dispersion populate so that
+      multiple runs of the tool can add coverage without overlapping
+      existing monitored partitions.
+
+    * swift-recon now supports filtering by region.
+
+    * Various other minor bug fixes and improvements.
+
 swift (2.1.0)
 
     * swift-ring-builder placement was improved to allow gradual addition

swift/account/server.py

@@ -156,7 +156,7 @@ def PUT(self, req):
                             for key, value in req.headers.iteritems()
                             if is_sys_or_user_meta('account', key))
             if metadata:
-                broker.update_metadata(metadata)
+                broker.update_metadata(metadata, validate_metadata=True)
             if created:
                 return HTTPCreated(request=req)
             else:
@@ -249,7 +249,7 @@ def POST(self, req):
                         for key, value in req.headers.iteritems()
                         if is_sys_or_user_meta('account', key))
         if metadata:
-            broker.update_metadata(metadata)
+            broker.update_metadata(metadata, validate_metadata=True)
         return HTTPNoContent(request=req)
 
     def __call__(self, env, start_response):

swift/cli/ringbuilder.py

@@ -830,10 +830,18 @@ def main(arguments=None):
 
     builder_file, ring_file = parse_builder_ring_filename_args(argv)
 
-    if exists(builder_file):
+    try:
         builder = RingBuilder.load(builder_file)
-    elif len(argv) < 3 or argv[2] not in('create', 'write_builder'):
-        print 'Ring Builder file does not exist: %s' % argv[1]
+    except exceptions.UnPicklingError as e:
+        print e
+        exit(EXIT_ERROR)
+    except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:
+        if len(argv) < 3 or argv[2] not in('create', 'write_builder'):
+            print e
+            exit(EXIT_ERROR)
+    except Exception as e:
+        print 'Problem occurred while reading builder file: %s. %s' % (
+            argv[1], e.message)
         exit(EXIT_ERROR)
 
     backup_dir = pathjoin(dirname(argv[1]), 'backups')

swift/common/constraints.py

@@ -101,7 +101,10 @@ def reload_constraints():
 
 def check_metadata(req, target_type):
     """
-    Check metadata sent in the request headers.
+    Check metadata sent in the request headers.  This should only check
+    that the metadata in the request given is valid.  Checks against
+    account/container overall metadata should be forwarded on to its
+    respective server to be checked.
 
     :param req: request object
     :param target_type: str: one of: object, container, or account: indicates

swift/common/db.py

@@ -30,9 +30,11 @@
 from eventlet import sleep, Timeout
 import sqlite3
 
+from swift.common.constraints import MAX_META_COUNT, MAX_META_OVERALL_SIZE
 from swift.common.utils import json, Timestamp, renamer, \
     mkdirs, lock_parent_directory, fallocate
 from swift.common.exceptions import LockTimeout
+from swift.common.swob import HTTPBadRequest
 
 
 #: Whether calls will be made to preallocate disk space for database files.
@@ -719,7 +721,35 @@ def metadata(self):
             metadata = {}
         return metadata
 
-    def update_metadata(self, metadata_updates):
+    @staticmethod
+    def validate_metadata(metadata):
+        """
+        Validates that metadata_falls within acceptable limits.
+
+        :param metadata: to be validated
+        :raises: HTTPBadRequest if MAX_META_COUNT or MAX_META_OVERALL_SIZE
+                 is exceeded
+        """
+        meta_count = 0
+        meta_size = 0
+        for key, (value, timestamp) in metadata.iteritems():
+            key = key.lower()
+            if value != '' and (key.startswith('x-account-meta') or
+                                key.startswith('x-container-meta')):
+                prefix = 'x-account-meta-'
+                if key.startswith('x-container-meta-'):
+                    prefix = 'x-container-meta-'
+                key = key[len(prefix):]
+                meta_count = meta_count + 1
+                meta_size = meta_size + len(key) + len(value)
+        if meta_count > MAX_META_COUNT:
+            raise HTTPBadRequest('Too many metadata items; max %d'
+                                 % MAX_META_COUNT)
+        if meta_size > MAX_META_OVERALL_SIZE:
+            raise HTTPBadRequest('Total metadata too large; max %d'
+                                 % MAX_META_OVERALL_SIZE)
+
+    def update_metadata(self, metadata_updates, validate_metadata=False):
         """
         Updates the metadata dict for the database. The metadata dict values
         are tuples of (value, timestamp) where the timestamp indicates when
@@ -752,6 +782,8 @@ def update_metadata(self, metadata_updates):
                 value, timestamp = value_timestamp
                 if key not in md or timestamp > md[key][1]:
                     md[key] = value_timestamp
+            if validate_metadata:
+                DatabaseBroker.validate_metadata(md)
             conn.execute('UPDATE %s_stat SET metadata = ?' % self.db_type,
                          (json.dumps(md),))
             conn.commit()

swift/common/direct_client.py

@@ -108,19 +108,19 @@ def direct_get_account(node, part, account, marker=None, limit=None,
     :param marker: marker query
     :param limit: query limit
     :param prefix: prefix query
-    :param delimeter: delimeter for the query
+    :param delimiter: delimiter for the query
     :param conn_timeout: timeout in seconds for establishing the connection
     :param response_timeout: timeout in seconds for getting the response
     :returns: a tuple of (response headers, a list of containers) The response
               headers will HeaderKeyDict.
     """
     path = '/' + account
     return _get_direct_account_container(path, "Account", node, part,
-                                         account, marker=None,
-                                         limit=None, prefix=None,
-                                         delimiter=None,
-                                         conn_timeout=5,
-                                         response_timeout=15)
+                                         account, marker=marker,
+                                         limit=limit, prefix=prefix,
+                                         delimiter=delimiter,
+                                         conn_timeout=conn_timeout,
+                                         response_timeout=response_timeout)
 
 
 def direct_delete_account(node, part, account, conn_timeout=5,
@@ -183,19 +183,19 @@ def direct_get_container(node, part, account, container, marker=None,
     :param marker: marker query
     :param limit: query limit
     :param prefix: prefix query
-    :param delimeter: delimeter for the query
+    :param delimiter: delimiter for the query
     :param conn_timeout: timeout in seconds for establishing the connection
     :param response_timeout: timeout in seconds for getting the response
     :returns: a tuple of (response headers, a list of objects) The response
               headers will be a HeaderKeyDict.
     """
     path = '/%s/%s' % (account, container)
     return _get_direct_account_container(path, "Container", node,
-                                         part, account, marker=None,
-                                         limit=None, prefix=None,
-                                         delimiter=None,
-                                         conn_timeout=5,
-                                         response_timeout=15)
+                                         part, account, marker=marker,
+                                         limit=limit, prefix=prefix,
+                                         delimiter=delimiter,
+                                         conn_timeout=conn_timeout,
+                                         response_timeout=response_timeout)
 
 
 def direct_delete_container(node, part, account, container, conn_timeout=5,

swift/common/exceptions.py

@@ -123,6 +123,18 @@ class DuplicateDeviceError(RingBuilderError):
     pass
 
 
+class UnPicklingError(SwiftException):
+    pass
+
+
+class FileNotFoundError(SwiftException):
+    pass
+
+
+class PermissionError(SwiftException):
+    pass
+
+
 class ListingIterError(SwiftException):
     pass
 

swift/common/manager.py

@@ -625,7 +625,7 @@ def launch(self, **kwargs):
         """
         conf_files = self.conf_files(**kwargs)
         if not conf_files:
-            return []
+            return {}
 
         pids = self.get_running_pids(**kwargs)
 
@@ -645,7 +645,7 @@ def launch(self, **kwargs):
 
         if already_started:
             print _("%s already started...") % self.server
-            return []
+            return {}
 
         if self.server not in START_ONCE_SERVERS:
             kwargs['once'] = False