Merge master to feature/ec

Change-Id: I38f9b55919b0db6e4648cc04fd35944e5d5ba676

doc/source/development_guidelines.rst

@@ -51,6 +51,25 @@ To execute the unit tests:
 
   - `tox -e pep8,py26`
 
+The functional tests may be executed against a :doc:`development_saio` or
+other running Swift cluster using the command:
+
+- `tox -e func`
+
+The endpoint and authorization credentials to be used by functional tests
+should be configured in the ``test.conf`` file as described in the section
+:ref:`setup_scripts`.
+
+If the ``test.conf`` file is not found then the functional test framework will
+instantiate a set of Swift servers in the same process that executes the
+functional tests. This 'in-process test' mode may also be enabled (or disabled)
+by setting the environment variable ``SWIFT_TEST_IN_PROCESS`` to a true (or
+false) value prior to executing `tox -e func`.
+
+When using the 'in-process test' mode, the optional in-memory
+object server may be selected by setting the environment variable
+``SWIFT_TEST_IN_MEMORY_OBJ`` to a true value.
+
 ------------
 Coding Style
 ------------
@@ -78,7 +97,7 @@ The documentation in docstrings should follow the PEP 257 conventions
 
 More specifically:
 
-    1.  Triple qutes should be used for all docstrings.
+    1.  Triple quotes should be used for all docstrings.
     2.  If the docstring is simple and fits on one line, then just use
         one line.
     3.  For docstrings that take multiple lines, there should be a newline
@@ -117,3 +136,4 @@ another year added, and date ranges are not needed.::
     # implied.
     # See the License for the specific language governing permissions and
     # limitations under the License.
+

doc/source/development_saio.rst

@@ -398,6 +398,8 @@ commands are as follows:
 
      .. literalinclude:: /../saio/swift/object-server/4.conf
 
+.. _setup_scripts:
+
 ------------------------------------
 Setting up scripts for running Swift
 ------------------------------------

etc/proxy-server.conf-sample

@@ -415,7 +415,7 @@ use = egg:swift#staticweb
 [filter:tempurl]
 use = egg:swift#tempurl
 # The methods allowed with Temp URLs.
-# methods = GET HEAD PUT
+# methods = GET HEAD PUT POST DELETE
 #
 # The headers to remove from incoming requests. Simply a whitespace delimited
 # list of header names and names can optionally end with '*' to indicate a

swift/account/auditor.py

@@ -36,6 +36,7 @@ def __init__(self, conf):
         self.devices = conf.get('devices', '/srv/node')
         self.mount_check = config_true_value(conf.get('mount_check', 'true'))
         self.interval = int(conf.get('interval', 1800))
+        self.logging_interval = 3600  # once an hour
         self.account_passes = 0
         self.account_failures = 0
         self.accounts_running_time = 0
@@ -53,7 +54,7 @@ def _one_audit_pass(self, reported):
                                             logger=self.logger)
         for path, device, partition in all_locs:
             self.account_audit(path)
-            if time.time() - reported >= 3600:  # once an hour
+            if time.time() - reported >= self.logging_interval:
                 self.logger.info(_('Since %(time)s: Account audits: '
                                    '%(passed)s passed audit,'
                                    '%(failed)s failed audit'),

swift/account/backend.py

@@ -312,9 +312,10 @@ def is_status_deleted(self):
         """Only returns true if the status field is set to DELETED."""
         with self.get() as conn:
             row = conn.execute('''
-                SELECT status
+                SELECT put_timestamp, delete_timestamp, status
                 FROM account_stat''').fetchone()
-            return (row['status'] == "DELETED")
+            return row['status'] == "DELETED" or (
+                row['delete_timestamp'] > row['put_timestamp'])
 
     def get_policy_stats(self):
         """

swift/common/middleware/tempurl.py

@@ -212,7 +212,8 @@ class TempURL(object):
     :param conf: The configuration dict for the middleware.
     """
 
-    def __init__(self, app, conf, methods=('GET', 'HEAD', 'PUT')):
+    def __init__(self, app, conf,
+                 methods=('GET', 'HEAD', 'PUT', 'POST', 'DELETE')):
         #: The next WSGI application/filter in the paste.deploy pipeline.
         self.app = app
         #: The filter configuration dict.
@@ -513,7 +514,7 @@ def filter_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
 
-    methods = conf.get('methods', 'GET HEAD PUT').split()
+    methods = conf.get('methods', 'GET HEAD PUT POST DELETE').split()
     register_swift_info('tempurl', methods=methods)
 
     return lambda app: TempURL(app, conf, methods=methods)

swift/common/utils.py

@@ -933,7 +933,7 @@ def xreadlines(self):
 
 class StatsdClient(object):
     def __init__(self, host, port, base_prefix='', tail_prefix='',
-                 default_sample_rate=1, sample_rate_factor=1):
+                 default_sample_rate=1, sample_rate_factor=1, logger=None):
         self._host = host
         self._port = port
         self._base_prefix = base_prefix
@@ -942,6 +942,7 @@ def __init__(self, host, port, base_prefix='', tail_prefix='',
         self._sample_rate_factor = sample_rate_factor
         self._target = (self._host, self._port)
         self.random = random
+        self.logger = logger
 
     def set_prefix(self, new_prefix):
         if new_prefix and self._base_prefix:
@@ -966,7 +967,13 @@ def _send(self, m_name, m_value, m_type, sample_rate):
         # Ideally, we'd cache a sending socket in self, but that
         # results in a socket getting shared by multiple green threads.
         with closing(self._open_socket()) as sock:
-            return sock.sendto('|'.join(parts), self._target)
+            try:
+                return sock.sendto('|'.join(parts), self._target)
+            except IOError as err:
+                if self.logger:
+                    self.logger.warn(
+                        'Error sending UDP message to %r: %s',
+                        self._target, err)
 
     def _open_socket(self):
         return socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
@@ -1332,7 +1339,7 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None,
             'log_statsd_sample_rate_factor', 1))
         statsd_client = StatsdClient(statsd_host, statsd_port, base_prefix,
                                      name, default_sample_rate,
-                                     sample_rate_factor)
+                                     sample_rate_factor, logger=logger)
         logger.statsd_client = statsd_client
     else:
         logger.statsd_client = None

swift/obj/mem_diskfile.py

@@ -395,5 +395,5 @@ def delete(self, timestamp):
         :param timestamp: timestamp to compare with each file
         """
         fp, md = self._filesystem.get_object(self._name)
-        if md['X-Timestamp'] < Timestamp(timestamp):
+        if md and md['X-Timestamp'] < Timestamp(timestamp):
             self._filesystem.del_object(self._name)

swift/proxy/controllers/obj.py

@@ -235,15 +235,6 @@ def HEAD(self, req):
     @delay_denial
     def POST(self, req):
         """HTTP POST request handler."""
-        if 'x-delete-after' in req.headers:
-            try:
-                x_delete_after = int(req.headers['x-delete-after'])
-            except ValueError:
-                return HTTPBadRequest(request=req,
-                                      content_type='text/plain',
-                                      body='Non-integer X-Delete-After')
-            req.headers['x-delete-at'] = normalize_delete_at_timestamp(
-                time.time() + x_delete_after)
         if self.app.object_post_as_copy:
             req.method = 'PUT'
             req.path_info = '/v1/%s/%s/%s' % (
@@ -279,29 +270,14 @@ def POST(self, req):
                     return aresp
             if not containers:
                 return HTTPNotFound(request=req)
-            if 'x-delete-at' in req.headers:
-                try:
-                    x_delete_at = normalize_delete_at_timestamp(
-                        int(req.headers['x-delete-at']))
-                    if int(x_delete_at) < time.time():
-                        return HTTPBadRequest(
-                            body='X-Delete-At in past', request=req,
-                            content_type='text/plain')
-                except ValueError:
-                    return HTTPBadRequest(request=req,
-                                          content_type='text/plain',
-                                          body='Non-integer X-Delete-At')
-                req.environ.setdefault('swift.log_info', []).append(
-                    'x-delete-at:%s' % x_delete_at)
-                delete_at_container = normalize_delete_at_timestamp(
-                    int(x_delete_at) /
-                    self.app.expiring_objects_container_divisor *
-                    self.app.expiring_objects_container_divisor)
-                delete_at_part, delete_at_nodes = \
-                    self.app.container_ring.get_nodes(
-                        self.app.expiring_objects_account, delete_at_container)
-            else:
-                delete_at_container = delete_at_part = delete_at_nodes = None
+
+            try:
+                req, delete_at_container, delete_at_part, \
+                    delete_at_nodes = self._config_obj_expiration(req)
+            except ValueError as e:
+                return HTTPBadRequest(request=req, content_type='text/plain',
+                                      body=str(e))
+
             # pass the policy index to storage nodes via req header
             policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
                                            container_info['storage_policy'])
@@ -447,6 +423,42 @@ def get_conn_response(conn):
             bodies.append('')
         return statuses, reasons, bodies, etags
 
+    def _config_obj_expiration(self, req):
+        delete_at_container = None
+        delete_at_part = None
+        delete_at_nodes = None
+
+        if 'x-delete-after' in req.headers:
+            try:
+                x_delete_after = int(req.headers['x-delete-after'])
+            except ValueError:
+                raise ValueError('Non-integer X-Delete-After')
+
+            req.headers['x-delete-at'] = normalize_delete_at_timestamp(
+                time.time() + x_delete_after)
+
+        if 'x-delete-at' in req.headers:
+            try:
+                x_delete_at = int(normalize_delete_at_timestamp(
+                    int(req.headers['x-delete-at'])))
+            except ValueError:
+                raise ValueError('Non-integer X-Delete-At')
+
+            if x_delete_at < time.time():
+                raise ValueError('X-Delete-At in past')
+
+            req.environ.setdefault('swift.log_info', []).append(
+                'x-delete-at:%s' % x_delete_at)
+            delete_at_container = normalize_delete_at_timestamp(
+                x_delete_at /
+                self.app.expiring_objects_container_divisor *
+                self.app.expiring_objects_container_divisor)
+            delete_at_part, delete_at_nodes = \
+                self.app.container_ring.get_nodes(
+                    self.app.expiring_objects_account, delete_at_container)
+
+        return req, delete_at_container, delete_at_part, delete_at_nodes
+
     @public
     @cors_validation
     @delay_denial
@@ -461,6 +473,7 @@ def PUT(self, req):
         policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
                                        container_info['storage_policy'])
         obj_ring = self.app.get_object_ring(policy_index)
+
         # pass the policy index to storage nodes via req header
         req.headers['X-Backend-Storage-Policy-Index'] = policy_index
         container_partition = container_info['partition']
@@ -472,8 +485,10 @@ def PUT(self, req):
             aresp = req.environ['swift.authorize'](req)
             if aresp:
                 return aresp
+
         if not containers:
             return HTTPNotFound(request=req)
+
         try:
             ml = req.message_length()
         except ValueError as e:
@@ -484,17 +499,10 @@ def PUT(self, req):
                                       body=str(e))
         if ml is not None and ml > constraints.MAX_FILE_SIZE:
             return HTTPRequestEntityTooLarge(request=req)
-        if 'x-delete-after' in req.headers:
-            try:
-                x_delete_after = int(req.headers['x-delete-after'])
-            except ValueError:
-                return HTTPBadRequest(request=req,
-                                      content_type='text/plain',
-                                      body='Non-integer X-Delete-After')
-            req.headers['x-delete-at'] = normalize_delete_at_timestamp(
-                time.time() + x_delete_after)
+
         partition, nodes = obj_ring.get_nodes(
             self.account_name, self.container_name, self.object_name)
+
         # do a HEAD request for container sync and checking object versions
         if 'x-timestamp' in req.headers or \
                 (object_versions and not
@@ -507,6 +515,7 @@ def PUT(self, req):
             hresp = self.GETorHEAD_base(
                 hreq, _('Object'), obj_ring, partition,
                 hreq.swift_entity_path)
+
         # Used by container sync feature
         if 'x-timestamp' in req.headers:
             try:
@@ -522,6 +531,7 @@ def PUT(self, req):
             req.headers['X-Timestamp'] = req_timestamp.internal
         else:
             req.headers['X-Timestamp'] = Timestamp(time.time()).internal
+
         # Sometimes the 'content-type' header exists, but is set to None.
         content_type_manually_set = True
         detect_content_type = \
@@ -585,6 +595,7 @@ def PUT(self, req):
             source_header = '/%s/%s/%s/%s' % (ver, acct,
                                               src_container_name, src_obj_name)
             source_req = req.copy_get()
+
             # make sure the source request uses it's container_info
             source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
             source_req.path_info = source_header
@@ -596,6 +607,7 @@ def PUT(self, req):
             sink_req = Request.blank(req.path_info,
                                      environ=req.environ, headers=req.headers)
             source_resp = self.GET(source_req)
+
             # This gives middlewares a way to change the source; for example,
             # this lets you COPY a SLO manifest and have the new object be the
             # concatenation of the segments (like what a GET request gives
@@ -620,6 +632,7 @@ def PUT(self, req):
             if sink_req.content_length > constraints.MAX_FILE_SIZE:
                 return HTTPRequestEntityTooLarge(request=req)
             sink_req.etag = source_resp.etag
+
             # we no longer need the X-Copy-From header
             del sink_req.headers['X-Copy-From']
             if not content_type_manually_set:
@@ -635,6 +648,7 @@ def PUT(self, req):
                 # copy/update existing sysmeta and user meta
                 copy_headers_into(source_resp, sink_req)
                 copy_headers_into(req, sink_req)
+
             # copy over x-static-large-object for POSTs and manifest copies
             if 'X-Static-Large-Object' in source_resp.headers and \
                     req.params.get('multipart-manifest') == 'get':
@@ -643,28 +657,12 @@ def PUT(self, req):
 
             req = sink_req
 
-        if 'x-delete-at' in req.headers:
-            try:
-                x_delete_at = normalize_delete_at_timestamp(
-                    int(req.headers['x-delete-at']))
-                if int(x_delete_at) < time.time():
-                    return HTTPBadRequest(
-                        body='X-Delete-At in past', request=req,
-                        content_type='text/plain')
-            except ValueError:
-                return HTTPBadRequest(request=req, content_type='text/plain',
-                                      body='Non-integer X-Delete-At')
-            req.environ.setdefault('swift.log_info', []).append(
-                'x-delete-at:%s' % x_delete_at)
-            delete_at_container = normalize_delete_at_timestamp(
-                int(x_delete_at) /
-                self.app.expiring_objects_container_divisor *
-                self.app.expiring_objects_container_divisor)
-            delete_at_part, delete_at_nodes = \
-                self.app.container_ring.get_nodes(
-                    self.app.expiring_objects_account, delete_at_container)
-        else:
-            delete_at_container = delete_at_part = delete_at_nodes = None
+        try:
+            req, delete_at_container, delete_at_part, \
+                delete_at_nodes = self._config_obj_expiration(req)
+        except ValueError as e:
+            return HTTPBadRequest(request=req, content_type='text/plain',
+                                  body=str(e))
 
         node_iter = GreenthreadSafeIterator(
             self.iter_nodes_local_first(obj_ring, partition))