Merge master into EC branch

Change-Id: I4466174c960d2a83b53a5c55f6cc82c2299b54dc

doc/manpages/container-server.conf.5

@@ -119,11 +119,11 @@ This is normally \fBegg:swift#container\fR.
 Label used when logging. The default is container-server.
 .IP "\fBset log_facility\fR
 Syslog log facility. The default is LOG_LOCAL0.
-.IP "\fB set log_level\fR
+.IP "\fBset log_level\fR
 Logging level. The default is INFO.
-.IP "\fB set log_requests\fR
+.IP "\fBset log_requests\fR
 Enables request logging. The default is True.
-.IP "\fB set log_address\fR
+.IP "\fBset log_address\fR
 Logging address. The default is /dev/log.
 .IP \fBnode_timeout\fR
 Request timeout to external services. The default is 3 seconds.
@@ -224,7 +224,7 @@ Number of reaper workers to spawn. The default is 4.
 Request timeout to external services. The default is 3 seconds.
 .IP \fBconn_timeout\fR
 Connection timeout to external services. The default is 0.5 seconds.
-.IP \fBslowdown = 0.01\fR
+.IP \fBslowdown\fR
 Slowdown will sleep that amount between containers. The default is 0.01 seconds.
 .IP \fBaccount_suppression_time\fR
 Seconds to suppress updating an account that has generated an error. The default is 60 seconds.
@@ -286,4 +286,4 @@ and
 
 
 .SH "SEE ALSO"
-.BR swift-container-server(1),
+.BR swift-container-server(1)

doc/manpages/object-server.conf.5

@@ -236,7 +236,7 @@ Number of reaper workers to spawn. The default is 1.
 Request timeout to external services. The default is 10 seconds.
 .IP \fBconn_timeout\fR
 Connection timeout to external services. The default is 0.5 seconds.
-.IP \fBslowdown = 0.01\fR
+.IP \fBslowdown\fR
 Slowdown will sleep that amount between objects. The default is 0.01 seconds.
 .RE
 .PD

doc/manpages/proxy-server.conf.5

@@ -475,7 +475,7 @@ Default is localhost.
 Default is 8125.
 .IP \fBaccess_log_statsd_default_sample_rate\fR
 Default is 1.
-.IP \fBaccess_log_statsd_metric_prefix =
+.IP \fBaccess_log_statsd_metric_prefix\fR
 Default is "" (empty-string)
 .IP \fBaccess_log_headers\fR
 Default is False.
@@ -499,13 +499,13 @@ that are acceptable within this section.
 .IP \fBuse\fR
 Entry point for paste.deploy for the proxy server. This is the reference to the installed python egg.
 This is normally \fBegg:swift#proxy\fR.
-.IP \fBset log_name\fR
+.IP "\fBset log_name\fR"
 Label used when logging. The default is proxy-server.
-.IP \fBset log_facility\fR
+.IP "\fBset log_facility\fR"
 Syslog log facility. The default is LOG_LOCAL0.
-.IP \fB set log_level\fR
+.IP "\fBset log_level\fR"
 Logging level. The default is INFO.
-.IP \fB set log_address\fR
+.IP "\fBset log_address\fR"
 Logging address. The default is /dev/log.
 .IP \fBlog_handoffs\fR
 Log when handoff locations are used.  Default is True.

doc/manpages/swift-orphans.1

@@ -14,31 +14,31 @@
 .\" implied.
 .\" See the License for the specific language governing permissions and
 .\" limitations under the License.
-.\"  
+.\"
 .TH swift-orphans 1 "3/15/2012" "Linux" "OpenStack Swift"
 
-.SH NAME 
+.SH NAME
 .LP
 .B swift-orphans
 \- Openstack-swift orphans tool
 
 .SH SYNOPSIS
 .LP
-.B swift-orphans 
+.B swift-orphans
 [-h|--help] [-a|--age] [-k|--kill] [-w|--wide] [-r|--run-dir]
 
 
-.SH DESCRIPTION 
+.SH DESCRIPTION
 .PP
 Lists and optionally kills orphaned Swift processes. This is done by scanning
-/var/run/swift or the directory specified to the -r switch for .pid files and
+/var/run/swift or the directory specified to the \-r switch for .pid files and
 listing any processes that look like Swift processes but aren't associated with
 the pids in those .pid files. Any Swift processes running with the 'once'
 parameter are ignored, as those are usually for full-speed audit scans and
 such.
 
-Example (sends SIGTERM to all orphaned Swift processes older than two hours): 
-swift-orphans -a 2 -k TERM
+Example (sends SIGTERM to all orphaned Swift processes older than two hours):
+swift-orphans \-a 2 \-k TERM
 
 The options are as follows:
 
@@ -62,9 +62,9 @@ The options are as follows:
 .PD
 .RE
 
-    
+
 .SH DOCUMENTATION
 .LP
-More documentation about Openstack-Swift can be found at 
+More documentation about Openstack-Swift can be found at
 .BI http://swift.openstack.org/index.html
 

doc/manpages/swift-recon.1

@@ -14,26 +14,26 @@
 .\" implied.
 .\" See the License for the specific language governing permissions and
 .\" limitations under the License.
-.\"  
+.\"
 .TH swift-recon 1 "8/26/2011" "Linux" "OpenStack Swift"
 
-.SH NAME 
+.SH NAME
 .LP
 .B swift-recon
 \- Openstack-swift recon middleware cli tool
 
 .SH SYNOPSIS
 .LP
-.B swift-recon 
+.B swift-recon
 \ <server_type> [-v] [--suppress] [-a] [-r] [-u] [-d] [-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
- 
-.SH DESCRIPTION 
+
+.SH DESCRIPTION
 .PP
 The swift-recon cli tool can be used to retrieve various metrics and telemetry information about
-a cluster that has been collected by the swift-recon middleware. 
+a cluster that has been collected by the swift-recon middleware.
 
-In order to make use of the swift-recon middleware, update the object-server.conf file and 
-enable the recon middleware by adding a pipeline entry and setting its option(s). You can view 
+In order to make use of the swift-recon middleware, update the object-server.conf file and
+enable the recon middleware by adding a pipeline entry and setting its option(s). You can view
 more information in the example section below.
 
 
@@ -69,13 +69,13 @@ Get cluster quarantine stats
 .IP "\fB--md5\fR"
 Get md5sum of servers ring and compare to local cop
 .IP "\fB--all\fR"
-Perform all checks. Equivalent to -arudlq --md5
+Perform all checks. Equivalent to \-arudlq \-\-md5
 .IP "\fB-z ZONE, --zone=ZONE\fR"
 Only query servers in specified zone
 .IP "\fB--swiftdir=PATH\fR"
 Default = /etc/swift
 .PD
-.RE  
+.RE
 
 
 
@@ -84,16 +84,16 @@ Default = /etc/swift
 .PD 0
 .RS 0
 .IP "ubuntu:~$ swift-recon -q --zone 3"
-.IP "==============================================================================="
+.IP "================================================================="
 .IP "[2011-10-18 19:36:00] Checking quarantine dirs on 1 hosts... "
 .IP "[Quarantined objects] low: 4, high: 4, avg: 4, total: 4 "
 .IP "[Quarantined accounts] low: 0, high: 0, avg: 0, total: 0 "
 .IP "[Quarantined containers] low: 0, high: 0, avg: 0, total: 0 "
-.IP "==============================================================================="
+.IP "================================================================="
 .RE
 
 .RS 0
-Finally if you also wish to track asynchronous pending’s you will need to setup a 
+Finally if you also wish to track asynchronous pending’s you will need to setup a
 cronjob to run the swift-recon-cron script periodically:
 
 .IP "*/5 * * * * swift /usr/bin/swift-recon-cron /etc/swift/object-server.conf"
@@ -104,9 +104,9 @@ cronjob to run the swift-recon-cron script periodically:
 
 .SH DOCUMENTATION
 .LP
-More documentation about Openstack-Swift can be found at 
-.BI http://swift.openstack.org/index.html 
-Also more specific documentation about swift-recon can be found at 
+More documentation about Openstack-Swift can be found at
+.BI http://swift.openstack.org/index.html
+Also more specific documentation about swift-recon can be found at
 .BI http://swift.openstack.org/admin_guide.html#cluster-telemetry-and-monitoring
 
 

doc/source/backends.rst

@@ -0,0 +1,16 @@
+======================================
+Pluggable Back-ends: API Documentation
+======================================
+
+.. automodule:: swift.account.backend
+    :private-members:
+    :members:
+    :undoc-members:
+
+.. automodule:: swift.container.backend
+    :private-members:
+    :members:
+    :undoc-members:
+
+.. automodule:: swift.obj.diskfile
+    :members:

doc/source/deployment_guide.rst

@@ -414,6 +414,19 @@ stats_interval      3600               Interval in seconds between logging
                                        replication statistics
 reclaim_age         604800             Time elapsed in seconds before an
                                        object can be reclaimed
+handoffs_first      false              If set to True, partitions that are
+                                       not supposed to be on the node will be
+                                       replicated first.  The default setting
+                                       should not be changed, except for
+                                       extreme situations.
+handoff_delete      auto               By default handoff partitions will be
+                                       removed when it has successfully
+                                       replicated to all the cannonical nodes.
+                                       If set to an integer n, it will remove
+                                       the partition if it is successfully
+                                       replicated to n nodes.  The default
+                                       setting should not be changed, except
+                                       for extremem situations.
 ==================  =================  =======================================
 
 [object-updater]

doc/source/development_saio.rst

@@ -35,14 +35,14 @@ Installing dependencies
 * On apt based systems,
 
   #. `apt-get update`
-  #. `apt-get install curl gcc memcached rsync sqlite3 xfsprogs git-core python-setuptools`
+  #. `apt-get install curl gcc memcached rsync sqlite3 xfsprogs git-core libffi-dev python-setuptools`
   #. `apt-get install python-coverage python-dev python-nose python-simplejson
      python-xattr python-eventlet python-greenlet python-pastedeploy
      python-netifaces python-pip python-dnspython python-mock`
 
 * On yum based systems,
 
-  #. `yum install curl gcc memcached rsync sqlite xfsprogs git-core xinetd python-setuptools`
+  #. `yum install curl gcc memcached rsync sqlite xfsprogs git-core libffi-devel xinetd python-setuptools`
   #. `yum install python-coverage python-devel python-nose python-simplejson
      python-xattr python-eventlet python-greenlet python-pastedeploy
      python-netifaces python-pip python-dnspython python-mock`
@@ -306,7 +306,8 @@ Sample configuration files are provided with all defaults in line-by-line commen
         eventlet_debug = true
 
         [pipeline:main]
-        pipeline = healthcheck cache tempauth proxy-logging proxy-server
+        # Yes, proxy-logging appears twice. This is not a mistake.
+        pipeline = healthcheck proxy-logging cache tempauth proxy-logging proxy-server
 
         [app:proxy-server]
         use = egg:swift#proxy

doc/source/index.rst

@@ -66,6 +66,7 @@ Developer Documentation
     development_guidelines
     development_saio
     development_auth
+    backends
 
 Administrator Documentation
 ===========================

doc/source/misc.rst

@@ -33,6 +33,15 @@ Utils
     :members:
     :show-inheritance:
 
+.. _ondisk:
+
+OnDisk
+======
+
+.. automodule:: swift.common.ondisk
+    :members:
+    :show-inheritance:
+
 .. _common_tempauth:
 
 TempAuth

examples/apache2/account-server.template

@@ -2,17 +2,19 @@
 #
 # Change %PORT% to the port that you wish to use on your system
 # Change %SERVICENAME% to the service name you are using
+# Change %USER% to the system user that will run the daemon process
 # Change the debug level as you see fit
 #
 # For example:
 #     Replace %PORT% by 6012
 #     Replace %SERVICENAME% by account-server-1
+#     Replace %USER% with apache (or remove it for default)
 
 NameVirtualHost *:%PORT%
 Listen %PORT%
 
 <VirtualHost *:%PORT%>
-    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1
+    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER%
     WSGIProcessGroup %SERVICENAME%
     WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi
     WSGIApplicationGroup %{GLOBAL}

examples/apache2/container-server.template

@@ -2,17 +2,19 @@
 #
 # Change %PORT% to the port that you wish to use on your system
 # Change %SERVICENAME% to the service name you are using
+# Change %USER% to the system user that will run the daemon process
 # Change the debug level as you see fit
 #
 # For example:
 #     Replace %PORT% by 6011
 #     Replace %SERVICENAME% by container-server-1
+#     Replace %USER% with apache (or remove it for default)
 
 NameVirtualHost *:%PORT%
 Listen %PORT%
 
 <VirtualHost *:%PORT%>
-    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1
+    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER%
     WSGIProcessGroup %SERVICENAME%
     WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi
     WSGIApplicationGroup %{GLOBAL}

examples/apache2/object-server.template

@@ -2,18 +2,19 @@
 #
 # Change %PORT% to the port that you wish to use on your system
 # Change %SERVICENAME% to the service name you are using
+# Change %USER% to the system user that will run the daemon process
 # Change the debug level as you see fit
 #
 # For example:
 #     Replace %PORT% by 6010
 #     Replace %SERVICENAME% by object-server-1
-
+#     Replace %USER% with apache (or remove it for default)
 
 NameVirtualHost *:%PORT%
 Listen %PORT%
 
 <VirtualHost *:%PORT%>
-    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1
+    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER%
     WSGIProcessGroup %SERVICENAME%
     WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi
     WSGIApplicationGroup %{GLOBAL}

examples/apache2/proxy-server.template

@@ -2,19 +2,21 @@
 #
 # Change %PORT% to the port that you wish to use on your system
 # Change %SERVICENAME% to the service name you are using
+# Change %USER% to the system user that will run the daemon process
 # Change the debug level as you see fit
 #
 # For example:
 #     Replace %PORT% by 8080
 #     Replace %SERVICENAME% by proxy-server
+#     Replace %USER% with apache (or remove it for default)
 
 NameVirtualHost *:%PORT%
 Listen %PORT%
 
 <VirtualHost *:%PORT%>
     # The limit of an object size
     LimitRequestBody 5368709122
-    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1
+    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER%
     WSGIProcessGroup %SERVICENAME%
     WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi
     WSGIApplicationGroup %{GLOBAL}

swift/account/auditor.py

@@ -20,9 +20,10 @@
 
 import swift.common.db
 from swift.account import server as account_server
-from swift.common.db import AccountBroker
-from swift.common.utils import get_logger, audit_location_generator, \
-    config_true_value, dump_recon_cache, ratelimit_sleep
+from swift.account.backend import AccountBroker
+from swift.common.utils import get_logger, config_true_value, \
+    dump_recon_cache, ratelimit_sleep
+from swift.common.ondisk import audit_location_generator
 from swift.common.daemon import Daemon
 
 from eventlet import Timeout