ipv6 support

bin/swauth-add-account

@@ -18,9 +18,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-add-user

@@ -18,9 +18,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-delete-account

@@ -18,9 +18,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-delete-user

@@ -18,9 +18,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-list

@@ -22,9 +22,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-prep

@@ -18,9 +18,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swauth-set-account-service

@@ -22,9 +22,9 @@ import gettext
 from optparse import OptionParser
 from os.path import basename
 from sys import argv, exit
-from urlparse import urlparse
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
+from swift.common.utils import urlparse
 
 
 if __name__ == '__main__':

bin/swift-ring-builder

@@ -235,10 +235,17 @@ swift-ring-builder <builder_file> add z<zone>-<ip>:<port>/<device_name>_<meta>
             print 'Invalid add value: %s' % argv[3]
             exit(EXIT_ERROR)
         i = 1
-        while i < len(rest) and rest[i] in '0123456789.':
+        if rest[i] == '[':
+            while i < len(rest) and rest[i] != ']':
+                i += 1
+            ip = rest[2:i]
             i += 1
-        ip = rest[1:i]
-        rest = rest[i:]
+            rest = rest[i:]
+        else:
+            while i < len(rest) and rest[i] in '0123456789.':
+                i += 1
+            ip = rest[1:i]
+            rest = rest[i:]
 
         if not rest.startswith(':'):
             print 'Invalid add value: %s' % argv[3]

etc/proxy-server.conf-sample

@@ -68,13 +68,13 @@ use = egg:swift#swauth
 # auth_prefix = /auth/
 # Cluster strings are of the format name:url where name is a short name for the
 # Swift cluster and url is the url to the proxy server(s) for the cluster.
-# default_swift_cluster = local:http://127.0.0.1:8080/v1
+# default_swift_cluster = local#http://127.0.0.1:8080/v1
 # You may also use the format name::url::url where the first url is the one
 # given to users to access their account (public url) and the second is the one
 # used by swauth itself to create and delete accounts (private url). This is
 # useful when a load balancer url should be used by users, but swauth itself is
 # behind the load balancer. Example:
-# default_swift_cluster = local::https://public.com:8080/v1::http://private.com:8080/v1
+# default_swift_cluster = local##https://public.com:8080/v1##http://private.com:8080/v1
 # token_life = 86400
 # node_timeout = 10
 # Highly recommended to change this.

swift/auth/server.py

@@ -20,7 +20,6 @@
 from time import gmtime, strftime, time
 from urllib import unquote, quote
 from uuid import uuid4
-from urlparse import urlparse
 from hashlib import md5, sha1
 import hmac
 import base64
@@ -32,7 +31,7 @@
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
 from swift.common.db import get_db_connection
-from swift.common.utils import get_logger, split_path
+from swift.common.utils import get_logger, split_path, urlparse
 
 
 class AuthController(object):

swift/common/bench.py

@@ -16,13 +16,12 @@
 import uuid
 import time
 import random
-from urlparse import urlparse
 from contextlib import contextmanager
 
 import eventlet.pools
 from eventlet.green.httplib import CannotSendRequest
 
-from swift.common.utils import TRUE_VALUES
+from swift.common.utils import TRUE_VALUES, urlparse
 from swift.common import client
 from swift.common import direct_client
 

swift/common/middleware/acl.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from urlparse import urlparse
+from swift.common.utils import urlparse
 
 
 def clean_acl(name, value):

swift/common/middleware/swauth.py

@@ -21,7 +21,6 @@
 from time import gmtime, strftime, time
 from traceback import format_exc
 from urllib import quote, unquote
-from urlparse import urlparse
 from uuid import uuid4
 
 from eventlet.timeout import Timeout
@@ -32,7 +31,7 @@
 
 from swift.common.bufferedhttp import http_connect_raw as http_connect
 from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
-from swift.common.utils import cache_from_env, get_logger, split_path
+from swift.common.utils import cache_from_env, get_logger, split_path, urlparse
 
 
 class Swauth(object):
@@ -61,23 +60,23 @@ def __init__(self, app, conf):
             self.auth_prefix += '/'
         self.auth_account = '%s.auth' % self.reseller_prefix
         self.default_swift_cluster = conf.get('default_swift_cluster',
-            'local:http://127.0.0.1:8080/v1')
+            'local#http://127.0.0.1:8080/v1')
         # This setting is a little messy because of the options it has to
         # provide. The basic format is cluster_name:url, such as the default
-        # value of local:http://127.0.0.1:8080/v1. But, often the url given to
+        # value of local#http://127.0.0.1:8080/v1. But, often the url given to
         # the user needs to be different than the url used by Swauth to
         # create/delete accounts. So there's a more complex format of
         # cluster_name::url::url, such as
-        # local::https://public.com:8080/v1::http://private.com:8080/v1.
+        # local##https://public.com:8080/v1##http://private.com:8080/v1.
         # The double colon is what sets the two apart.
-        if '::' in self.default_swift_cluster:
+        if '##' in self.default_swift_cluster:
             self.dsc_name, self.dsc_url, self.dsc_url2 = \
-                self.default_swift_cluster.split('::', 2)
+                self.default_swift_cluster.split('##', 2)
             self.dsc_url = self.dsc_url.rstrip('/')
             self.dsc_url2 = self.dsc_url2.rstrip('/')
         else:
             self.dsc_name, self.dsc_url = \
-                self.default_swift_cluster.split(':', 1)
+                self.default_swift_cluster.split('#', 1)
             self.dsc_url = self.dsc_url2 = self.dsc_url.rstrip('/')
         self.dsc_parsed = urlparse(self.dsc_url)
         if self.dsc_parsed.scheme not in ('http', 'https'):

swift/common/utils.py

@@ -34,6 +34,7 @@
 from optparse import OptionParser
 from tempfile import mkstemp
 import cPickle as pickle
+from urlparse import urlparse as stdlib_urlparse, ParseResult
 
 import eventlet
 from eventlet import greenio, GreenPool, sleep, Timeout, listen
@@ -845,3 +846,35 @@ def ratelimit_sleep(running_time, max_rate, incr_by=1):
     elif running_time - now > time_per_request:
         eventlet.sleep((running_time - now) / clock_accuracy)
     return running_time + time_per_request
+
+
+class ModifiedParseResult(ParseResult):
+    "Parse results class for urlparse."
+
+    @property
+    def hostname(self):
+        netloc = self.netloc.split('@', 1)[-1]
+        if netloc.startswith('['):
+            return netloc[1:].split(']')[0]
+        elif ':' in netloc:
+            return netloc.rsplit(':')[0]
+        return netloc
+
+    @property
+    def port(self):
+        netloc = self.netloc.split('@', 1)[-1]
+        if netloc.startswith('['):
+            netloc = netloc.rsplit(']')[1]
+        if ':' in netloc:
+            return int(netloc.rsplit(':')[1])
+        return None
+
+
+def urlparse(url):
+    """
+    urlparse augmentation.
+    This is necessary because urlparse can't handle RFC 2732 URLs.
+
+    :param url: URL to parse.
+    """
+    return ModifiedParseResult(*stdlib_urlparse(url))

swift/common/wsgi.py

@@ -68,11 +68,15 @@ def get_socket(conf, default_port=8080):
     """
     bind_addr = (conf.get('bind_ip', '0.0.0.0'),
                  int(conf.get('bind_port', default_port)))
+    address_family = [addr[0] for addr in socket.getaddrinfo(bind_addr[0],
+            bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM)
+            if addr[0] in (socket.AF_INET, socket.AF_INET6)][0]
     sock = None
     retry_until = time.time() + 30
     while not sock and time.time() < retry_until:
         try:
-            sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)))
+            sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)),
+                        family=address_family)
             if 'cert_file' in conf:
                 sock = ssl.wrap_socket(sock, certfile=conf['cert_file'],
                     keyfile=conf['key_file'])

swift/container/server.py

@@ -88,7 +88,7 @@ def account_update(self, req, account, container, broker):
         account_partition = req.headers.get('X-Account-Partition')
         account_device = req.headers.get('X-Account-Device')
         if all([account_host, account_partition, account_device]):
-            account_ip, account_port = account_host.split(':')
+            account_ip, account_port = account_host.rsplit(':', 1)
             new_path = '/' + '/'.join([account, container])
             info = broker.get_info()
             account_headers = {'x-put-timestamp': info['put_timestamp'],

swift/obj/server.py

@@ -294,7 +294,7 @@ def container_update(self, op, account, container, obj, headers_in,
         full_path = '/%s/%s/%s' % (account, container, obj)
         try:
             with ConnectionTimeout(self.conn_timeout):
-                ip, port = host.split(':')
+                ip, port = host.rsplit(':', 1)
                 conn = http_connect(ip, port, contdevice, partition, op,
                         full_path, headers_out)
             with Timeout(self.node_timeout):

test/unit/common/middleware/test_swauth.py

@@ -151,21 +151,21 @@ def test_default_swift_cluster_init(self):
         app = FakeApp()
         self.assertRaises(Exception, auth.filter_factory({
             'super_admin_key': 'supertest',
-            'default_swift_cluster': 'local:badscheme://host/path'}), app)
+            'default_swift_cluster': 'local#badscheme://host/path'}), app)
         ath = auth.filter_factory({'super_admin_key': 'supertest'})(app)
         self.assertEquals(ath.default_swift_cluster,
-                          'local:http://127.0.0.1:8080/v1')
+                          'local#http://127.0.0.1:8080/v1')
         ath = auth.filter_factory({'super_admin_key': 'supertest',
-            'default_swift_cluster': 'local:http://host/path'})(app)
+            'default_swift_cluster': 'local#http://host/path'})(app)
         self.assertEquals(ath.default_swift_cluster,
-                          'local:http://host/path')
+                          'local#http://host/path')
         ath = auth.filter_factory({'super_admin_key': 'supertest',
-            'default_swift_cluster': 'local:https://host/path/'})(app)
+            'default_swift_cluster': 'local#https://host/path/'})(app)
         self.assertEquals(ath.dsc_url, 'https://host/path')
         self.assertEquals(ath.dsc_url2, 'https://host/path')
         ath = auth.filter_factory({'super_admin_key': 'supertest',
             'default_swift_cluster':
-                'local::https://host/path/::http://host2/path2/'})(app)
+                'local##https://host/path/##http://host2/path2/'})(app)
         self.assertEquals(ath.dsc_url, 'https://host/path')
         self.assertEquals(ath.dsc_url2, 'http://host2/path2')
 
@@ -2882,15 +2882,15 @@ def test_get_conn_default(self):
 
     def test_get_conn_default_https(self):
         local_auth = auth.filter_factory({'super_admin_key': 'supertest',
-            'default_swift_cluster': 'local:https://1.2.3.4/v1'})(FakeApp())
+            'default_swift_cluster': 'local#https://1.2.3.4/v1'})(FakeApp())
         conn = local_auth.get_conn()
         self.assertEquals(conn.__class__, auth.HTTPSConnection)
         self.assertEquals(conn.host, '1.2.3.4')
         self.assertEquals(conn.port, 443)
 
     def test_get_conn_overridden(self):
         local_auth = auth.filter_factory({'super_admin_key': 'supertest',
-            'default_swift_cluster': 'local:https://1.2.3.4/v1'})(FakeApp())
+            'default_swift_cluster': 'local#https://1.2.3.4/v1'})(FakeApp())
         conn = \
             local_auth.get_conn(urlparsed=auth.urlparse('http://5.6.7.8/v1'))
         self.assertEquals(conn.__class__, auth.HTTPConnection)
@@ -2899,7 +2899,7 @@ def test_get_conn_overridden(self):
 
     def test_get_conn_overridden_https(self):
         local_auth = auth.filter_factory({'super_admin_key': 'supertest',
-            'default_swift_cluster': 'local:http://1.2.3.4/v1'})(FakeApp())
+            'default_swift_cluster': 'local#http://1.2.3.4/v1'})(FakeApp())
         conn = \
             local_auth.get_conn(urlparsed=auth.urlparse('https://5.6.7.8/v1'))
         self.assertEquals(conn.__class__, auth.HTTPSConnection)

test/unit/common/test_utils.py

@@ -477,6 +477,27 @@ def test_ratelimit_sleep_with_incr(self):
             total += i
         self.assertTrue(abs(50 - (time.time() - start) * 100) < 10)
 
+    def test_urlparse(self):
+        parsed = utils.urlparse('http://127.0.0.1/')
+        self.assertEquals(parsed.scheme, 'http')
+        self.assertEquals(parsed.hostname, '127.0.0.1')
+        self.assertEquals(parsed.path, '/')
+
+        parsed = utils.urlparse('http://127.0.0.1:8080/')
+        self.assertEquals(parsed.port, 8080)
+
+        parsed = utils.urlparse('https://127.0.0.1/')
+        self.assertEquals(parsed.scheme, 'https')
+
+        parsed = utils.urlparse('http://[::1]/')
+        self.assertEquals(parsed.hostname, '::1')
+
+        parsed = utils.urlparse('http://[::1]:8080/')
+        self.assertEquals(parsed.hostname, '::1')
+        self.assertEquals(parsed.port, 8080)
+
+        parsed = utils.urlparse('www.example.com')
+        self.assertEquals(parsed.hostname, '')
 
 if __name__ == '__main__':
     unittest.main()