full merge of storage policies to master

bin/swift-container-reconciler

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from swift.container.reconciler import ContainerReconciler
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
+
+if __name__ == '__main__':
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerReconciler, conf_file, **options)

bin/swift-get-nodes

@@ -14,129 +14,69 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import optparse
 import sys
-import urllib
+import os
+from optparse import OptionParser
 
 from swift.common.ring import Ring
-from swift.common.utils import hash_path, storage_directory
+from swift.cli.info import print_item_locations, InfoSystemExit
 
 
-parser = optparse.OptionParser()
-parser.add_option('-a', '--all', action='store_true',
-                  help='Show all handoff nodes')
-parser.add_option('-p', '--partition', metavar='PARTITION',
-                  help='Show nodes for a given partition')
-(options, args) = parser.parse_args()
+if __name__ == '__main__':
 
-if (len(args) < 2 or len(args) > 4) and \
-   (options.partition is None or not args):
-    print 'Usage: %s [-a] <ring.gz> <account> [<container>] [<object>]' \
-        % sys.argv[0]
-    print '   Or: %s [-a] <ring.gz> -p partition' % sys.argv[0]
-    print '   Note: account, container, object can also be a single arg ' \
-        'separated by /'
-    print 'Shows the nodes responsible for the item specified.'
-    print 'Example:'
-    print '    $ %s /etc/swift/account.ring.gz MyAccount' % sys.argv[0]
-    print '    Partition 5743883'
-    print '    Hash 96ae332a60b58910784e4417a03e1ad0'
-    print '    10.1.1.7:8000 sdd1'
-    print '    10.1.9.2:8000 sdb1'
-    print '    10.1.5.5:8000 sdf1'
-    print '    10.1.5.9:8000 sdt1 # [Handoff]'
-    sys.exit(1)
+    usage = '''
+    Shows the nodes responsible for the item specified.
+    Usage: %prog [-a] <ring.gz> <account> [<container>] [<object>]
+    Or: %prog [-a] <ring.gz> -p partition
+    Or: %prog [-a] -P policy_name <account> <container> <object>
+    Note: account, container, object can also be a single arg separated by /
+    Example:
+        $ %prog -a /etc/swift/account.ring.gz MyAccount
+        Partition 5743883
+        Hash 96ae332a60b58910784e4417a03e1ad0
+        10.1.1.7:8000 sdd1
+        10.1.9.2:8000 sdb1
+        10.1.5.5:8000 sdf1
+        10.1.5.9:8000 sdt1 # [Handoff]
+    '''
+    parser = OptionParser(usage)
+    parser.add_option('-a', '--all', action='store_true',
+                      help='Show all handoff nodes')
+    parser.add_option('-p', '--partition', metavar='PARTITION',
+                      help='Show nodes for a given partition')
+    parser.add_option('-P', '--policy-name', dest='policy_name',
+                      help='Specify which policy to use')
+    parser.add_option('-d', '--swift-dir', default='/etc/swift',
+                      dest='swift_dir', help='Path to swift directory')
+    options, args = parser.parse_args()
 
-if len(args) == 2 and '/' in args[1]:
-    # Parse single path arg, as noted in above help text.
-    path = args[1].lstrip('/')
-    args = [args[0]] + [p for p in path.split('/', 2) if p]
-
-ringloc = None
-account = None
-container = None
-obj = None
+    # swift-get-nodes -P nada -p 1
+    if len(args) == 0:
+        if not options.policy_name or not options.partition:
+            sys.exit(parser.print_help())
+    elif len(args) > 4 or len(args) < 1:
+        sys.exit(parser.print_help())
 
-if len(args) == 4:
-    # Account, Container and Object
-    ring_file, account, container, obj = args
-    ring = Ring(ring_file)
-    hash_str = hash_path(account, container, obj)
-    part, nodes = ring.get_nodes(account, container, obj)
-    target = "%s/%s/%s" % (account, container, obj)
-    loc = 'objects'
-elif len(args) == 3:
-    # Account, Container
-    ring_file, account, container = args
-    ring = Ring(ring_file)
-    hash_str = hash_path(account, container)
-    part, nodes = ring.get_nodes(account, container)
-    target = "%s/%s" % (account, container)
-    loc = 'containers'
-elif len(args) == 2:
-    # Account
-    ring_file, account = args
-    ring = Ring(ring_file)
-    hash_str = hash_path(account)
-    part, nodes = ring.get_nodes(account)
-    target = "%s" % (account)
-    loc = 'accounts'
-elif len(args) == 1:
-    # Partition
-    ring_file = args[0]
-    ring = Ring(ring_file)
-    hash_str = None
-    part = int(options.partition)
-    nodes = ring.get_part_nodes(part)
-    target = ''
-    loc = ring_file.rsplit('/', 1)[-1].split('.', 1)[0]
-    if loc in ('account', 'container', 'object'):
-        loc += 's'
-    else:
-        loc = '<type>'
+    # Parse single path arg, as noted in above help text.
+    if len(args) == 1 and options.policy_name and '/' in args[0]:
+        path = args[0].lstrip('/')
+        args = [p for p in path.split('/', 2) if p]
+    if len(args) == 2 and '/' in args[1]:
+        path = args[1].lstrip('/')
+        args = [args[0]] + [p for p in path.split('/', 2) if p]
 
-more_nodes = []
-for more_node in ring.get_more_nodes(part):
-    more_nodes.append(more_node)
-    if not options.all and len(more_nodes) >= len(nodes):
-        break
+    ring = None
+    ring_name = None
 
-print '\nAccount \t%s' % account
-print 'Container\t%s' % container
-print 'Object   \t%s\n' % obj
-print '\nPartition\t%s' % part
-print 'Hash       \t%s\n' % hash_str
+    if len(args) >= 1 and args[0].endswith('ring.gz'):
+        if os.path.exists(args[0]):
+            ring_name = args[0].rsplit('/', 1)[-1].split('.', 1)[0]
+            ring = Ring(args[0])
+        else:
+            print 'Ring file does not exist'
+        args.pop(0)
 
-for node in nodes:
-    print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'],
-                                            node['device'])
-for mnode in more_nodes:
-    print 'Server:Port Device\t%s:%s %s\t [Handoff]' \
-        % (mnode['ip'], mnode['port'], mnode['device'])
-print "\n"
-for node in nodes:
-    print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \
-        % (node['ip'], node['port'], node['device'], part,
-           urllib.quote(target))
-for mnode in more_nodes:
-    print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s" # [Handoff]' \
-        % (mnode['ip'], mnode['port'], mnode['device'], part,
-           urllib.quote(target))
-print "\n"
-print 'Use your own device location of servers:'
-print 'such as "export DEVICE=/srv/node"'
-for node in nodes:
-    if hash_str:
-        print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/"' % (
-            node['ip'], node['device'], storage_directory(loc, part, hash_str))
-    else:
-        print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/%s/"' % (
-            node['ip'], node['device'], loc, part)
-for mnode in more_nodes:
-    if hash_str:
-        print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/" '\
-              '# [Handoff]' % (mnode['ip'], mnode['device'],
-                               storage_directory(loc, part, hash_str))
-    else:
-        print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/%s/" # [Handoff]' % (
-            mnode['ip'], mnode['device'], loc, part)
+    try:
+        print_item_locations(ring, ring_name, *args, **vars(options))
+    except InfoSystemExit:
+        sys.exit(1)

bin/swift-object-info

@@ -14,112 +14,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 import sys
-from datetime import datetime
-from hashlib import md5
 from optparse import OptionParser
 
-from swift.common.ring import Ring
-from swift.obj.diskfile import read_metadata
-from swift.common.utils import hash_path, storage_directory
-
-
-def print_object_info(datafile, check_etag=True, swift_dir='/etc/swift'):
-    if not os.path.exists(datafile) or not datafile.endswith('.data'):
-        print "Data file doesn't exist"
-        sys.exit(1)
-    try:
-        ring = Ring(swift_dir, ring_name='object')
-    except Exception:
-        ring = None
-    fp = open(datafile, 'rb')
-    metadata = read_metadata(fp)
-    path = metadata.pop('name', '')
-    content_type = metadata.pop('Content-Type', '')
-    ts = metadata.pop('X-Timestamp', '')
-    etag = metadata.pop('ETag', '')
-    length = metadata.pop('Content-Length', '')
-    if path:
-        print 'Path: %s' % path
-        account, container, obj = path.split('/', 3)[1:]
-        print '  Account: %s' % account
-        print '  Container: %s' % container
-        print '  Object: %s' % obj
-        obj_hash = hash_path(account, container, obj)
-        print '  Object hash: %s' % obj_hash
-    else:
-        print 'Path: Not found in metadata'
-    if content_type:
-        print 'Content-Type: %s' % content_type
-    else:
-        print 'Content-Type: Not found in metadata'
-    if ts:
-        print 'Timestamp: %s (%s)' % (datetime.fromtimestamp(float(ts)), ts)
-    else:
-        print 'Timestamp: Not found in metadata'
-
-    file_len = None
-    if check_etag:
-        h = md5()
-        file_len = 0
-        while True:
-            data = fp.read(64 * 1024)
-            if not data:
-                break
-            h.update(data)
-            file_len += len(data)
-        h = h.hexdigest()
-        if etag:
-            if h == etag:
-                print 'ETag: %s (valid)' % etag
-            else:
-                print "Etag: %s doesn't match file hash of %s!" % (etag, h)
-        else:
-            print 'ETag: Not found in metadata'
-    else:
-        print 'ETag: %s (not checked)' % etag
-        file_len = os.fstat(fp.fileno()).st_size
-
-    if length:
-        if file_len == int(length):
-            print 'Content-Length: %s (valid)' % length
-        else:
-            print "Content-Length: %s doesn't match file length of %s" % (
-                length, file_len)
-    else:
-        print 'Content-Length: Not found in metadata'
-    print 'User Metadata: %s' % metadata
-    if ring is not None:
-        print 'Ring locations:'
-        part, nodes = ring.get_nodes(account, container, obj)
-        for node in nodes:
-            print ('  %s:%s - /srv/node/%s/%s/%s.data' %
-                   (node['ip'], node['port'], node['device'],
-                    storage_directory('objects', part, obj_hash), ts))
-        print
-        print 'note: /srv/node is used as default value of `devices`, '\
-              'the real value is set in object-server.conf '\
-              'on each storage node.'
-    fp.close()
+from swift.cli.info import print_obj, InfoSystemExit
 
 
 if __name__ == '__main__':
-    parser = OptionParser()
-    parser.set_defaults(check_etag=True, swift_dir='/etc/swift')
+    parser = OptionParser('%prog [options] OBJECT_FILE')
     parser.add_option(
-        '-n', '--no-check-etag',
+        '-n', '--no-check-etag', default=True,
         action="store_false", dest="check_etag",
         help="Don't verify file contents against stored etag")
     parser.add_option(
-        '-d', '--swift-dir',
+        '-d', '--swift-dir', default='/etc/swift', dest='swift_dir',
         help="Pass location of swift directory")
+    parser.add_option(
+        '-P', '--policy-name', dest='policy_name',
+        help="Specify storage policy name")
 
     options, args = parser.parse_args()
 
-    if len(args) < 1:
-        print "Usage: %s [-n] [-d] OBJECT_FILE" % sys.argv[0]
-        sys.exit(1)
+    if len(args) != 1:
+        sys.exit(parser.print_help())
 
-    print_object_info(args[0], check_etag=options.check_etag,
-                      swift_dir=options.swift_dir)
+    try:
+        print_obj(*args, **vars(options))
+    except InfoSystemExit:
+        sys.exit(1)