swift-stats-report

#!/usr/bin/python -u
# Copyright (c) 2010-2011 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import csv
import os
import socket
from ConfigParser import ConfigParser
from httplib import HTTPException
from optparse import OptionParser
from sys import argv, exit, stderr
from time import time
from uuid import uuid4

from eventlet import GreenPool, hubs, patcher, sleep, Timeout
from eventlet.pools import Pool

from swift.common import direct_client
from swift.common.client import ClientException, Connection, get_auth
from swift.common.ring import Ring
from swift.common.utils import compute_eta, get_time_units


unmounted = []

def get_error_log(prefix):
    def error_log(msg_or_exc):
        global unmounted
        if hasattr(msg_or_exc, 'http_status') and \
                msg_or_exc.http_status == 507:
            identifier = '%s:%s/%s'
            if identifier not in unmounted:
                unmounted.append(identifier)
                print >>stderr, 'ERROR: %s:%s/%s is unmounted -- This will ' \
                    'cause replicas designated for that device to be ' \
                    'considered missing until resolved or the ring is ' \
                    'updated.' % (msg_or_exc.http_host, msg_or_exc.http_port,
                    msg_or_exc.http_device)
        if not hasattr(msg_or_exc, 'http_status') or \
                msg_or_exc.http_status not in (404, 507):
            print >>stderr, 'ERROR: %s: %s' % (prefix, msg_or_exc)
    return error_log


def audit(coropool, connpool, account, container_ring, object_ring, options):
    begun = time()
    with connpool.item() as conn:
        estimated_items = \
            [int(conn.head_account()['x-account-container-count'])]
    items_completed = [0]
    retries_done = [0]
    containers_missing_replicas = {}
    objects_missing_replicas = {}
    next_report = [time() + 2]
    def report():
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = \
                compute_eta(begun, items_completed[0], estimated_items[0])
            print '\r\x1B[KAuditing items: %d of %d, %d%s left, %d ' \
                  'retries' % (items_completed[0], estimated_items[0],
                  round(eta), eta_unit, retries_done[0]),
    def direct_container(container, part, nodes):
        estimated_objects = 0
        for node in nodes:
            found = False
            error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
            try:
                attempts, info = direct_client.retry(
                                    direct_client.direct_head_container, node,
                                    part, account, container,
                                    error_log=error_log,
                                    retries=options.retries)
                retries_done[0] += attempts - 1
                found = True
                if not estimated_objects:
                    estimated_objects = int(info['x-container-object-count'])
            except ClientException, err:
                if err.http_status not in (404, 507):
                    error_log('Giving up on /%s/%s/%s: %s' % (part, account,
                              container, err))
            except (Exception, Timeout), err:
                error_log('Giving up on /%s/%s/%s: %s' % (part, account,
                          container, err))
            if not found:
                if container in containers_missing_replicas:
                    containers_missing_replicas[container].append(node)
                else:
                    containers_missing_replicas[container] = [node]
        estimated_items[0] += estimated_objects
        items_completed[0] += 1
        report()
    def direct_object(container, obj, part, nodes):
        for node in nodes:
            found = False
            error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
            try:
                attempts, _junk = direct_client.retry(
                                direct_client.direct_head_object, node, part,
                                account, container, obj, error_log=error_log,
                                retries=options.retries)
                retries_done[0] += attempts - 1
                found = True
            except ClientException, err:
                if err.http_status not in (404, 507):
                    error_log('Giving up on /%s/%s/%s: %s' % (part, account,
                              container, err))
            except (Exception, Timeout), err:
                error_log('Giving up on /%s/%s/%s: %s' % (part, account,
                          container, err))
            if not found:
                opath = '/%s/%s' % (container, obj)
                if opath in objects_missing_replicas:
                    objects_missing_replicas[opath].append(node)
                else:
                    objects_missing_replicas[opath] = [node]
        items_completed[0] += 1
        report()
    cmarker = ''
    while True:
        with connpool.item() as conn:
            containers = \
                [c['name'] for c in conn.get_account(marker=cmarker)[1]]
        if not containers:
            break
        cmarker = containers[-1]
        for container in containers:
            part, nodes = container_ring.get_nodes(account, container)
            coropool.spawn(direct_container, container, part, nodes)
        for container in containers:
            omarker = ''
            while True:
                with connpool.item() as conn:
                    objects = [o['name'] for o in
                        conn.get_container(container, marker=omarker)[1]]
                if not objects:
                    break
                omarker = objects[-1]
                for obj in objects:
                    part, nodes = object_ring.get_nodes(account, container, obj)
                    coropool.spawn(direct_object, container, obj, part, nodes)
    coropool.waitall()
    print '\r\x1B[K\r',
    if not containers_missing_replicas and not objects_missing_replicas:
        print 'No missing items.'
        return
    if containers_missing_replicas:
        print 'Containers Missing'
        print '-' * 78
        for container in sorted(containers_missing_replicas.keys()):
            part, _junk = container_ring.get_nodes(account, container)
            for node in containers_missing_replicas[container]:
                print 'http://%s:%s/%s/%s/%s/%s' % (node['ip'], node['port'],
                      node['device'], part, account, container)
    if objects_missing_replicas:
        if containers_missing_replicas:
            print
        print 'Objects Missing'
        print '-' * 78
        for opath in sorted(objects_missing_replicas.keys()):
            _junk, container, obj = opath.split('/', 2)
            part, _junk = object_ring.get_nodes(account, container, obj)
            for node in objects_missing_replicas[opath]:
                print 'http://%s:%s/%s/%s/%s/%s/%s' % (node['ip'],
                      node['port'], node['device'], part, account, container,
                      obj)


def container_dispersion_report(coropool, connpool, account, container_ring,
                                options):
    """ Returns (number of containers listed, number of distinct partitions,
                 number of container copies found) """
    with connpool.item() as conn:
        containers = [c['name'] for c in
                      conn.get_account(prefix='stats_container_dispersion_',
                                       full_listing=True)[1]]
    containers_listed = len(containers)
    if not containers_listed:
        print >>stderr, 'No containers to query. Has stats-populate been run?'
        return 0
    retries_done = [0]
    containers_queried = [0]
    container_copies_found = [0, 0, 0, 0]
    begun = time()
    next_report = [time() + 2]
    def direct(container, part, nodes):
        found_count = 0
        for node in nodes:
            error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
            try:
                attempts, _junk = direct_client.retry(
                                direct_client.direct_head_container, node,
                                part, account, container, error_log=error_log,
                                retries=options.retries)
                retries_done[0] += attempts - 1
                found_count += 1
            except ClientException, err:
                if err.http_status not in (404, 507):
                    error_log('Giving up on /%s/%s/%s: %s' % (part, account,
                              container, err))
            except (Exception, Timeout), err:
                error_log('Giving up on /%s/%s/%s: %s' % (part, account,
                          container, err))
        container_copies_found[found_count] += 1
        containers_queried[0] += 1
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, containers_queried[0],
                                        containers_listed)
            print '\r\x1B[KQuerying containers: %d of %d, %d%s left, %d ' \
                  'retries' % (containers_queried[0], containers_listed,
                  round(eta), eta_unit, retries_done[0]),
    container_parts = {}
    for container in containers:
        part, nodes = container_ring.get_nodes(account, container)
        if part not in container_parts:
            container_parts[part] = part
            coropool.spawn(direct, container, part, nodes)
    coropool.waitall()
    distinct_partitions = len(container_parts)
    copies_expected = distinct_partitions * container_ring.replica_count
    copies_found = sum(a * b for a, b in enumerate(container_copies_found))
    value = 100.0 * copies_found / copies_expected
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KQueried %d containers for dispersion reporting, ' \
              '%d%s, %d retries' % (containers_listed, round(elapsed),
              elapsed_unit, retries_done[0])
        if containers_listed - distinct_partitions:
            print 'There were %d overlapping partitions' % (
                  containers_listed - distinct_partitions)
        if container_copies_found[2]:
            print 'There were %d partitions missing one copy.' % \
                  container_copies_found[2]
        if container_copies_found[1]:
            print '! There were %d partitions missing two copies.' % \
                  container_copies_found[1]
        if container_copies_found[0]:
            print '!!! There were %d partitions missing all copies.' % \
                  container_copies_found[0]
        print '%.02f%% of container copies found (%d of %d)' % (
              value, copies_found, copies_expected)
        print 'Sample represents %.02f%% of the container partition space' % (
              100.0 * distinct_partitions / container_ring.partition_count)
    return value


def object_dispersion_report(coropool, connpool, account, object_ring, options):
    """ Returns (number of objects listed, number of distinct partitions,
                 number of object copies found) """
    container = 'stats_objects'
    with connpool.item() as conn:
        try:
            objects = [o['name'] for o in conn.get_container(container,
                prefix='stats_object_dispersion_', full_listing=True)[1]]
        except ClientException, err:
            if err.http_status != 404:
                raise
            print >>stderr, 'No objects to query. Has stats-populate been run?'
            return 0
    objects_listed = len(objects)
    if not objects_listed:
        print >>stderr, 'No objects to query. Has stats-populate been run?'
        return 0
    retries_done = [0]
    objects_queried = [0]
    object_copies_found = [0, 0, 0, 0]
    begun = time()
    next_report = [time() + 2]
    def direct(obj, part, nodes):
        found_count = 0
        for node in nodes:
            error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
            try:
                attempts, _junk = direct_client.retry(
                                direct_client.direct_head_object, node, part,
                                account, container, obj, error_log=error_log,
                                retries=options.retries)
                retries_done[0] += attempts - 1
                found_count += 1
            except ClientException, err:
                if err.http_status not in (404, 507):
                    error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account,
                              container, obj, err))
            except (Exception, Timeout), err:
                error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account,
                          container, obj, err))
        object_copies_found[found_count] += 1
        objects_queried[0] += 1
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, objects_queried[0],
                                        objects_listed)
            print '\r\x1B[KQuerying objects: %d of %d, %d%s left, %d ' \
                  'retries' % (objects_queried[0], objects_listed, round(eta),
                  eta_unit, retries_done[0]),
    object_parts = {}
    for obj in objects:
        part, nodes = object_ring.get_nodes(account, container, obj)
        if part not in object_parts:
            object_parts[part] = part
            coropool.spawn(direct, obj, part, nodes)
    coropool.waitall()
    distinct_partitions = len(object_parts)
    copies_expected = distinct_partitions * object_ring.replica_count
    copies_found = sum(a * b for a, b in enumerate(object_copies_found))
    value = 100.0 * copies_found / copies_expected
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KQueried %d objects for dispersion reporting, ' \
              '%d%s, %d retries' % (objects_listed, round(elapsed),
              elapsed_unit, retries_done[0])
        if objects_listed - distinct_partitions:
            print 'There were %d overlapping partitions' % (
                  objects_listed - distinct_partitions)
        if object_copies_found[2]:
            print 'There were %d partitions missing one copy.' % \
                  object_copies_found[2]
        if object_copies_found[1]:
            print '! There were %d partitions missing two copies.' % \
                  object_copies_found[1]
        if object_copies_found[0]:
            print '!!! There were %d partitions missing all copies.' % \
                  object_copies_found[0]
        print '%.02f%% of object copies found (%d of %d)' % (
              value, copies_found, copies_expected)
        print 'Sample represents %.02f%% of the object partition space' % (
              100.0 * distinct_partitions / object_ring.partition_count)
    return value


def container_put_report(coropool, connpool, count, options):
    successes = [0]
    failures = [0]
    retries_done = [0]
    begun = time()
    next_report = [time() + 2]
    def put(container):
        with connpool.item() as conn:
            try:
                conn.put_container(container)
                successes[0] += 1
            except (Exception, Timeout):
                failures[0] += 1
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, successes[0] + failures[0],
                                        count)
            print '\r\x1B[KCreating containers: %d of %d, %d%s left, %d ' \
                  'retries' % (successes[0] + failures[0], count, eta,
                  eta_unit, retries_done[0]),
    for x in xrange(count):
        coropool.spawn(put, 'stats_container_put_%02x' % x)
    coropool.waitall()
    successes = successes[0]
    failures = failures[0]
    value = 100.0 * successes / count
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KCreated %d containers for performance reporting, ' \
              '%d%s, %d retries' % (count, round(elapsed), elapsed_unit,
              retries_done[0])
        print '%d succeeded, %d failed, %.02f%% success rate' % (
              successes, failures, value)
    return value


def container_head_report(coropool, connpool, options):
    successes = [0]
    failures = [0]
    retries_done = [0]
    begun = time()
    next_report = [time() + 2]
    with connpool.item() as conn:
        containers = [c['name'] for c in
                      conn.get_account(prefix='stats_container_put_',
                      full_listing=True)[1]]
    count = len(containers)
    def head(container):
        with connpool.item() as conn:
            try:
                conn.head_container(container)
                successes[0] += 1
            except (Exception, Timeout):
                failures[0] += 1
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, successes[0] + failures[0],
                                        count)
            print '\r\x1B[KHeading containers: %d of %d, %d%s left, %d ' \
                  'retries' % (successes[0] + failures[0], count, eta,
                  eta_unit, retries_done[0]),
    for container in containers:
        coropool.spawn(head, container)
    coropool.waitall()
    successes = successes[0]
    failures = failures[0]
    value = 100.0 * successes / len(containers)
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KHeaded %d containers for performance reporting, ' \
              '%d%s, %d retries' % (count, round(elapsed), elapsed_unit,
              retries_done[0])
        print '%d succeeded, %d failed, %.02f%% success rate' % (
              successes, failures, value)
    return value


def container_get_report(coropool, connpool, options):
    successes = [0]
    failures = [0]
    retries_done = [0]
    begun = time()
    next_report = [time() + 2]
    with connpool.item() as conn:
        containers = [c['name'] for c in
                      conn.get_account(prefix='stats_container_put_',
                      full_listing=True)[1]]
    count = len(containers)
    def get(container):
        with connpool.item() as conn:
            try:
                conn.get_container(container)
                successes[0] += 1
            except (Exception, Timeout):
                failures[0] += 1
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, successes[0] + failures[0],
                                        count)
            print '\r\x1B[KListing containers: %d of %d, %d%s left, %d ' \
                  'retries' % (successes[0] + failures[0], count, eta,
                  eta_unit, retries_done[0]),
    for container in containers:
        coropool.spawn(get, container)
    coropool.waitall()
    successes = successes[0]
    failures = failures[0]
    value = 100.0 * successes / len(containers)
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KListing %d containers for performance reporting, ' \
              '%d%s, %d retries' % (count, round(elapsed), elapsed_unit,
              retries_done[0])
        print '%d succeeded, %d failed, %.02f%% success rate' % (
              successes, failures, value)
    return value


def container_standard_listing_report(coropool, connpool, options):
    begun = time()
    if options.verbose:
        print 'Listing big_container',
    with connpool.item() as conn:
        try:
            value = \
                len(conn.get_container('big_container', full_listing=True)[1])
        except ClientException, err:
            if err.http_status != 404:
                raise
            print >>stderr, \
                  "big_container doesn't exist. Has stats-populate been run?"
            return 0
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\rGot %d objects (standard listing) in big_container, %d%s' % \
              (value, elapsed, elapsed_unit)
    return value


def container_prefix_listing_report(coropool, connpool, options):
    begun = time()
    if options.verbose:
        print 'Prefix-listing big_container',
    value = 0
    with connpool.item() as conn:
        try:
            for x in xrange(256):
                value += len(conn.get_container('big_container',
                                prefix=('%02x' % x), full_listing=True)[1])
        except ClientException, err:
            if err.http_status != 404:
                raise
            print >>stderr, \
                  "big_container doesn't exist. Has stats-populate been run?"
            return 0
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\rGot %d objects (prefix listing) in big_container, %d%s' % \
              (value, elapsed, elapsed_unit)
    return value


def container_prefix_delimiter_listing_report(coropool, connpool, options):
    begun = time()
    if options.verbose:
        print 'Prefix-delimiter-listing big_container',
    value = [0]
    def list(prefix=None):
        marker = None
        while True:
            try:
                with connpool.item() as conn:
                    listing = conn.get_container('big_container',
                                marker=marker, prefix=prefix, delimiter='/')[1]
            except ClientException, err:
                if err.http_status != 404:
                    raise
                print >>stderr, "big_container doesn't exist. " \
                                "Has stats-populate been run?"
                return 0
            if not len(listing):
                break
            marker = listing[-1].get('name', listing[-1].get('subdir'))
            value[0] += len(listing)
            subdirs = []
            i = 0
            # Capping the subdirs we'll list per dir to 10
            while len(subdirs) < 10 and i < len(listing):
                if 'subdir' in listing[i]:
                    subdirs.append(listing[i]['subdir'])
                i += 1
            del listing
            for subdir in subdirs:
                coropool.spawn(list, subdir)
                sleep()
    coropool.spawn(list)
    coropool.waitall()
    value = value[0]
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\rGot %d objects/subdirs in big_container, %d%s' % (value,
              elapsed, elapsed_unit)
    return value


def container_delete_report(coropool, connpool, options):
    successes = [0]
    failures = [0]
    retries_done = [0]
    begun = time()
    next_report = [time() + 2]
    with connpool.item() as conn:
        containers = [c['name'] for c in
                      conn.get_account(prefix='stats_container_put_',
                      full_listing=True)[1]]
    count = len(containers)
    def delete(container):
        with connpool.item() as conn:
            try:
                conn.delete_container(container)
                successes[0] += 1
            except (Exception, Timeout):
                failures[0] += 1
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, successes[0] + failures[0],
                                        count)
            print '\r\x1B[KDeleting containers: %d of %d, %d%s left, %d ' \
                  'retries' % (successes[0] + failures[0], count, eta,
                  eta_unit, retries_done[0]),
    for container in containers:
        coropool.spawn(delete, container)
    coropool.waitall()
    successes = successes[0]
    failures = failures[0]
    value = 100.0 * successes / len(containers)
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KDeleting %d containers for performance reporting, ' \
              '%d%s, %d retries' % (count, round(elapsed), elapsed_unit,
              retries_done[0])
        print '%d succeeded, %d failed, %.02f%% success rate' % (
              successes, failures, value)
    return value


def object_put_report(coropool, connpool, count, options):
    successes = [0]
    failures = [0]
    retries_done = [0]
    begun = time()
    next_report = [time() + 2]
    def put(obj):
        with connpool.item() as conn:
            try:
                conn.put_object('stats_object_put', obj, '')
                successes[0] += 1
            except (Exception, Timeout):
                failures[0] += 1
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, successes[0] + failures[0],
                                        count)
            print '\r\x1B[KCreating objects: %d of %d, %d%s left, %d ' \
                  'retries' % (successes[0] + failures[0], count, eta,
                  eta_unit, retries_done[0]),
    with connpool.item() as conn:
        conn.put_container('stats_object_put')
    for x in xrange(count):
        coropool.spawn(put, 'stats_object_put_%02x' % x)
    coropool.waitall()
    successes = successes[0]
    failures = failures[0]
    value = 100.0 * successes / count
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KCreated %d objects for performance reporting, ' \
              '%d%s, %d retries' % (count, round(elapsed), elapsed_unit,
              retries_done[0])
        print '%d succeeded, %d failed, %.02f%% success rate' % (
              successes, failures, value)
    return value


def object_head_report(coropool, connpool, options):
    successes = [0]
    failures = [0]
    retries_done = [0]
    begun = time()
    next_report = [time() + 2]
    with connpool.item() as conn:
        objects = [o['name'] for o in conn.get_container('stats_object_put',
                   prefix='stats_object_put_', full_listing=True)[1]]
    count = len(objects)
    def head(obj):
        with connpool.item() as conn:
            try:
                conn.head_object('stats_object_put', obj)
                successes[0] += 1
            except (Exception, Timeout):
                failures[0] += 1
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, successes[0] + failures[0],
                                        count)
            print '\r\x1B[KHeading objects: %d of %d, %d%s left, %d ' \
                  'retries' % (successes[0] + failures[0], count, eta,
                  eta_unit, retries_done[0]),
    for obj in objects:
        coropool.spawn(head, obj)
    coropool.waitall()
    successes = successes[0]
    failures = failures[0]
    value = 100.0 * successes / len(objects)
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KHeaded %d objects for performance reporting, ' \
              '%d%s, %d retries' % (count, round(elapsed), elapsed_unit,
              retries_done[0])
        print '%d succeeded, %d failed, %.02f%% success rate' % (
              successes, failures, value)
    return value


def object_get_report(coropool, connpool, options):
    successes = [0]
    failures = [0]
    retries_done = [0]
    begun = time()
    next_report = [time() + 2]
    with connpool.item() as conn:
        objects = [o['name'] for o in conn.get_container('stats_object_put',
                   prefix='stats_object_put_', full_listing=True)[1]]
    count = len(objects)
    def get(obj):
        with connpool.item() as conn:
            try:
                conn.get_object('stats_object_put', obj)
                successes[0] += 1
            except (Exception, Timeout):
                failures[0] += 1
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, successes[0] + failures[0],
                                        count)
            print '\r\x1B[KRetrieving objects: %d of %d, %d%s left, %d ' \
                  'retries' % (successes[0] + failures[0], count, eta,
                  eta_unit, retries_done[0]),
    for obj in objects:
        coropool.spawn(get, obj)
    coropool.waitall()
    successes = successes[0]
    failures = failures[0]
    value = 100.0 * successes / len(objects)
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KRetrieved %d objects for performance reporting, ' \
              '%d%s, %d retries' % (count, round(elapsed), elapsed_unit,
              retries_done[0])
        print '%d succeeded, %d failed, %.02f%% success rate' % (
              successes, failures, value)
    return value


def object_delete_report(coropool, connpool, options):
    successes = [0]
    failures = [0]
    retries_done = [0]
    begun = time()
    next_report = [time() + 2]
    with connpool.item() as conn:
        objects = [o['name'] for o in conn.get_container('stats_object_put',
                   prefix='stats_object_put_', full_listing=True)[1]]
    count = len(objects)
    def delete(obj):
        with connpool.item() as conn:
            try:
                conn.delete_object('stats_object_put', obj)
                successes[0] += 1
            except (Exception, Timeout):
                failures[0] += 1
        if options.verbose and time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, successes[0] + failures[0],
                                        count)
            print '\r\x1B[KDeleting objects: %d of %d, %d%s left, %d ' \
                  'retries' % (successes[0] + failures[0], count, eta,
                  eta_unit, retries_done[0]),
    for obj in objects:
        coropool.spawn(delete, obj)
    coropool.waitall()
    successes = successes[0]
    failures = failures[0]
    value = 100.0 * successes / len(objects)
    if options.verbose:
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KDeleted %d objects for performance reporting, ' \
              '%d%s, %d retries' % (count, round(elapsed), elapsed_unit,
              retries_done[0])
        print '%d succeeded, %d failed, %.02f%% success rate' % (
              successes, failures, value)
    return value


if __name__ == '__main__':
    patcher.monkey_patch()
    hubs.get_hub().debug_exceptions = False

    parser = OptionParser(usage='''
Usage: %prog [options] [conf_file]

[conf_file] defaults to /etc/swift/stats.conf'''.strip())
    parser.add_option('-a', '--audit', action='store_true',
                      dest='audit', default=False,
                      help='Run the audit checks')
    parser.add_option('-d', '--dispersion', action='store_true',
                      dest='dispersion', default=False,
                      help='Run the dispersion reports')
    parser.add_option('-o', '--output', dest='csv_output',
                      default=None,
                      help='Override where the CSV report is written '
                           '(default from conf file); the keyword None will '
                           'suppress the CSV report')
    parser.add_option('-p', '--performance', action='store_true',
                      dest='performance', default=False,
                      help='Run the performance reports')
    parser.add_option('-q', '--quiet', action='store_false', dest='verbose',
                      default=True, help='Suppress status output')
    parser.add_option('-r', '--retries', dest='retries',
                      default=None,
                      help='Override retry attempts (default from conf file)')
    args = argv[1:]
    if not args:
        args.append('-h')
    (options, args) = parser.parse_args(args)

    conf_file = '/etc/swift/stats.conf'
    if args:
        conf_file = args.pop(0)
    c = ConfigParser()
    if not c.read(conf_file):
        exit('Unable to read config file: %s' % conf_file)
    conf = dict(c.items('stats'))
    swift_dir = conf.get('swift_dir', '/etc/swift')
    dispersion_coverage = int(conf.get('dispersion_coverage', 1))
    container_put_count = int(conf.get('container_put_count', 1000))
    object_put_count = int(conf.get('object_put_count', 1000))
    concurrency = int(conf.get('concurrency', 50))
    if options.retries:
        options.retries = int(options.retries)
    else:
        options.retries = int(conf.get('retries', 5))
    if not options.csv_output:
        options.csv_output = conf.get('csv_output', '/etc/swift/stats.csv')

    coropool = GreenPool(size=concurrency)

    url, token = get_auth(conf['auth_url'], conf['auth_user'],
                          conf['auth_key'])
    account = url.rsplit('/', 1)[1]
    connpool = Pool(max_size=concurrency)
    connpool.create = lambda: Connection(conf['auth_url'],
                                conf['auth_user'], conf['auth_key'],
                                retries=options.retries, preauthurl=url,
                                preauthtoken=token)

    report = [time(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
              0, 0, 0, 0]
    (R_TIMESTAMP, R_CDR_TIME, R_CDR_VALUE, R_ODR_TIME, R_ODR_VALUE,
     R_CPUT_TIME, R_CPUT_RATE, R_CHEAD_TIME, R_CHEAD_RATE, R_CGET_TIME,
     R_CGET_RATE, R_CDELETE_TIME, R_CDELETE_RATE, R_CLSTANDARD_TIME,
     R_CLPREFIX_TIME, R_CLPREDELIM_TIME, R_OPUT_TIME, R_OPUT_RATE, R_OHEAD_TIME,
     R_OHEAD_RATE, R_OGET_TIME, R_OGET_RATE, R_ODELETE_TIME, R_ODELETE_RATE) = \
     xrange(len(report))

    container_ring = Ring(os.path.join(swift_dir, 'container.ring.gz'))
    object_ring = Ring(os.path.join(swift_dir, 'object.ring.gz'))

    if options.audit:
        audit(coropool, connpool, account, container_ring, object_ring, options)
        if options.verbose and (options.dispersion or options.performance):
            print

    if options.dispersion:
        begin = time()
        report[R_CDR_VALUE] = container_dispersion_report(coropool, connpool,
                                account, container_ring, options)
        report[R_CDR_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        report[R_ODR_VALUE] = object_dispersion_report(coropool, connpool,
                                account, object_ring, options)
        report[R_ODR_TIME] = time() - begin
        if options.verbose and options.performance:
            print

    if options.performance:
        begin = time()
        report[R_CPUT_RATE] = container_put_report(coropool, connpool,
                                                   container_put_count, options)
        report[R_CPUT_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        report[R_CHEAD_RATE] = \
            container_head_report(coropool, connpool, options)
        report[R_CHEAD_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        report[R_CGET_RATE] = container_get_report(coropool, connpool, options)
        report[R_CGET_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        report[R_CDELETE_RATE] = \
            container_delete_report(coropool, connpool, options)
        report[R_CDELETE_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        container_standard_listing_report(coropool, connpool, options)
        report[R_CLSTANDARD_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        container_prefix_listing_report(coropool, connpool, options)
        report[R_CLPREFIX_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        container_prefix_delimiter_listing_report(coropool, connpool, options)
        report[R_CLPREDELIM_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        report[R_OPUT_RATE] = \
            object_put_report(coropool, connpool, object_put_count, options)
        report[R_OPUT_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        report[R_OHEAD_RATE] = object_head_report(coropool, connpool, options)
        report[R_OHEAD_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        report[R_OGET_RATE] = object_get_report(coropool, connpool, options)
        report[R_OGET_TIME] = time() - begin
        if options.verbose:
            print

        begin = time()
        report[R_ODELETE_RATE] = \
            object_delete_report(coropool, connpool, options)
        report[R_ODELETE_TIME] = time() - begin

    if options.csv_output != 'None':
        try:
            if not os.path.exists(options.csv_output):
                f = open(options.csv_output, 'wb')
                f.write('Timestamp,'
                        'Container Dispersion Report Time,'
                        'Container Dispersion Report Value,'
                        'Object Dispersion Report Time,'
                        'Object Dispersion Report Value,'
                        'Container PUT Report Time,'
                        'Container PUT Report Success Rate,'
                        'Container HEAD Report Time,'
                        'Container HEAD Report Success Rate,'
                        'Container GET Report Time,'
                        'Container GET Report Success Rate'
                        'Container DELETE Report Time,'
                        'Container DELETE Report Success Rate,'
                        'Container Standard Listing Time,'
                        'Container Prefix Listing Time,'
                        'Container Prefix Delimiter Listing Time,'
                        'Object PUT Report Time,'
                        'Object PUT Report Success Rate,'
                        'Object HEAD Report Time,'
                        'Object HEAD Report Success Rate,'
                        'Object GET Report Time,'
                        'Object GET Report Success Rate'
                        'Object DELETE Report Time,'
                        'Object DELETE Report Success Rate\r\n')
                csv = csv.writer(f)
            else:
                csv = csv.writer(open(options.csv_output, 'ab'))
            csv.writerow(report)
        except Exception, err:
            print >>stderr, 'Could not write CSV report:', err