cbcollect_info

#!/usr/bin/python
# -*- python -*-
import os
import sys
import tempfile
import time
import subprocess
import string
import re
import platform
import glob

USAGE = """usage: %prog [options] output_file.zip

- Linux/Windows:
    %prog output_file.zip
    %prog -v output_file.zip

- Mac OSX:
    %prog -r ../ output_file.zip"""

class TempFile(object):
    unlink = os.unlink
    def __init__(self):
        fd, self.name = tempfile.mkstemp(text=True)
        self.fp = os.fdopen(fd, 'w+')

    def __getattr__(self, name):
        return getattr(self.__dict__['fp'], name)

    def __delete__(self):
        try:
            self.fp.close()
        except:
            pass

        self.unlink(self.name)


class Task(object):
    privileged = False
    num_samples = 0
    interval = 0
    def __init__(self, description, command, **kwargs):
        self.description = description
        self.command = command
        self.__dict__.update(kwargs)

    def execute(self, fp):
        """Run the task"""
        import subprocess
        if hasattr(self, 'reformat') and self.reformat:
            p = subprocess.Popen(self.command, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, shell=True)
            print >> fp, p.stdout.read()
        else:
            p = subprocess.Popen(self.command, bufsize=-1, stdout=fp, stderr=fp,
                             shell=True)
        return p.wait()

    def will_run(self):
        """Determine if this task will run on this platform."""
        return sys.platform in self.platforms


class TaskRunner(object):
    default_name = "couchbase.log"

    def __init__(self, verbosity=0):
        self.files = {}
        self.tasks = {}
        self.verbosity = verbosity
        self.start_time = time.strftime("%Y%m%d-%H%M%S", time.gmtime())

    def get_file(self, filename):
        if filename in self.files:
            fp = self.files[filename]
        else:
            fp = TempFile()
            self.files[filename] = fp

        return fp

    def header(self, fp, title, subtitle):
        separator = '=' * 78
        print >> fp, separator
        print >> fp, title
        print >> fp, subtitle
        print >> fp, separator
        fp.flush()

    def log_result(self, result):
        if result == 0:
            print >> sys.stderr, "OK"
        else:
            print >> sys.stderr, "Exit code %d" % result

    def run(self, task):
        """Run a task with a file descriptor corresponding to its log file"""
        if task.will_run():
            print >> sys.stderr, "%s (%s) - " % (task.description, task.command),
            if task.privileged and os.getuid() != 0:
                print >> sys.stderr, "skipped (needs root privs)"
                return

            if hasattr(task, 'log_file'):
                filename = task.log_file
            else:
                filename = self.default_name

            fp = self.get_file(filename)
            self.header(fp, task.description, task.command)
            result = task.execute(fp)
            fp.flush()
            self.log_result(result)
            for i in xrange(2, task.num_samples + 2):
                print >> sys.stderr, "Taking sample %d after %f seconds - " % \
                    (i, task.interval)
                time.sleep(task.interval)
                result = task.execute(fp)
                self.log_result(result)
        elif self.verbosity >= 2:
            print >> sys.stderr, 'Skipping "%s" (%s): not for platform %s' \
                % (task.description, task.command, sys.platform)

    def zip(self, filename, node):
        """Write all our logs to a zipfile"""
        from zipfile import ZipFile, ZIP_DEFLATED
        zf = ZipFile(filename, mode='w', compression=ZIP_DEFLATED)
        try:
            for name, fp in self.files.iteritems():
                fp.close()
                zf.write(fp.name,
                         "cbcollect_info_%s_%s/%s" % (node, self.start_time, name))
        finally:
            zf.close()

class SolarisTask(Task):
    platforms = ['sunos5', 'solaris']


class LinuxTask(Task):
    platforms = ['linux2']


class WindowsTask(Task):
    platforms = ['win32', 'cygwin']


class MacOSXTask(Task):
    platforms = ['darwin']


class UnixTask(SolarisTask, LinuxTask, MacOSXTask):
    platforms = SolarisTask.platforms + LinuxTask.platforms + MacOSXTask.platforms


class AllOsTask(UnixTask, WindowsTask):
    platforms = UnixTask.platforms + WindowsTask.platforms

def basedir():
    mydir = os.path.dirname(sys.argv[0])
    if mydir == "":
        mydir = "."
    return mydir

def retrieve_config_value(config_key, node=None):
    if platform.system() == 'Windows':
        return retrieve_config_value_ex("%s\\escript.exe" % basedir(),
                                        "%s\\cbdump-config" % basedir(),
                                        "%s/../var/lib/couchbase/config/config.dat" % basedir(),
                                        config_key, node=node)

    places = [("%s/escript" % basedir(),
               os.path.dirname(os.path.abspath(__file__)) +
               "/cbdump-config",
               os.path.dirname(os.path.abspath(__file__)) +
               "/../var/lib/couchbase/config/config.dat"),
              ("%s/escript" % basedir(),
               "cbdump-config",
               "~/Library/Application Support/Couchbase/var/lib/couchbase/config/config.dat")]

    for escript_cmd, script, config_file in places:
        rv = retrieve_config_value_ex(escript_cmd, script, config_file, config_key, node=node)
        if rv:
            return rv
    return None

def retrieve_config_value_ex(escript_cmd, script, config_file, config_key, node=None):
    cmd_list = [escript_cmd, script, config_file]
    if node:
        cmd_list.append("node")
        cmd_list.append(node)

    try:
        node_cfg = subprocess.Popen(cmd_list,
                                    stdout=subprocess.PIPE).communicate()[0]
        node_cfg = string.strip(node_cfg)
        if len(node_cfg) <= 0:
            return None

        node_cfg = node_cfg.split("\n")
        for line in node_cfg:
            m = re.match(config_key, line)
            if m:
                return m.groups()
    except:
        pass

    return None

def get_node():
    ip = None
    try:
        f = open('/opt/couchbase/var/lib/couchbase/ip', 'r')
        ip = string.strip(f.read())
        f.close()
    except:
        pass
    if (not ip) or (len(ip) <= 0):
        if platform.system() == 'Windows':
            ip = subprocess.Popen("ip_addr.bat",
                                  stdout=subprocess.PIPE).communicate()[0]
        else:
            ip = '127.0.0.1'

    node = 'ns_1@' + ip
    return node

def get_dbdir(parentdir="."):
    for f in ["../etc/runtime.ini", "../etc/couchdb/local.ini"]:
        path = os.path.join(parentdir, f)
        for line in open(path):
            if "database_dir = " in line:
                return string.strip(line.split("= ")[-1])

    if platform.system() == 'Windows':
        return "c:/Program Files/Couchbase/Server/var/lib/couchbase/data"
    else:
        return "/opt/couchbase/var/lib/couchbase/data"

def get_credential():
    ret = retrieve_config_value('\s*{creds,\[\{"(.+)",\s*\[\{password,\s*"(.*)"\}')
    if ret:
        return "%s:%s" % (ret[0], ret[1])
    else:
        return ""

def make_tasks(root="/opt/couchbase", dbdir=""):
  _tasks = [
    UnixTask("uname", "uname -a"),
    WindowsTask("System information", "systeminfo"),
    WindowsTask("Computer system", "wmic computersystem", reformat=True),
    WindowsTask("Computer OS", "wmic os", reformat=True),
    UnixTask("Directory structure",
             "ls -lR '%s' /opt/membase /var/membase /etc/opt/membase" % root),
    UnixTask("Database directory structure",
             "ls -lR " + dbdir),
    WindowsTask("Database directory structure",
                ["dir", "/s", dbdir.replace("/", "\\")]),
    UnixTask("Directory structure membase - previous versions",
             "ls -lR /opt/membase /var/membase /var/opt/membase /etc/opt/membase"),
    SolarisTask("Process list snapshot", "prstat -a -c -n 100 -t -v -L 1 10"),
    SolarisTask("Process list", "ps -ef"),
    SolarisTask("Service configuration", "svcs -a"),
    SolarisTask("Swap configuration", "swap -l"),
    SolarisTask("Disk activity", "zpool iostat 1 10"),
    SolarisTask("Disk activity", "iostat -E 1 10"),
    LinuxTask("Process list snapshot", "top -H -n 1"),
    LinuxTask("Process list ", "ps -AwwL -o user,pid,lwp,ppid,nlwp,pcpu,pri,nice,vsize,rss,tty,stat,wchan:12,start,bsdtime,command"),
    LinuxTask("Swap configuration", "free -t"),
    LinuxTask("Swap configuration", "swapon -s"),
    LinuxTask("Kernel modules", "lsmod"),
    LinuxTask("Distro version", "cat /etc/redhat-release"),
    LinuxTask("Distro version", "lsb_release -a"),
    LinuxTask("Installed software", "rpm -qa"),
    LinuxTask("Installed software", "COLUMNS=300 dpkg -l"),
    LinuxTask("Extended iostat", "iostat -x -p ALL 1 10 || iostat -x 1 10"),
    LinuxTask("Process usage", "export TERM=linux; top -b -n1 | egrep 'moxi|memcached|vbucketmigrator|CPU|load|Mem:|Swap:|Cpu(s)'"),
    LinuxTask("Core dump settings", "find /proc/sys/kernel -type f -name '*core*' -print -exec cat '{}' ';'"),
    LinuxTask("netstat -nap", "netstat -nap"),
    LinuxTask("relevant lsof output", "lsof -n | grep 'moxi\|memcached\|vbucketmigrator\|beam'"),
    MacOSXTask("Process list snapshot", "top -l 1"),
    MacOSXTask("Disk activity", "iostat 1 10"),
    MacOSXTask("Process list ",
               "ps -Aww -o user,pid,lwp,ppid,nlwp,pcpu,pri,nice,vsize,rss,tty,"
               "stat,wchan:12,start,bsdtime,command"),
    WindowsTask("Service list", "wmic service where state=\"running\" GET caption, name, state", reformat=True),
    WindowsTask("Process list", "wmic process", reformat=True),
    WindowsTask("Swap settings", "wmic pagefile", reformat=True),
    WindowsTask("Disk partition", "wmic partition", reformat=True),
    WindowsTask("Disk volumes", "wmic volume", reformat=True),
    UnixTask("Network configuration", "ifconfig -a", interval=10,
              num_samples=1),
    WindowsTask("Network configuration", "ipconfig /all", interval=10,
                num_samples=1),
    UnixTask("Network status", "netstat -an"),
    WindowsTask("Network status", "netstat -an"),
    AllOsTask("Network routing table", "netstat -rn"),
    UnixTask("Arp cache", "arp -na"),
    WindowsTask("Arp cache", "arp -a"),
    WindowsTask("Network Interface Controller", "wmic nic", reformat=True),
    WindowsTask("Network Adapter", "wmic nicconfig", reformat=True),
    WindowsTask("Active network connection", "wmic netuse", reformat=True),
    WindowsTask("Protocols", "wmic netprotocol", reformat=True),
    WindowsTask("Cache memory", "wmic memcache", reformat=True),
    WindowsTask("Physical memory", "wmic memphysical", reformat=True),
    WindowsTask("Physical memory chip info", "wmic memorychip", reformat=True),
    WindowsTask("Local storage devices", "wmic logicaldisk", reformat=True),
    WindowsTask("Version file",
                "type " + basedir() + "\\..\\VERSION.txt", reformat=True),
    WindowsTask("Manifest file",
                "type " + basedir() + "\\..\\manifest.txt", reformat=True),
    WindowsTask("Manifest file",
                "type " + basedir() + "\\..\\manifest.xml", reformat=True),
    WindowsTask("Couchbase config",
                basedir() + "\\escript.exe " +
                basedir() + "\\cbdump-config " +
                basedir() + "\\..\\var\\lib\\couchbase\\config\\config.dat",
                reformat=True),
    WindowsTask("Memcached logs",
                "cd " + basedir() + "\\..\\var\\lib\\couchbase\\logs && " +
                "for /f %a IN ('dir /od /b memcached.log.*') do type %a",
                log_file="memcached.log", reformat=True),
    UnixTask("Filesystem", "df -ha"),
    UnixTask("System activity reporter", "sar 1 10"),
    UnixTask("System paging activity", "vmstat 1 10"),
    UnixTask("System uptime", "uptime"),
    UnixTask("couchbase user definition", "getent passwd couchbase"),
    UnixTask("couchbase user limits", "su couchbase -c \"ulimit -a\"",
         privileged=True),
    UnixTask("membase user definition", "getent passwd membase"),
    UnixTask("couchbase user limits", "su couchbase -c \"ulimit -a\"",
         privileged=True),
    UnixTask("membase user limits", "su membase -c \"ulimit -a\"",
         privileged=True),
    UnixTask("Interrupt status", "intrstat 1 10"),
    UnixTask("Processor status", "mpstat 1 10"),
    UnixTask("System log", "cat /var/adm/messages"),
    LinuxTask("System log", "cat /var/log/syslog"),
    LinuxTask("System log", "cat /var/log/messages"),
    LinuxTask("Version file", "cat '%s/VERSION.txt'" % root),
    LinuxTask("Manifest file", "cat '%s/manifest.txt'" % root),
    LinuxTask("Manifest file", "cat '%s/manifest.xml'" % root),
    LinuxTask("Memcached logs",
              "cd '%s'/var/lib/couchbase/logs && cat $(ls -tr memcached.log.*)" % root,
              log_file="memcached.log"),
    UnixTask("Kernel log buffer", "dmesg"),
    LinuxTask("couchbase config",
              "'%s'/escript '%s'/cbdump-config '%s/var/lib/couchbase/config/config.dat'"
              % (os.path.dirname(sys.argv[0]), os.path.dirname(sys.argv[0]), root)),
    AllOsTask("couchbase logs (debug)", "cbbrowse_logs",
              log_file="ns_server.debug.log"),
    AllOsTask("couchbase logs (info)", "cbbrowse_logs info",
              log_file="ns_server.info.log"),
    AllOsTask("couchbase logs (error)", "cbbrowse_logs error",
              log_file="ns_server.error.log"),
    AllOsTask("couchbase logs (couchdb)", "cbbrowse_logs couchdb",
              log_file="ns_server.couchdb.log"),
    AllOsTask("couchbase logs (xdcr)", "cbbrowse_logs xdcr",
              log_file="ns_server.xdcr.log"),
    AllOsTask("couchbase logs (xdcr_errors)", "cbbrowse_logs xdcr_errors",
              log_file="ns_server.xdcr_errors.log"),
    AllOsTask("couchbase logs (views)", "cbbrowse_logs views",
              log_file="ns_server.views.log"),
    AllOsTask("couchbase logs (mapreduce errors)", "cbbrowse_logs mapreduce_errors",
              log_file="ns_server.mapreduce_errors.log"),
    AllOsTask("couchbase logs (stats)", "cbbrowse_logs stats",
              log_file="ns_server.stats.log"),
    AllOsTask("memcached stats all",
              "cbstats -a 127.0.0.1:11210 all -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats checkpoint",
              "cbstats -a 127.0.0.1:11210 checkpoint -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats config",
              "cbstats -a 127.0.0.1:11210 config -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats dispatcher",
              "cbstats -a 127.0.0.1:11210 dispatcher logs -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats hash",
              "cbstats -a 127.0.0.1:11210 hash detail -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats klog",
              "cbstats -a 127.0.0.1:11210 klog -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats kvstore",
              "cbstats -a 127.0.0.1:11210 kvstore -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats kvtimings",
              "cbstats -a 127.0.0.1:11210 kvtimings -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats tap",
              "cbstats -a 127.0.0.1:11210 tap -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats tapagg",
              "cbstats -a 127.0.0.1:11210 tapagg -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats timings",
              "cbstats -a 127.0.0.1:11210 timings -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached memory stats",
              "cbstats -a 127.0.0.1:11210 raw memory -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached allocator stats",
              "cbstats -a 127.0.0.1:11210 raw allocator -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats prev-vbucket",
              "cbstats -a 127.0.0.1:11210 prev-vbucket -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats vbucket",
              "cbstats -a 127.0.0.1:11210 vbucket -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats vbucket details",
              "cbstats -a 127.0.0.1:11210 vbucket-details -b _admin -p _admin",
              log_file="stats.log"),
    AllOsTask("memcached stats warmup",
              "cbstats -a 127.0.0.1:11210 warmup -b _admin -p _admin",
              log_file="stats.log")
  ]

  c = get_credential()
  if c:
      _tasks.append(
          AllOsTask("couchbase diags",
                    "curl -sS \"http://%s@127.0.0.1:8091/diag?noLogs=1\"" % (c),
                    log_file="diag.log"))

  for f in os.listdir(dbdir):
      path = os.path.join(dbdir, f)
      #find bucket directory
      if os.path.isdir(path) and f not in [".delete", "@indexes"]:
          files = glob.glob("%s/master.couch.[0-9]*" % path)
          if files:
              ddoc = sorted(files, key=lambda x: int(x.split(".")[-1]))[-1]
              _tasks.append(
                  AllOsTask("couchbase design docs - bucket:%s" % f,
                            "couch_dbdump \"%s\"" % (os.path.join(path, ddoc)),
                            log_file="ddocs.log"))

  return _tasks

def main():
    from optparse import OptionParser
    parser = OptionParser(usage=USAGE)
    parser.add_option("-r", dest="root",
                      help="root directory - defaults to /opt/couchbase",
                      default="/opt/couchbase")
    parser.add_option("-v", dest="verbosity", help="increase verbosity level",
                      action="count", default=0)
    options, args = parser.parse_args()

    if len(args) != 1:
        parser.error("incorrect number of arguments")

    mydir = os.path.dirname(sys.argv[0])
    erldir = os.path.join(mydir, 'erlang', 'bin')
    if os.name == 'posix':
        path = [mydir,
                '/bin',
                '/sbin',
                '/usr/bin',
                '/usr/sbin',
                '/opt/couchbase/bin',
                erldir,
                os.environ['PATH']]
        os.environ['PATH'] = ':'.join(path)
    elif os.name == 'nt':
        path = [mydir, erldir, os.environ['PATH']]
        os.environ['PATH'] = ';'.join(path)
    dbdir = get_dbdir(mydir)
    runner = TaskRunner(verbosity=options.verbosity)
    for task in make_tasks(options.root, dbdir):
        runner.run(task)

    runner.zip(args[0], get_node())


if __name__ == '__main__':
    main()