From 047a207b8f3d90ac8e4223cec0b6de44c1a36611 Mon Sep 17 00:00:00 2001 From: Jake Hunsaker Date: Tue, 28 Jun 2016 16:02:20 -0400 Subject: [PATCH] [kubernetes] new data, namespace support, and options This patch collects additional data from kubernetes masters, changes the available plugin options, and makes collection namespace aware. By default, json output will be collected for events, limitranges, pods, pvcs, replicationcontrollers, resourcequotas and services on a per-namespace basis. Note that the 'serviceaccount' and 'secrets' resources are not collected due to the high risk of exposing secure information. Version and 'config view' output is now collected. The 'all' option (default enabled) will collect a non-json listing of each resource across all namespaces for ease of reference. The 'describe' option (default enabled) will collect 'kubectl describe' output, non-json, for each object of each resource in each namespace, if present. The 'podslog' option has been renamed to 'podlogs' and is disabled by default. Note that while this will run on OpenShift v3 masters, only resources shared by OpenShift and 'plain' Kubernetes are collected by this plugin. OpenShift routes for example are not collected. Signed-off-by: Jake Hunsaker --- sos/plugins/kubernetes.py | 140 ++++++++++++++++++++++++++++++-------- 1 file changed, 110 insertions(+), 30 deletions(-) diff --git a/sos/plugins/kubernetes.py b/sos/plugins/kubernetes.py index 32857a81e3..e0b3a96c0e 100644 --- a/sos/plugins/kubernetes.py +++ b/sos/plugins/kubernetes.py @@ -15,6 +15,7 @@ # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. from sos.plugins import Plugin, RedHatPlugin +from os import path class kubernetes(Plugin, RedHatPlugin): @@ -22,41 +23,120 @@ class kubernetes(Plugin, RedHatPlugin): """Kubernetes plugin """ - option_list = [("podslog", "capture logs for pods", 'slow', False)] + # Red Hat Atomic Platform and OpenShift Enterprise use the + # atomic-openshift-master package to provide kubernetes + packages = ('kubernetes', 'kubernetes-master', 'atomic-openshift-master') + files = ("/etc/origin/master/master-config.yaml",) + + option_list = [ + ("all", "also collect --all-namespaces output separately", + 'fast', True), + ("describe", "capture descriptions of all kube resources", + 'fast', True), + ("podlogs", "capture logs for pods", 'slow', False), + ] + + def check_is_master(self): + if any([ + path.exists("/var/run/kubernetes/apiserver.key"), + path.exists("/etc/origin/master/master-config.yaml") + ]): + return True + return False def setup(self): self.add_copy_spec("/etc/kubernetes") self.add_copy_spec("/var/run/flannel") - # Kubernetes master info - self.add_cmd_output("kubectl version") - self.add_cmd_output("kubectl get -o json pods") - self.add_cmd_output("kubectl get -o json nodes") - self.add_cmd_output("kubectl get -o json services") - self.add_cmd_output("kubectl get -o json replicationController") - self.add_cmd_output("kubectl get -o json events") - - # This could use a single call: - # - # add_journal(units=["kubelet", "kube-apiserver", ... ]) - # - # But this would merge all units into a single text stream - to - # preserve existing file layout in archives keep these as - # separate journalctl calls. - self.add_journal(units="kubelet") - self.add_journal(units="kube-apiserver") - self.add_journal(units="kube-controller-manager") - self.add_journal(units="kube-scheduler") - self.add_journal(units="kube-proxy") - - if self.get_option('podslog'): - result = self.get_command_output("kubectl get pods") - if result['status'] == 0: - for line in result['output'].splitlines()[1:]: - pod_name = line.split(" ")[0] - self.add_cmd_output([ - "{0} log {1}".format("kubectl", pod_name) - ]) + svcs = [ + 'kubelet', + 'kube-apiserver', + 'kube-proxy', + 'kube-scheduler', + 'kube-controller-manager' + ] + + for svc in svcs: + self.add_journal(units=svc) + + # We can only grab kubectl output from the master + if self.check_is_master(): + kube_cmd = "kubectl " + if path.exists('/etc/origin/master/admin.kubeconfig'): + kube_cmd += "--config=/etc/origin/master/admin.kubeconfig" + + kube_get_cmd = "get -o json " + for subcmd in ['version', 'config view']: + self.add_cmd_output('%s %s' % (kube_cmd, subcmd)) + + # get all namespaces in use + kn = self.get_command_output('%s get namespaces' % kube_cmd) + knsps = [n.split()[0] for n in kn['output'].splitlines()[1:] if n] + + resources = [ + 'limitrange', + 'pods', + 'pvc', + 'rc', + 'resourcequota', + 'services' + ] + + # nodes and pvs are not namespaced, must pull separately + self.add_cmd_output([ + "{} get -o json nodes".format(kube_cmd), + "{} get -o json pv".format(kube_cmd) + ]) + + for n in knsps: + knsp = '--namespace=%s' % n + k_cmd = '%s %s %s' % (kube_cmd, kube_get_cmd, knsp) + + self.add_cmd_output('%s events' % k_cmd) + + for res in resources: + self.add_cmd_output('%s %s' % (k_cmd, res)) + + if self.get_option('describe'): + # need to drop json formatting for this + k_cmd = '%s get %s' % (kube_cmd, knsp) + for res in resources: + r = self.get_command_output( + '%s %s' % (k_cmd, res)) + if r['status'] == 0: + k_list = [k.split()[0] for k in + r['output'].splitlines()[1:]] + for k in k_list: + k_cmd = '%s %s' % (kube_cmd, knsp) + self.add_cmd_output( + '%s describe %s %s' % (k_cmd, res, k)) + + if self.get_option('podlogs'): + k_cmd = '%s get %s' % (kube_cmd, knsp) + r = self.get_command_output('$s get pods' % k_cmd) + if r['status'] == 0: + pods = [p.split()[0] for p in + r['output'].splitlines()[1:]] + for pod in pods: + self.add_cmd_output('%s logs %s' % (k_cmd, pod)) + + if self.get_option('all'): + k_cmd = '%s get --all-namespaces=true' % kube_cmd + for res in resources: + self.add_cmd_output('%s %s' % (k_cmd, res)) + + def postproc(self): + # First, clear sensitive data from the json output collected. + # This will mask values when the "name" looks susceptible of + # values worth obfuscating, i.e. if the name contains strings + # like "pass", "pwd", "key" or "token" + env_regexp = r'(?P{\s*"name":\s*[^,]*' \ + r'(pass|pwd|key|token|cred|PASS|PWD|KEY)[^,]*,\s*"value":)[^}]*' + self.do_cmd_output_sub('kubectl', env_regexp, + r'\g "********"') + # Next, we need to handle the private keys and certs in some + # output that is not hit by the previous iteration. + self.do_cmd_private_sub('kubectl') # vim: et ts=5 sw=4