Start tracking k8s liveness/readiness probes

K8s has a similar but not identical method as docker for container
health checks. They are called liveness/readiness probes and are a part
of the pod specification, and not a part of the image.

Luckily, the pod configuration *is* a part of the container metadata as
stringified json, with a label
"annotation.kubectl.kubernetes.io/last-applied-configuration", so we can
use that label to identify liveness/readiness probes.

New methods in the docker container resolver handle parsing the pod
specification (and healthcheck info) out of the container json and
creating health probes from them.

A new class sinsp_container_info::container_health_probe represents one
of these health probes. It has a probe
type (healthcheck/liveness/readiness), the executable and arguments, and
methods to serialize/unserialize from json. The serialization doesn't
preserve the original container json--they only keep the exe + args.

The container info now has a list of possible health probe objects and
iterates over them when dumping the container to json.

For threads, switch everything to use a threadinfo category instead of a
simple bool for has healthcheck. The possible values for the category
are:

  - CAT_NONE: no specific category
  - CAT_CONTAINER: a process run in a container and *not* any
    of the following more specific categories.
  - CAT_HEALTHCHECK: part of a container healthcheck
  - CAT_LIVENESS_PROBE: part of a k8s liveness probe
  - CAT_READINESS_PROBE: part of a k8s readiness probe

Identify_healthcheck becomes identify_category() but
otherwise behaves the same (passing categories down and checking the
args list otherwise).

The logic in indentify_healthcheck tries to handle the common cases
first:

 - not running in a container or container info not present: CAT_NONE
 - vpid=1: CAT_CONTAINER
 - inherit categories other than CAT_NONE directly from parent

If those fail, the more expensive steps of matching against the health
check args and possibly traversing the parent state are done.

The filterchecks aren't quite as generic as the threadinfo categories to
keep the filtering simple. A new field
proc.is_container_{liveness,readiness}_probe checks for k8s
liveness/readiness probes, and container.{liveness,readiness}_probe
prints the exe + args.

userspace/libsinsp/container.cpp

@@ -17,6 +17,8 @@ limitations under the License.
 
 */
 
+#include <algorithm>
+
 #include "container_engine/cri.h"
 #include "container_engine/docker.h"
 #include "container_engine/rkt.h"
@@ -131,8 +133,8 @@ bool sinsp_container_manager::resolve_container(sinsp_threadinfo* tinfo, bool qu
 		}
 	}
 
-	// Also identify if this thread is part of a container healthcheck
-	identify_healthcheck(tinfo);
+	// Also possibly set the category for the threadinfo
+	identify_category(tinfo);
 
 	return matches;
 }
@@ -169,10 +171,7 @@ string sinsp_container_manager::container_to_json(const sinsp_container_info& co
 
 	container["Mounts"] = mounts;
 
-	if(!container_info.m_healthcheck_obj.isNull())
-	{
-		container["Healthcheck"] = container_info.m_healthcheck_obj;
-	}
+	sinsp_container_info::container_health_probe::add_health_probes(container_info.m_health_probes, container);
 
 	char addrbuff[100];
 	uint32_t iph = htonl(container_info.m_container_ip);
@@ -356,15 +355,10 @@ string sinsp_container_manager::get_container_name(sinsp_threadinfo* tinfo)
 	return res;
 }
 
-void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo)
+void sinsp_container_manager::identify_category(sinsp_threadinfo *tinfo)
 {
-	// This thread is a part of a container healthcheck if its
-	// parent thread is part of a health check.
-	sinsp_threadinfo* ptinfo = tinfo->get_parent_thread();
-
-	if(ptinfo && ptinfo->m_is_container_healthcheck)
+	if(tinfo->m_container_id.empty())
 	{
-		tinfo->m_is_container_healthcheck = true;
 		return;
 	}
 
@@ -375,23 +369,65 @@ void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo)
 		return;
 	}
 
-	// Otherwise, the thread is a part of a container healthcheck if:
+	if(tinfo->m_vpid == 1)
+	{
+		if(g_logger.get_severity() >= sinsp_logger::SEV_DEBUG)
+		{
+			g_logger.format(sinsp_logger::SEV_DEBUG,
+					"identify_category (%ld) (%s): initial process for container, assigning CAT_CONTAINER",
+					tinfo->m_tid, tinfo->m_comm.c_str());
+		}
+
+		tinfo->m_category = sinsp_threadinfo::CAT_CONTAINER;
+
+		return;
+	}
+
+	// Categories are passed from parent to child threads
+	sinsp_threadinfo* ptinfo = tinfo->get_parent_thread();
+
+	if(ptinfo && ptinfo->m_category != sinsp_threadinfo::CAT_NONE)
+	{
+		if(g_logger.get_severity() >= sinsp_logger::SEV_DEBUG)
+		{
+			g_logger.format(sinsp_logger::SEV_DEBUG,
+					"identify_category (%ld) (%s): taking parent category %d",
+					tinfo->m_tid, tinfo->m_comm.c_str(), ptinfo->m_category);
+		}
+
+		tinfo->m_category = ptinfo->m_category;
+		return;
+	}
+
+	if(!cinfo->m_metadata_complete)
+	{
+		if(g_logger.get_severity() >= sinsp_logger::SEV_DEBUG)
+		{
+			g_logger.format(sinsp_logger::SEV_DEBUG,
+					"identify_category (%ld) (%s): container metadata incomplete",
+					tinfo->m_tid, tinfo->m_comm.c_str());
+		}
+
+		return;
+	}
+
+	// Otherwise, the thread is a part of a container health probe if:
 	//
-	// 1. the comm and args match the container's healthcheck
+	// 1. the comm and args match one of the container's health probes
 	// 2. we traverse the parent state and do *not* find vpid=1,
 	//    or find a process not in a container
 	//
-	// This indicates the initial process of the healthcheck.
+	// This indicates the initial process of the health probe.
 
-	if(!cinfo->m_has_healthcheck ||
-	   cinfo->m_healthcheck_exe != tinfo->m_exe ||
-	   cinfo->m_healthcheck_args != tinfo->m_args)
-	{
-		return;
-	}
+	sinsp_container_info::container_health_probe::probe_type ptype =
+		cinfo->match_health_probe(tinfo);
 
-	if(tinfo->m_vpid == 1)
+	if(ptype == sinsp_container_info::container_health_probe::PT_NONE)
 	{
+		g_logger.format(sinsp_logger::SEV_DEBUG,
+				"identify_category (%ld) (%s): container health probe PT_NONE",
+				tinfo->m_tid, tinfo->m_comm.c_str());
+
 		return;
 	}
 
@@ -413,7 +449,27 @@ void sinsp_container_manager::identify_healthcheck(sinsp_threadinfo *tinfo)
 
 	if(!found_container_init)
 	{
-		tinfo->m_is_container_healthcheck = true;
+		g_logger.format(sinsp_logger::SEV_DEBUG,
+				"identify_category (%ld) (%s): not under container init, assigning category %s",
+				tinfo->m_tid, tinfo->m_comm.c_str(),
+				sinsp_container_info::container_health_probe::probe_type_names[ptype].c_str());
+
+		// Each health probe type maps to a command category
+		switch(ptype)
+		{
+		case sinsp_container_info::container_health_probe::PT_NONE:
+		case sinsp_container_info::container_health_probe::PT_END:
+			break;
+		case sinsp_container_info::container_health_probe::PT_HEALTHCHECK:
+			tinfo->m_category = sinsp_threadinfo::CAT_HEALTHCHECK;
+			break;
+		case sinsp_container_info::container_health_probe::PT_LIVENESS_PROBE:
+			tinfo->m_category = sinsp_threadinfo::CAT_LIVENESS_PROBE;
+			break;
+		case sinsp_container_info::container_health_probe::PT_READINESS_PROBE:
+			tinfo->m_category = sinsp_threadinfo::CAT_READINESS_PROBE;
+			break;
+		}
 	}
 }
 

userspace/libsinsp/container.h

@@ -48,12 +48,11 @@ class sinsp_container_manager
 	void dump_containers(scap_dumper_t* dumper);
 	string get_container_name(sinsp_threadinfo* tinfo);
 
-	// Set tinfo's is_container_healthcheck attribute to true if
-	// it is identified as a container healthcheck. It will *not*
-	// set it to false by default, so a threadinfo that is
-	// initially identified as a health check will remain one
+	// Set tinfo's m_category based on the container context.  It
+	// will *not* change any category to NONE, so a threadinfo
+	// that initially has a category will retain its category
 	// across execs e.g. "sh -c /bin/true" execing /bin/true.
-	void identify_healthcheck(sinsp_threadinfo *tinfo);
+	void identify_category(sinsp_threadinfo *tinfo);
 
 	bool container_exists(const string& container_id) const {
 		return m_containers.find(container_id) != m_containers.end();

userspace/libsinsp/container_engine/docker.h

@@ -79,6 +79,30 @@ class docker_async_source : public sysdig::async_key_value_source<std::string, c
 
 	bool parse_docker(std::string &container_id, sinsp_container_info *container);
 
+	// Look for a pod specification in this container's labels and
+	// if found set spec to the pod spec.
+	bool get_k8s_pod_spec(const Json::Value &config_obj,
+			      Json::Value &spec);
+
+	std::string normalize_arg(const std::string &arg);
+
+	// Parse a healthcheck out of the provided healthcheck object,
+	// updating the container info with any healthcheck found.
+	void parse_healthcheck(const Json::Value &healthcheck_obj,
+			       sinsp_container_info *container);
+
+	// Parse either a readiness or liveness probe out of the
+	// provided object, updating the container info with any probe
+	// found.
+	bool parse_liveness_readiness_probe(const Json::Value &probe_obj,
+					    sinsp_container_info::container_health_probe::probe_type ptype,
+					    sinsp_container_info *container);
+
+	// Parse all healthchecks/liveness probes/readiness probes out
+	// of the provided object, updating the container info as required.
+	void parse_health_probes(const Json::Value &config_obj,
+				 sinsp_container_info *container);
+
 	sinsp *m_inspector;
 
 	std::string m_docker_unix_socket_path;