k8s delegation and json filtering (jq)

Author: aleks-f

.travis.yml

@@ -51,6 +51,15 @@ script:
     - make
     - sudo cp -r include/* /usr/local/include/
     - sudo cp src/libb64.a /usr/local/lib/
+    - cd ..
+    - wget http://download.draios.com/dependencies/jq-1.5.tar.gz
+    - tar -xzf jq-1.5.tar.gz
+    - cd jq-1.5
+    - ./configure --disable-maintainer-mode
+    - make LDFLAGS=-all-static
+    - sudo cp -r ./*.h /usr/local/include/
+    - sudo cp .libs/libjq.a /usr/local/lib/
+    - cd ..
     - popd
     - rm -rf userspace/libsinsp/third-party/jsoncpp
     - sudo apt-get install libncurses5-dev libluajit-5.1-dev libcurl4-openssl-dev libssl-dev
@@ -67,4 +76,4 @@ notifications:
       - https://webhooks.gitter.im/e/fdbc2356fb0ea2f15033
     on_success: change
     on_failure: always
-    on_start: never
+    on_start: never

CMakeLists.txt

@@ -201,6 +201,34 @@ else()
 	endif()
 endif()
 
+#
+# jq
+#
+if(NOT WIN32 AND NOT APPLE)
+	option(USE_BUNDLED_JQ "Enable building of the bundled jq" ${USE_BUNDLED_DEPS})
+	if(NOT USE_BUNDLED_JQ)
+		find_path(JQ_INCLUDE jq.h PATH_SUFFIXES jq)
+		find_library(JQ_LIB NAMES jq)
+		if(JQ_INCLUDE AND JQ_LIB)
+			message(STATUS "Found jq: include: ${JQ_INCLUDE}, lib: ${JQ_LIB}")
+		else()
+			message(FATAL_ERROR "Couldn't find system jq")
+		endif()
+	else()
+		set(JQ_SRC "${PROJECT_BINARY_DIR}/jq-prefix/src/jq")
+		message(STATUS "Using bundled jq in '${JQ_SRC}'")
+		set(JQ_INCLUDE "${JQ_SRC}")
+		set(JQ_LIB "${JQ_SRC}/.libs/libjq.a")
+		ExternalProject_Add(jq
+		URL "http://download.draios.com/dependencies/jq-1.5.tar.gz"
+		URL_MD5 "0933532b086bd8b6a41c1b162b1731f9"
+		CONFIGURE_COMMAND ./configure --disable-maintainer-mode --enable-all-static --disable-dependency-tracking
+		BUILD_COMMAND ${CMD_MAKE} LDFLAGS=-all-static
+		BUILD_IN_SOURCE 1
+		INSTALL_COMMAND "")
+	endif()
+endif()
+
 #
 # ncurses, keep it simple for the moment
 #
@@ -267,6 +295,7 @@ if(NOT WIN32 AND NOT APPLE)
 	else()
 		set(OPENSSL_BUNDLE_DIR "${PROJECT_BINARY_DIR}/openssl-prefix/src/openssl")
 		set(OPENSSL_INSTALL_DIR "${OPENSSL_BUNDLE_DIR}/target")
+		set(OPENSSL_INCLUDE_DIR "${PROJECT_BINARY_DIR}/openssl-prefix/src/openssl/include")
 		set(OPENSSL_LIBRARY_SSL "${OPENSSL_INSTALL_DIR}/lib/libssl.a")
 		set(OPENSSL_LIBRARY_CRYPTO "${OPENSSL_INSTALL_DIR}/lib/libcrypto.a")
 

userspace/libsinsp/CMakeLists.txt

@@ -8,6 +8,8 @@ if(NOT WIN32 AND NOT APPLE)
 	include_directories("${B64_INCLUDE}")
 	include_directories("${CURL_INCLUDE_DIR}")
 	include_directories("${CURSES_INCLUDE_DIR}")
+	include_directories("${JQ_INCLUDE}")
+	include_directories("${OPENSSL_INCLUDE_DIR}")
 endif()
 
 add_library(sinsp STATIC
@@ -28,6 +30,7 @@ add_library(sinsp STATIC
 	filter.cpp
 	filterchecks.cpp
 	ifinfo.cpp
+	json_query.cpp
 	k8s.cpp
 	k8s_collector.cpp
 	k8s_component.cpp
@@ -75,6 +78,7 @@ if(NOT WIN32)
 
 	if(NOT APPLE)
 		target_link_libraries(sinsp
+			"${JQ_LIB}"
 			"${B64_LIB}"
 			"${CURL_LIBRARIES}"
 			"${OPENSSL_LIBRARY_SSL}"

userspace/libsinsp/docker.cpp

@@ -11,15 +11,14 @@
 
 const std::string docker::DOCKER_SOCKET_FILE = "/var/run/docker.sock";
 
-docker::docker(const std::string& url,
+docker::docker(std::string url,
 	const std::string& path,
 	const std::string& http_version,
 	int timeout_ms,
 	bool is_captured,
 	bool verbose,
 	event_filter_ptr_t event_filter): m_id("docker"),
 #ifdef HAS_CAPTURE
-		m_url(!url.empty() ? url : std::string(scap_get_host_root()) + DOCKER_SOCKET_FILE),
 		m_collector(false),
 #endif // HAS_CAPTURE
 		m_timeout_ms(timeout_ms),
@@ -33,6 +32,46 @@ docker::docker(const std::string& url,
 		m_image_events{"delete", "import", "pull", "push", "tag", "untag"},
 		m_volume_events{"create", "mount", "unmount", "destroy"},
 		m_network_events{"create", "connect", "disconnect", "destroy"},
+		m_severity_map
+		{
+			// container
+			{ "attach",      sinsp_logger::SEV_EVT_INFORMATION },
+			{ "commit",      sinsp_logger::SEV_EVT_INFORMATION },
+			{ "copy",        sinsp_logger::SEV_EVT_INFORMATION },
+			{ "create",      sinsp_logger::SEV_EVT_INFORMATION },
+			{ "destroy",     sinsp_logger::SEV_EVT_WARNING     },
+			{ "die",         sinsp_logger::SEV_EVT_WARNING     },
+			{ "exec_create", sinsp_logger::SEV_EVT_INFORMATION },
+			{ "exec_start",  sinsp_logger::SEV_EVT_INFORMATION },
+			{ "export",      sinsp_logger::SEV_EVT_INFORMATION },
+			{ "kill",        sinsp_logger::SEV_EVT_WARNING     },
+			{ "oom",         sinsp_logger::SEV_EVT_WARNING     },
+			{ "pause",       sinsp_logger::SEV_EVT_INFORMATION },
+			{ "rename",      sinsp_logger::SEV_EVT_INFORMATION },
+			{ "resize",      sinsp_logger::SEV_EVT_INFORMATION },
+			{ "restart",     sinsp_logger::SEV_EVT_WARNING     },
+			{ "start",       sinsp_logger::SEV_EVT_INFORMATION },
+			{ "stop",        sinsp_logger::SEV_EVT_INFORMATION },
+			{ "top",         sinsp_logger::SEV_EVT_INFORMATION },
+			{ "unpause",     sinsp_logger::SEV_EVT_INFORMATION },
+			{ "update",      sinsp_logger::SEV_EVT_INFORMATION },
+
+			// image
+			{ "delete", sinsp_logger::SEV_EVT_INFORMATION },
+			{ "import", sinsp_logger::SEV_EVT_INFORMATION },
+			{ "pull",   sinsp_logger::SEV_EVT_INFORMATION },
+			{ "push",   sinsp_logger::SEV_EVT_INFORMATION },
+			{ "tag",    sinsp_logger::SEV_EVT_INFORMATION },
+			{ "untag",  sinsp_logger::SEV_EVT_INFORMATION },
+
+			// volume
+			{ "mount",   sinsp_logger::SEV_EVT_INFORMATION },
+			{ "unmount", sinsp_logger::SEV_EVT_INFORMATION },
+
+			// network
+			{ "connect",    sinsp_logger::SEV_EVT_INFORMATION },
+			{ "disconnect", sinsp_logger::SEV_EVT_INFORMATION }
+		},
 		m_name_translation
 		{
 			// Container
@@ -78,54 +117,19 @@ docker::docker(const std::string& url,
 			// { "destroy"     "Destroyed"    } duplicate
 		}
 {
-#ifdef HAS_CAPTURE
 	g_logger.log(std::string("Creating Docker object for " +
-							(m_url.empty() ? std::string("capture replay") : m_url)),
+							(url.empty() ? std::string("capture replay") : url)),
 				 sinsp_logger::SEV_DEBUG);
-
-	m_event_http = std::make_shared<handler_t>(*this, "events", m_url, path, http_version, timeout_ms);
+#ifdef HAS_CAPTURE
+	if(url.empty())
+	{
+		url = std::string("file://").append(scap_get_host_root()).append(DOCKER_SOCKET_FILE);
+	}
+	m_event_http = std::make_shared<handler_t>(*this, "docker", url, path, http_version, timeout_ms);
 	m_event_http->set_json_callback(&docker::set_event_json);
 	m_event_http->set_json_end("}\n");
 	m_collector.add(m_event_http);
 	send_data_request();
-
-	// container
-	m_severity_map["attach"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["commit"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["copy"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["create"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["destroy"] = sinsp_logger::SEV_EVT_WARNING;
-	m_severity_map["die"] = sinsp_logger::SEV_EVT_WARNING;
-	m_severity_map["exec_create"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["exec_start"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["export"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["kill"] = sinsp_logger::SEV_EVT_WARNING;
-	m_severity_map["oom"] = sinsp_logger::SEV_EVT_WARNING;
-	m_severity_map["pause"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["rename"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["resize"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["restart"] = sinsp_logger::SEV_EVT_WARNING;
-	m_severity_map["start"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["stop"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["top"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["unpause"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["update"] = sinsp_logger::SEV_EVT_INFORMATION;
-
-	// image
-	m_severity_map["delete"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["import"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["pull"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["push"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["tag"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["untag"] = sinsp_logger::SEV_EVT_INFORMATION;
-
-	// volume
-	m_severity_map["mount"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["unmount"] = sinsp_logger::SEV_EVT_INFORMATION;
-
-	// network
-	m_severity_map["connect"] = sinsp_logger::SEV_EVT_INFORMATION;
-	m_severity_map["disconnect"] = sinsp_logger::SEV_EVT_INFORMATION;
 #endif
 }
 
@@ -389,4 +393,23 @@ void docker::handle_event(Json::Value&& root)
 	}
 }
 
+std::string docker::get_socket_file()
+{
+	string sock_file = scap_get_host_root();
+	std::string::size_type len = sock_file.length();
+	if(len && sock_file[len - 1] == '/')
+	{
+		if((len - 1) > 0)
+		{
+			sock_file = sock_file.substr(0, len - 1);
+		}
+		else
+		{
+			sock_file.clear();
+		}
+	}
+	sock_file.append(DOCKER_SOCKET_FILE);
+	return sock_file;
+}
+
 #endif // __linux__

userspace/libsinsp/docker.h

@@ -17,16 +17,14 @@
 class docker
 {
 public:
-	static const std::string DOCKER_SOCKET_FILE;
-
 	typedef std::vector<std::string> uri_list_t;
 	typedef std::shared_ptr<Json::Value> json_ptr_t;
 	typedef std::set<std::string, ci_compare> event_filter_t;
 	typedef user_event_filter_t::ptr_t event_filter_ptr_t;
 
 	static const int default_timeout_ms = 1000L;
 
-	docker(const std::string& url = "",
+	docker(std::string url = "",
 		const std::string& path = "/events",
 		const std::string& http_version = "1.0",
 		int timeout_ms = default_timeout_ms,
@@ -48,7 +46,10 @@ class docker
 	void set_machine_id(const std::string& machine_id);
 	const std::string& get_machine_id() const;
 
+	static std::string get_socket_file();
+
 private:
+	static const std::string DOCKER_SOCKET_FILE;
 	void connect();
 	void send_event_data_request();
 	void check_collector_status(int expected);
@@ -78,58 +79,41 @@ class docker
 		return false;
 	}
 
-	bool is_container_event(const std::string& evt_name)
-	{
-		return m_container_events.find(evt_name) != m_container_events.end();
-	}
-	bool is_image_event(const std::string& evt_name)
-	{
-		return m_image_events.find(evt_name) != m_image_events.end();
-	}
-	bool is_volume_event(const std::string& evt_name)
-	{
-		return m_volume_events.find(evt_name) != m_volume_events.end();
-	}
-	bool is_network_event(const std::string& evt_name)
-	{
-		return m_network_events.find(evt_name) != m_network_events.end();
-	}
+	bool is_container_event(const std::string& evt_name);
+	bool is_image_event(const std::string& evt_name);
+	bool is_volume_event(const std::string& evt_name);
+	bool is_network_event(const std::string& evt_name);
 
 	typedef socket_data_handler<docker> handler_t;
 	typedef handler_t::ptr_t            handler_ptr_t;
 	typedef socket_collector<handler_t> collector_t;
 
 	std::string   m_id;
-	std::string   m_url;
 	handler_ptr_t m_event_http;
 	collector_t   m_collector;
 	std::string   m_event_uri;
 #endif // HAS_CAPTURE
 
 private:
 
-	const std::string& translate_name(const std::string& event_name);
-
-	long               m_timeout_ms;
-	bool               m_is_captured;
-	bool               m_verbose;
-	event_filter_ptr_t m_event_filter;
-	std::string        m_machine_id;
-
 	typedef std::vector<json_ptr_t> event_list_t;
 	typedef sinsp_logger::event_severity severity_t;
 	typedef std::unordered_map<std::string, severity_t> severity_map_t;
-
-	event_list_t   m_events;
-	severity_map_t m_severity_map;
-
+	typedef std::unordered_map<std::string, std::string> name_translation_map_t;
 	typedef std::set<std::string> entity_events_t;
-	const entity_events_t m_container_events;
-	const entity_events_t m_image_events;
-	const entity_events_t m_volume_events;
-	const entity_events_t m_network_events;
+	const std::string& translate_name(const std::string& event_name);
 
-	typedef std::unordered_map<std::string, std::string> name_translation_map_t;
+	long                   m_timeout_ms;
+	bool                   m_is_captured;
+	bool                   m_verbose;
+	event_filter_ptr_t     m_event_filter;
+	std::string            m_machine_id;
+	event_list_t           m_events;
+	const entity_events_t  m_container_events;
+	const entity_events_t  m_image_events;
+	const entity_events_t  m_volume_events;
+	const entity_events_t  m_network_events;
+	severity_map_t         m_severity_map;
 	name_translation_map_t m_name_translation;
 };
 
@@ -163,4 +147,24 @@ inline const std::string& docker::translate_name(const std::string& event_name)
 	return event_name;
 }
 
+inline bool docker::is_container_event(const std::string& evt_name)
+{
+	return m_container_events.find(evt_name) != m_container_events.end();
+}
+
+inline bool docker::is_image_event(const std::string& evt_name)
+{
+	return m_image_events.find(evt_name) != m_image_events.end();
+}
+
+inline bool docker::is_volume_event(const std::string& evt_name)
+{
+	return m_volume_events.find(evt_name) != m_volume_events.end();
+}
+
+inline bool docker::is_network_event(const std::string& evt_name)
+{
+	return m_network_events.find(evt_name) != m_network_events.end();
+}
+
 #endif // __linux__

userspace/libsinsp/ifinfo.h

@@ -63,9 +63,8 @@ class SINSP_PUBLIC sinsp_ipv6_ifinfo
 class SINSP_PUBLIC sinsp_network_interfaces
 {
 public:
-	sinsp_network_interfaces(sinsp* inspector)
+	sinsp_network_interfaces(sinsp* inspector): m_inspector(inspector)
 	{
-		m_inspector = inspector;
 	}
 
 	void import_interfaces(scap_addrlist* paddrlist);