Bline (draios#759)

* minor friendliness changes

* minor refactories

* few more event listeners

* listener callback for clone()

* merge dev

* some inlining

* small interface change

* the set_output_format chisel API call now supports base64 and jsonbase64

* calculate a simple hash for each process that falco can use

* minor typo

* save container IP with the right endianess

* improve local address detection by matching against the full list of container addresses

* falco process hash includes the arguments if the process is a scripting language

* extract the image ID from the docker API

* save/load the container ID from trace files

* fix a merge issue

* Add container image id filtercheck. (draios#661)

Add support for displaying container image ids via the filtercheck
container.image.id. Only supported for docker containers right now.

* minor changes required by the agent

* compile error

* some logging for debugging purposes

* a bit more debug info

* a bit more debug info

* a bit more debug info

* a bit more debug info

* a bit more debug info

* a bit more debug info

* more debug info

* more debug info

* debug info fix

* decrease container verbosity

* more debug info

* dump to memory functionality implemented

* proper support for tracers in memory dumps

* bugfix: potential buffer underrun

* fixed a bug when converting sinsp IPv6 FDs to scap

* compression experiments

* cleanups

* cleanups

* fix a comment

* a couple of helper functions for memory dumps

* don't restart event numbering when reading merged captures + FD initialization bugfix

* remove some logging

* small changes to support memory dumping

* dump a circular capture file when a command is run in the cassandra container

* cleanups

* a bit on infrastructure for a notification event

* notification event type

* apply the filter in the successive segments of a merged capture only if there actually is a filter

* temporarily enable dump of any execve

* some debug info

* less aggressive logging

* Restore scap_savefile

* heuristic to determine if a thread is part of a shell pipe

* fixes to the pipe detection heuristic

* propagate bash pipe flags in the execve parser

* a coule of helper functions

* make sure the analyzer thread info is accessed only if available

* EOLs

* cleanups

* cleanups

* removed an unused variable

CMakeLists.txt

@@ -356,6 +356,7 @@ endif() # NOT WIN32 AND NOT APPLE
 add_subdirectory(userspace/sysdig)
 add_subdirectory(userspace/libscap)
 add_subdirectory(userspace/libsinsp)
+#add_subdirectory(userspace/falcobl)
 
 set(CPACK_PACKAGE_NAME "${PACKAGE_NAME}")
 set(CPACK_PACKAGE_VENDOR "Sysdig Inc.")

driver/event_table.c

@@ -301,5 +301,7 @@ const struct ppm_event_info g_event_info[PPM_EVENT_MAX] = {
 	/* PPME_SYSCALL_MKDIR_2_E */{"mkdir", EC_FILE, EF_NONE, 1, {{"mode", PT_UINT32, PF_HEX} } },
 	/* PPME_SYSCALL_MKDIR_2_X */{"mkdir", EC_FILE, EF_NONE, 2, {{"res", PT_ERRNO, PF_DEC}, {"path", PT_FSPATH, PF_NA} } },
 	/* PPME_SYSCALL_RMDIR_2_E */{"rmdir", EC_FILE, EF_NONE, 0},
-	/* PPME_SYSCALL_RMDIR_2_X */{"rmdir", EC_FILE, EF_NONE, 2, {{"res", PT_ERRNO, PF_DEC}, {"path", PT_FSPATH, PF_NA} } }
+	/* PPME_SYSCALL_RMDIR_2_X */{"rmdir", EC_FILE, EF_NONE, 2, {{"res", PT_ERRNO, PF_DEC}, {"path", PT_FSPATH, PF_NA} } },
+	/* PPME_NOTIFICATION_E */{"notification", EC_OTHER, EF_SKIPPARSERESET, 2, {{"id", PT_CHARBUF, PF_DEC}, {"desc", PT_CHARBUF, PF_NA}, } },
+	/* PPME_NOTIFICATION_X */{"NA4", EC_SYSTEM, EF_UNUSED, 0}
 };

driver/ppm_events_public.h

@@ -134,6 +134,10 @@ along with sysdig.  If not, see <http://www.gnu.org/licenses/>.
 #define PPM_CL_ACTIVE (1 << 19)			/* libsinsp-specific flag. Set in the first non-clone event for
 										   this thread. */
 #define PPM_CL_CLONE_NEWUSER (1 << 20)
+#define PPM_CL_PIPE_SRC (1 << 21)			/* libsinsp-specific flag. Set if this thread has been 
+										       detected to be the source in a shell pipe. */
+#define PPM_CL_PIPE_DST (1 << 22)			/* libsinsp-specific flag. Set if this thread has been 
+										       detected to be the destination in a shell pipe. */
 
 /*
  * Futex Operations
@@ -770,7 +774,9 @@ enum ppm_event_type {
 	PPME_SYSCALL_MKDIR_2_X = 277,
 	PPME_SYSCALL_RMDIR_2_E = 278,
 	PPME_SYSCALL_RMDIR_2_X = 279,
-	PPM_EVENT_MAX = 280
+	PPME_NOTIFICATION_E = 280,
+	PPME_NOTIFICATION_X = 281,
+	PPM_EVENT_MAX = 282
 };
 /*@}*/
 

userspace/libscap/event_table.c

@@ -301,5 +301,7 @@ const struct ppm_event_info g_event_info[PPM_EVENT_MAX] = {
 	/* PPME_SYSCALL_MKDIR_2_E */{"mkdir", EC_FILE, EF_NONE, 1, {{"mode", PT_UINT32, PF_HEX} } },
 	/* PPME_SYSCALL_MKDIR_2_X */{"mkdir", EC_FILE, EF_NONE, 2, {{"res", PT_ERRNO, PF_DEC}, {"path", PT_FSPATH, PF_NA} } },
 	/* PPME_SYSCALL_RMDIR_2_E */{"rmdir", EC_FILE, EF_NONE, 0},
-	/* PPME_SYSCALL_RMDIR_2_X */{"rmdir", EC_FILE, EF_NONE, 2, {{"res", PT_ERRNO, PF_DEC}, {"path", PT_FSPATH, PF_NA} } }
+	/* PPME_SYSCALL_RMDIR_2_X */{"rmdir", EC_FILE, EF_NONE, 2, {{"res", PT_ERRNO, PF_DEC}, {"path", PT_FSPATH, PF_NA} } },
+	/* PPME_NOTIFICATION_E */{"notification", EC_OTHER, EF_SKIPPARSERESET, 2, {{"id", PT_CHARBUF, PF_DEC}, {"desc", PT_CHARBUF, PF_NA}, } },
+	/* PPME_NOTIFICATION_X */{"NA4", EC_SYSTEM, EF_UNUSED, 0}
 };

userspace/libscap/scap-int.h

@@ -105,14 +105,16 @@ struct scap
 typedef enum ppm_dumper_type
 {
 	DT_FILE = 0,
-	DT_MEM = 0,
+	DT_MEM = 1,
 }ppm_dumper_type;
 
 struct scap_dumper
 {
 	gzFile m_f;
 	ppm_dumper_type m_type;
-	uint64_t m_off;
+	uint8_t* m_targetbuf;
+	uint8_t* m_targetbufcurpos;
+	uint8_t* m_targetbufend;
 };
 
 struct scap_ns_socket_list

userspace/libscap/scap.h

@@ -886,6 +886,9 @@ int32_t scap_enable_tracers_capture(scap_t* handle);
 uint64_t scap_get_unexpected_block_readsize(scap_t* handle);
 int32_t scap_proc_add(scap_t* handle, uint64_t tid, scap_threadinfo* tinfo);
 int32_t scap_fd_add(scap_threadinfo* tinfo, uint64_t fd, scap_fdinfo* fdinfo);
+scap_dumper_t *scap_memory_dump_open(scap_t *handle, uint8_t* targetbuf, uint64_t targetbufsize);
+int32_t compr(uint8_t* dest, uint64_t* destlen, const uint8_t* source, uint64_t sourcelen, int level);
+uint8_t* scap_get_memorydumper_curpos(scap_dumper_t *d);
 
 #ifdef __cplusplus
 }

userspace/libscap/scap_savefile.c

@@ -42,7 +42,46 @@ int scap_dump_write(scap_dumper_t *d, void* buf, unsigned len)
 	{
 		return gzwrite(d->m_f, buf, len);
 	}
-	return 0;
+	else
+	{
+		if(d->m_targetbufcurpos + len < d->m_targetbufend)
+		{
+			memcpy(d->m_targetbufcurpos, buf, len);
+
+			d->m_targetbufcurpos += len;
+			return len;
+		}
+		else
+		{
+			return -1;
+		}
+	}
+}
+
+int32_t compr(uint8_t* dest, uint64_t* destlen, const uint8_t* source, uint64_t sourcelen, int level)
+{
+	uLongf dl = compressBound(sourcelen);
+
+	if(dl >= *destlen)
+	{
+		return SCAP_FAILURE;
+	}
+
+	int res = compress2(dest, &dl, source, sourcelen, level);
+	if(res == Z_OK)
+	{
+		*destlen = (uint64_t)dl;
+		return SCAP_SUCCESS;
+	}
+	else
+	{
+		return SCAP_FAILURE;
+	}
+}
+
+uint8_t* scap_get_memorydumper_curpos(scap_dumper_t *d)
+{
+	return d->m_targetbufcurpos;
 }
 
 #ifndef _WIN32
@@ -715,7 +754,9 @@ scap_dumper_t *scap_dump_open(scap_t *handle, const char *fname, compression_mod
 	scap_dumper_t* res = (scap_dumper_t*)malloc(sizeof(scap_dumper_t));
 	res->m_f = f;
 	res->m_type = DT_FILE;
-	res->m_off = 0;
+	res->m_targetbuf = NULL;
+	res->m_targetbufcurpos = NULL;
+	res->m_targetbufend = NULL;
 
 	if(scap_setup_dump(handle, res, fname) != SCAP_SUCCESS)
 	{
@@ -725,6 +766,33 @@ scap_dumper_t *scap_dump_open(scap_t *handle, const char *fname, compression_mod
 	return res;
 }
 
+//
+// Open a memory "savefile"
+//
+scap_dumper_t *scap_memory_dump_open(scap_t *handle, uint8_t* targetbuf, uint64_t targetbufsize)
+{
+	scap_dumper_t* res = (scap_dumper_t*)malloc(sizeof(scap_dumper_t));
+	if(res == NULL)
+	{
+		snprintf(handle->m_lasterr, SCAP_LASTERR_SIZE, "scap_dump_memory_open memory allocation failure (1)");
+		return NULL;
+	}
+
+	res->m_f = NULL;
+	res->m_type = DT_MEM;
+	res->m_targetbuf = targetbuf;
+	res->m_targetbufcurpos = targetbuf;
+	res->m_targetbufend = targetbuf + targetbufsize;
+
+	if(scap_setup_dump(handle, res, "") != SCAP_SUCCESS)
+	{
+		free(res);
+		res = NULL;
+	}
+
+	return res;
+}
+
 //
 // Close a "savefile" opened with scap_dump_open
 //
@@ -749,7 +817,7 @@ int64_t scap_dump_get_offset(scap_dumper_t *d)
 	}
 	else
 	{
-		return d->m_off;
+		return (int64_t)d->m_targetbufcurpos - (int64_t)d->m_targetbuf;
 	}
 }
 

userspace/libsinsp/dumper.cpp

@@ -25,6 +25,16 @@ sinsp_dumper::sinsp_dumper(sinsp* inspector)
 {
 	m_inspector = inspector;
 	m_dumper = NULL;
+	m_target_memory_buffer = NULL;
+	m_target_memory_buffer_size = 0;
+}
+
+sinsp_dumper::sinsp_dumper(sinsp* inspector, uint8_t* target_memory_buffer, uint64_t target_memory_buffer_size)
+{
+	m_inspector = inspector;
+	m_dumper = NULL;
+	m_target_memory_buffer = target_memory_buffer;
+	m_target_memory_buffer_size = target_memory_buffer_size;
 }
 
 sinsp_dumper::~sinsp_dumper()
@@ -47,13 +57,20 @@ void sinsp_dumper::open(const string& filename, bool compress, bool threads_from
 		m_inspector->m_thread_manager->to_scap();
 	}
 
-	if(compress)
+	if(m_target_memory_buffer)
 	{
-		m_dumper = scap_dump_open(m_inspector->m_h, filename.c_str(), SCAP_COMPRESSION_GZIP);
+		m_dumper = scap_memory_dump_open(m_inspector->m_h, m_target_memory_buffer, m_target_memory_buffer_size);
 	}
 	else
 	{
-		m_dumper = scap_dump_open(m_inspector->m_h, filename.c_str(), SCAP_COMPRESSION_NONE);
+		if(compress)
+		{
+			m_dumper = scap_dump_open(m_inspector->m_h, filename.c_str(), SCAP_COMPRESSION_GZIP);
+		}
+		else
+		{
+			m_dumper = scap_dump_open(m_inspector->m_h, filename.c_str(), SCAP_COMPRESSION_NONE);
+		}
 	}
 
 	if(m_dumper == NULL)
@@ -64,6 +81,15 @@ void sinsp_dumper::open(const string& filename, bool compress, bool threads_from
 	m_inspector->m_container_manager.dump_containers(m_dumper);
 }
 
+void sinsp_dumper::close()
+{
+	if(m_dumper != NULL)
+	{
+		scap_dump_close(m_dumper);
+		m_dumper = NULL;
+	}
+}
+
 void sinsp_dumper::dump(sinsp_evt* evt)
 {
 	if(m_dumper == NULL)
@@ -86,7 +112,7 @@ uint64_t sinsp_dumper::written_bytes()
 {
 	if(m_dumper == NULL)
 	{
-		throw sinsp_exception("dumper not opened yet");
+		return 0;
 	}
 
 	int64_t written_bytes = scap_dump_get_offset(m_dumper);

userspace/libsinsp/dumper.h

@@ -40,6 +40,15 @@ class SINSP_PUBLIC sinsp_dumper
 	*/
 	sinsp_dumper(sinsp* inspector);
 
+	/*!
+	  \brief Constructs a dumper that saves to memory instead of disk.
+	  Takes the address and the size of a preallocated memory buffer
+	  where the data will go.
+	*/
+	sinsp_dumper(sinsp* inspector, 
+		uint8_t* target_memory_buffer, 
+		uint64_t target_memory_buffer_size);
+
 	~sinsp_dumper();
 
 	/*!
@@ -56,7 +65,14 @@ class SINSP_PUBLIC sinsp_dumper
 	  \note There's no close() because the file is closed when the dumper is
 	   destroyed.
 	*/
-	void open(const string& filename, bool compress, bool threads_from_sinsp=false);
+	void open(const string& filename, 
+		bool compress, 
+		bool threads_from_sinsp=false);
+
+	/*!
+	  \brief Closes the dump file.
+	*/
+	void close();
 
 	/*!
 	  \brief Return the current size of a tracefile.
@@ -77,9 +93,16 @@ class SINSP_PUBLIC sinsp_dumper
 	*/
 	void dump(sinsp_evt* evt);
 
+	inline uint8_t* get_memory_dump_cur_buf()
+	{
+		return scap_get_memorydumper_curpos(m_dumper);
+	}
+
 private:
 	sinsp* m_inspector;
 	scap_dumper_t* m_dumper;
+	uint8_t* m_target_memory_buffer;
+	uint64_t m_target_memory_buffer_size;
 };
 
 /*@}*/

userspace/libsinsp/event.h

@@ -419,6 +419,8 @@ VISIBILITY_PRIVATE
 	friend class sinsp_table;
 	friend class sinsp_cursesui;
 	friend class sisnp_baseliner;
+	friend class sinsp_memory_dumper;
+	friend class sinsp_memory_dumper_job;
 };
 
 /*@}*/

userspace/libsinsp/parsers.cpp

@@ -275,7 +275,7 @@ void sinsp_parser::process_event(sinsp_evt *evt)
 		store_event(evt);
 		break;
 	case PPME_SYSCALL_WRITE_E:
-		if(!m_inspector->m_dumper)
+		if(!m_inspector->m_is_dumping)
 		{
 			evt->m_fdinfo = evt->m_tinfo->get_fd(evt->m_tinfo->m_lastevent_fd);
 			if(evt->m_fdinfo)
@@ -1344,7 +1344,7 @@ void sinsp_parser::parse_clone_exit(sinsp_evt *evt)
 	ASSERT(parinfo->m_len == sizeof(int32_t));
 	tinfo.m_uid = *(int32_t *)parinfo->m_val;
 
-	// Copy the uid
+	// Copy the gid
 	switch(etype)
 	{
 	case PPME_SYSCALL_CLONE_11_X:
@@ -1596,10 +1596,16 @@ void sinsp_parser::parse_execve_exit(sinsp_evt *evt)
 	//  scap_fd_free_table(handle, tinfo);
 
 	//
-	// Clear the flags for this thread, making sure to propagate the inverted flag
+	// Clear the flags for this thread, making sure to propagate the inverted 
+	// and shell pipe flags
 	//
+
+	auto spf = evt->m_tinfo->m_flags & (PPM_CL_PIPE_SRC | PPM_CL_PIPE_DST);
 	bool inverted = ((evt->m_tinfo->m_flags & PPM_CL_CLONE_INVERTED) != 0);
+
 	evt->m_tinfo->m_flags = PPM_CL_ACTIVE;
+
+	evt->m_tinfo->m_flags |= spf;
 	if(inverted)
 	{
 		evt->m_tinfo->m_flags |= PPM_CL_CLONE_INVERTED;
@@ -1616,7 +1622,10 @@ void sinsp_parser::parse_execve_exit(sinsp_evt *evt)
 	evt->m_tinfo->compute_program_hash();
 
 #ifdef HAS_ANALYZER
-	evt->m_tinfo->m_ainfo->clear_role_flags();
+	if(evt->m_tinfo->m_ainfo != NULL)
+	{
+		evt->m_tinfo->m_ainfo->clear_role_flags();
+	}
 #endif
 
 	//
@@ -2846,7 +2855,7 @@ uint32_t sinsp_parser::parse_tracer(sinsp_evt *evt, int64_t retval)
 
 	if(p->m_res == sinsp_tracerparser::RES_TRUNCATED)
 	{
-		if(!m_inspector->m_dumper)
+		if(!m_inspector->m_is_dumping)
 		{
 			evt->m_filtered_out = true;
 		}
@@ -3481,6 +3490,18 @@ void sinsp_parser::parse_dup_exit(sinsp_evt *evt)
 	//
 	if(retval >= 0)
 	{
+		//
+		// Heuristic to determine if a thread is part of a shell pipe
+		//
+		if(retval == 0)
+		{
+			evt->m_tinfo->m_flags |= PPM_CL_PIPE_DST;
+		}
+		if(retval == 1)
+		{
+			evt->m_tinfo->m_flags |= PPM_CL_PIPE_SRC;
+		}
+
 		if(evt->m_fdinfo == NULL)
 		{
 			return;