more cache improvements

- don't clean cache by number of items.
- clean inodes from cache every 2' if the descriptor symlink doesn't exist
  anymore, or if the lastSeen time is more than 5 minutes.
- launch cache cleaners before start a new process monitoring method,
  and start it only once for the life time of the daemon.
- do not store in cache the Time objects, only the nanoseconds of
  the last updated time.
- if the inode of a connection is found in cache, reorder the
  descriptors to push the descritptor to the top of the list.
  Also add cached the inode.

It turns out that when a new connection is about to be established,
when the process resolves the domain, the same inode is used to open the
tcp connection to the target. So if it's cached we save CPU cycles.

This also occurs when we block a connection and the process retries it,
or when a connection timeouts and the process retries it
(telnet 1.1.1.1).

daemon/procmon/activepids.go

@@ -35,19 +35,22 @@ func monitorActivePids() {
 			if err != nil {
 				//file does not exists, pid has quit
 				delete(activePids, k)
+				deleteProcEntry(int(k))
 				continue
 			}
 			startTime, err := strconv.Atoi(strings.Split(string(data), " ")[21])
 			if err != nil {
 				log.Error("Could not find or convert Starttime. This should never happen. Please report this incident to the Opensnitch developers.")
 				delete(activePids, k)
+				deleteProcEntry(int(k))
 				continue
 			}
 			if uint32(startTime) != v.Starttime {
 				//extremely unlikely: the original process has quit and another process
 				//was started with the same PID - all this in less than 1 second
 				log.Error("Same PID but different Starttime. Please report this incident to the Opensnitch developers.")
 				delete(activePids, k)
+				deleteProcEntry(int(k))
 				continue
 			}
 		}

daemon/procmon/cache.go

@@ -4,63 +4,87 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"sync"
 	"time"
+
+	"github.com/evilsocket/opensnitch/daemon/core"
+)
+
+var (
+	cLock       = sync.RWMutex{}
+	cacheTicker = time.NewTicker(2 * time.Minute)
 )
 
 // Inode represents an item of the InodesCache.
 // the key is formed as follow:
 // inode+srcip+srcport+dstip+dstport
 type Inode struct {
-	Pid    int
-	FdPath string
+	Pid      int
+	FdPath   string
+	LastSeen int64
 }
 
 // ProcEntry represents an item of the pidsCache
 type ProcEntry struct {
 	Pid         int
 	FdPath      string
 	Descriptors []string
-	Time        time.Time
+	LastSeen    int64
 }
 
 var (
 	// cache of inodes, which help to not iterate over all the pidsCache and
 	// descriptors of /proc/<pid>/fd/
-	// 20-50us vs 50-80ms
-	inodesCache     = make(map[string]*Inode)
-	maxCachedInodes = 128
+	// 15-50us vs 50-80ms
+	// we hit this cache when:
+	// - we've blocked a connection and the process retries it several times until it gives up,
+	// - or when a process timeouts connecting to an IP/domain and it retries it again,
+	// - or when a process resolves a domain and then connects to the IP.
+	inodesCache = make(map[string]*Inode)
+	maxTTL      = 5 // maximum 5 minutes of inactivity in cache. Really rare, usually they lasts less than a minute.
+
 	// 2nd cache of already known running pids, which also saves time by
-	// iterating only over a few pids' descriptors, (30us-2ms vs. 50-80ms)
+	// iterating only over a few pids' descriptors, (30us-20ms vs. 50-80ms)
 	// since it's more likely that most of the connections will be made by the
 	// same (running) processes.
 	// The cache is ordered by time, placing in the first places those PIDs with
 	// active connections.
 	pidsCache            []*ProcEntry
 	pidsDescriptorsCache = make(map[int][]string)
-	maxCachedPids        = 24
 )
 
 func addProcEntry(fdPath string, fdList []string, pid int) {
 	for n := range pidsCache {
 		if pidsCache[n].Pid == pid {
 			pidsCache[n].Descriptors = fdList
-			pidsCache[n].Time = time.Now()
+			pidsCache[n].LastSeen = time.Now().UnixNano()
 			return
 		}
 	}
 	procEntry := &ProcEntry{
 		Pid:         pid,
 		FdPath:      fdPath,
 		Descriptors: fdList,
-		Time:        time.Now(),
+		LastSeen:    time.Now().UnixNano(),
 	}
 	pidsCache = append([]*ProcEntry{procEntry}, pidsCache...)
 }
 
+func addInodeEntry(key, descLink string, pid int) {
+	cLock.Lock()
+	defer cLock.Unlock()
+
+	inodesCache[key] = &Inode{
+		FdPath:   descLink,
+		Pid:      pid,
+		LastSeen: time.Now().UnixNano(),
+	}
+}
+
 func sortProcEntries() {
 	sort.Slice(pidsCache, func(i, j int) bool {
-		t := pidsCache[i].Time.UnixNano()
-		u := pidsCache[j].Time.UnixNano()
+		t := pidsCache[i].LastSeen
+		u := pidsCache[j].LastSeen
 		return t > u || t == u
 	})
 }
@@ -76,28 +100,48 @@ func deleteProcEntry(pid int) {
 }
 
 func deleteInodeEntry(pid int) {
+	cLock.Lock()
+	defer cLock.Unlock()
+
 	for k, inodeEntry := range inodesCache {
 		if inodeEntry.Pid == pid {
 			delete(inodesCache, k)
 		}
 	}
 }
 
-func cleanUpCaches() {
-	if len(inodesCache) > maxCachedInodes {
-		for k := range inodesCache {
-			delete(inodesCache, k)
+func cacheCleanerTask() {
+	for {
+		select {
+		case <-cacheTicker.C:
+			cleanupInodes()
 		}
 	}
-	if len(pidsCache) > maxCachedPids {
-		pidsCache = nil
+}
+
+func cleanupInodes() {
+	cLock.Lock()
+	defer cLock.Unlock()
+
+	now := time.Now()
+	for k := range inodesCache {
+		lastSeen := now.Sub(
+			time.Unix(0, inodesCache[k].LastSeen),
+		)
+		if core.Exists(inodesCache[k].FdPath) == false || int(lastSeen.Minutes()) > maxTTL {
+			delete(inodesCache, k)
+		}
 	}
 }
 
 func getPidByInodeFromCache(inodeKey string) int {
+	cLock.Lock()
+	defer cLock.Unlock()
+
 	if _, found := inodesCache[inodeKey]; found == true {
 		// sometimes the process may have disappeared at this point
 		if _, err := os.Lstat(fmt.Sprint("/proc/", inodesCache[inodeKey].Pid, "/exe")); err == nil {
+			inodesCache[inodeKey].LastSeen = time.Now().UnixNano()
 			return inodesCache[inodeKey].Pid
 		}
 		deleteProcEntry(inodesCache[inodeKey].Pid)
@@ -106,7 +150,7 @@ func getPidByInodeFromCache(inodeKey string) int {
 	return -1
 }
 
-func getPidDescriptorsFromCache(fdPath string, expect string, descriptors *[]string) (int, *[]string) {
+func getPidDescriptorsFromCache(fdPath, inodeKey, expect string, descriptors *[]string, pid int) (int, *[]string) {
 	for fdIdx := 0; fdIdx < len(*descriptors); fdIdx++ {
 		descLink := fmt.Sprint(fdPath, (*descriptors)[fdIdx])
 		if link, err := os.Readlink(descLink); err == nil && link == expect {
@@ -116,6 +160,9 @@ func getPidDescriptorsFromCache(fdPath string, expect string, descriptors *[]str
 				*descriptors = append((*descriptors)[:fdIdx], (*descriptors)[fdIdx+1:]...)
 				*descriptors = append([]string{fd}, *descriptors...)
 			}
+			if _, found := inodesCache[inodeKey]; !found {
+				addInodeEntry(inodeKey, descLink, pid)
+			}
 			return fdIdx, descriptors
 		}
 	}
@@ -126,31 +173,26 @@ func getPidDescriptorsFromCache(fdPath string, expect string, descriptors *[]str
 func getPidFromCache(inode int, inodeKey string, expect string) (int, int) {
 	// loop over the processes that have generated connections
 	for n := 0; n < len(pidsCache); n++ {
-		procEntry := pidsCache[n]
-
-		if idxDesc, newFdList := getPidDescriptorsFromCache(procEntry.FdPath, expect, &procEntry.Descriptors); idxDesc != -1 {
-			pidsCache[n].Time = time.Now()
+		if idxDesc, newFdList := getPidDescriptorsFromCache(pidsCache[n].FdPath, inodeKey, expect, &pidsCache[n].Descriptors, pidsCache[n].Pid); idxDesc != -1 {
+			pidsCache[n].LastSeen = time.Now().UnixNano()
 			pidsCache[n].Descriptors = *newFdList
-			return procEntry.Pid, n
+			return pidsCache[n].Pid, n
 		}
 	}
 	// inode not found in cache, we need to refresh the list of descriptors
-	// to see if any of the known PIDs has opened a new socket, and update
-	// the new list of file descriptors for that PID.
-
+	// to see if any known PID has opened a new socket
 	for n := 0; n < len(pidsCache); n++ {
-		procEntry := pidsCache[n]
-		descriptors := lookupPidDescriptors(procEntry.FdPath, procEntry.Pid)
+		descriptors := lookupPidDescriptors(pidsCache[n].FdPath, pidsCache[n].Pid)
 		if descriptors == nil {
-			deleteProcEntry(procEntry.Pid)
+			deleteProcEntry(pidsCache[n].Pid)
 			continue
 		}
 
 		pidsCache[n].Descriptors = descriptors
-		if idxDesc, newFdList := getPidDescriptorsFromCache(procEntry.FdPath, expect, &descriptors); idxDesc != -1 {
-			pidsCache[n].Time = time.Now()
+		if idxDesc, newFdList := getPidDescriptorsFromCache(pidsCache[n].FdPath, inodeKey, expect, &descriptors, pidsCache[n].Pid); idxDesc != -1 {
+			pidsCache[n].LastSeen = time.Now().UnixNano()
 			pidsCache[n].Descriptors = *newFdList
-			return procEntry.Pid, n
+			return pidsCache[n].Pid, n
 		}
 	}
 

daemon/procmon/cache_test.go

@@ -3,6 +3,7 @@ package procmon
 import (
 	"fmt"
 	"testing"
+	"time"
 )
 
 func TestCacheProcs(t *testing.T) {
@@ -22,8 +23,10 @@ func TestCacheProcs(t *testing.T) {
 		if len(pidsCache) != 1 {
 			t.Error("pidsCache should be still 1!", pidsCache)
 		}
-		if oldPid.Time.Equal(pidsCache[0].Time) == false {
-			t.Error("pidsCache, time not updated: ", oldPid.Time, pidsCache[0].Time)
+		oldTime := time.Unix(0, oldPid.LastSeen)
+		newTime := time.Unix(0, pidsCache[0].LastSeen)
+		if oldTime.Equal(newTime) == false {
+			t.Error("pidsCache, time not updated: ", oldTime, newTime)
 		}
 	})
 
@@ -42,16 +45,27 @@ func TestCacheProcs(t *testing.T) {
 		}
 	})
 
-	for pid := 3; pid < 27; pid++ {
-		addProcEntry(fmt.Sprint("/proc/", pid, "/fd/"), fdList, pid)
-	}
-	if len(pidsCache) != 25 {
-		t.Error("pidsCache should be 0:", len(pidsCache))
-	}
-	cleanUpCaches()
-	t.Run("Test cleanUpCaches", func(t *testing.T) {
-		if len(pidsCache) != 0 {
-			t.Error("pidsCache should be 0:", len(pidsCache))
+	// the key of an inodeCache entry is formed as: inodeNumer + srcIP + srcPort + dstIP + dstPort
+	inodeKey := "000000000127.0.0.144444127.0.0.153"
+	addInodeEntry(inodeKey, "/proc/fd/123", myPid)
+	t.Run("Test addInodeEntry", func(t *testing.T) {
+		if _, found := inodesCache[inodeKey]; !found {
+			t.Error("inodesCache, inode not added:", len(inodesCache), inodesCache)
+		}
+	})
+
+	pid = getPidByInodeFromCache(inodeKey)
+	t.Run("Test getPidByInodeFromCache", func(t *testing.T) {
+		if pid != myPid {
+			t.Error("inode not found in cache", pid, inodeKey, len(inodesCache), inodesCache)
+		}
+	})
+
+	// should delete all inodes of a pid
+	deleteInodeEntry(myPid)
+	t.Run("Test deleteInodeEntry", func(t *testing.T) {
+		if _, found := inodesCache[inodeKey]; found {
+			t.Error("inodesCache, key found in cache but it should not exist", inodeKey, len(inodesCache), inodesCache)
 		}
 	})
 }

daemon/procmon/find.go

@@ -31,7 +31,7 @@ func inodeFound(pidsPath, expect, inodeKey string, inode, pid int) bool {
 	for idx := 0; idx < len(fdList); idx++ {
 		descLink := fmt.Sprint(fdPath, fdList[idx])
 		if link, err := os.Readlink(descLink); err == nil && link == expect {
-			inodesCache[inodeKey] = &Inode{FdPath: descLink, Pid: pid}
+			addInodeEntry(inodeKey, descLink, pid)
 			addProcEntry(fdPath, fdList, pid)
 			return true
 		}

daemon/procmon/parse.go

@@ -44,7 +44,6 @@ func GetPIDFromINode(inode int, inodeKey string) int {
 		return found
 	}
 	start := time.Now()
-	cleanUpCaches()
 
 	expect := fmt.Sprintf("socket:[%d]", inode)
 	if cachedPidInode := getPidByInodeFromCache(inodeKey); cachedPidInode != -1 {

daemon/procmon/process.go

@@ -7,6 +7,10 @@ import (
 	"github.com/evilsocket/opensnitch/daemon/procmon/audit"
 )
 
+var (
+	cacheMonitorsRunning = false
+)
+
 // man 5 proc; man procfs
 type procIOstats struct {
 	RChar        int64
@@ -104,6 +108,12 @@ func End() {
 
 // Init starts parsing connections using the method specified.
 func Init() {
+	if cacheMonitorsRunning == false {
+		go monitorActivePids()
+		go cacheCleanerTask()
+		cacheMonitorsRunning = true
+	}
+
 	if methodIsFtrace() {
 		err := Start()
 		if err == nil {
@@ -125,5 +135,4 @@ func Init() {
 	// if any of the above methods have failed, fallback to proc
 	log.Info("Process monitor method /proc")
 	SetMonitorMethod(MethodProc)
-	go monitorActivePids()
 }