Calculate memory used as (rss + cache + swap) - inactive_file.

pkg/memory/memory.go

@@ -237,7 +237,8 @@ func readUsage() (memory, memory) {
 		log.Printf("couldn't access memory limit: %v", err)
 		return 0, unlimited
 	}
-	usage, err := readMemory("/sys/fs/cgroup/memory/memory.usage_in_bytes")
+
+	stats, err := readMemoryStat("/sys/fs/cgroup/memory/memory.stat")
 	if err != nil {
 		if errors.Is(err, os.ErrPermission) || errors.Is(err, os.ErrNotExist) {
 			// Don't complain if we don't have permission or the info doesn't exist.
@@ -247,7 +248,19 @@ func readUsage() (memory, memory) {
 		return 0, limit
 	}
 
-	return usage, limit
+	// We calculate memory usage according to the OOMKiller as (rss + cache + swap) - inactive_file.
+	// This is substantiated by this article[1] which claims we need to track container_memory_working_set_bytes.
+	// According to this stack overflow[2], container_memory_working_set_bytes is "total usage" - "inactive file".
+	// Best as I can tell from the cgroup docs[3], "total usage" is computed from memory.stat by
+	// adding (rss + cache + swap), and "inactive file" is just the inactive_file field.
+	//
+	// [1]: https://faun.pub/how-much-is-too-much-the-linux-oomkiller-and-used-memory-d32186f29c9d
+	// [2]: https://stackoverflow.com/questions/65428558/what-is-the-difference-between-container-memory-working-set-bytes-and-contain
+	// [3]: https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
+
+	totalUsage := stats.Rss + stats.Cache + stats.Swap
+	OOMUsage := totalUsage - stats.InactiveFile
+	return memory(OOMUsage), limit
 }
 
 // Read an int64 from a file and convert it to memory.
@@ -312,3 +325,48 @@ func readPerProcess() map[int]*ProcessUsage {
 
 	return result
 }
+
+type memoryStat struct {
+	Rss          uint64 // rss field
+	Cache        uint64 // cache field
+	Swap         uint64 // swap field
+	InactiveFile uint64 // inactive_file field
+}
+
+func readMemoryStat(fpath string) (memoryStat, error) {
+	bytes, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return memoryStat{}, err
+	}
+
+	return parseMemoryStat(string(bytes))
+}
+
+func parseMemoryStat(content string) (memoryStat, error) {
+	result := memoryStat{}
+	lines := strings.Split(content, "\n")
+	for _, line := range lines {
+		line = strings.TrimSuffix(line, "\n")
+		parts := strings.Fields(line)
+		if len(parts) != 2 {
+			continue
+		}
+
+		n, err := strconv.ParseUint(parts[1], 10, 64)
+		if err != nil {
+			return result, err
+		}
+
+		switch parts[0] {
+		case "rss":
+			result.Rss = n
+		case "swap":
+			result.Swap = n
+		case "cache":
+			result.Cache = n
+		case "inactive_file":
+			result.InactiveFile = n
+		}
+	}
+	return result, nil
+}

pkg/memory/memory_test.go

@@ -148,3 +148,51 @@ func TestMemoryUsageGCExited(t *testing.T) {
 	assert.Contains(t, m.perProcess, 5)
 
 }
+
+func TestParseMemoryStat(t *testing.T) {
+	assert := assert.New(t)
+	contents := `
+cache 175247360
+rss 403296256
+rss_huge 65011712
+shmem 0
+mapped_file 93401088
+dirty 0
+writeback 0
+swap 1
+pgpgin 5829351
+pgpgout 5726886
+pgfault 5848359
+pgmajfault 792
+inactive_anon 0
+active_anon 309968896
+inactive_file 222568448
+active_file 46092288
+unevictable 0
+hierarchical_memory_limit 2097152000
+hierarchical_memsw_limit 9223372036854771712
+total_cache 175247360
+total_rss 403296256
+total_rss_huge 65011712
+total_shmem 0
+total_mapped_file 93401088
+total_dirty 0
+total_writeback 0
+total_swap 0
+total_pgpgin 5829351
+total_pgpgout 5726886
+total_pgfault 5848359
+total_pgmajfault 792
+total_inactive_anon 0
+total_active_anon 309968896
+total_inactive_file 222568448
+total_active_file 46092288
+total_unevictable 0
+`
+	result, err := parseMemoryStat(contents)
+	assert.NoError(err)
+	assert.Equal(uint64(403296256), result.Rss)
+	assert.Equal(uint64(175247360), result.Cache)
+	assert.Equal(uint64(1), result.Swap)
+	assert.Equal(uint64(222568448), result.InactiveFile)
+}