Complete and fix cgroup cpu and memory limiting

* Implement GetCurrentProcessCpuCount for Unix
* Add CGroup CFS CPU limit support
* Support docker cgroup limits
 * The parsing would find the wrong '-' in lines like this:
354 347 0:28 /system.slice/docker-654dd7b6b8bbfe1739ae3309b471e95ccc82b3a3f56b7879f0a811d68b5c4e1d.scope /sys/fs/cgroup/cpuacct,cpu ro,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpuacct,cpu
 * In a docker container, the mount root and relative path in the cgroup have the same value, the relative path must not be appended to the mount point.

Author: janvorli

src/classlibnative/bcltype/system.cpp

@@ -393,6 +393,13 @@ INT32 QCALLTYPE SystemNative::GetProcessorCount()
         processorCount = systemInfo.dwNumberOfProcessors;
     }
 
+#ifdef FEATURE_PAL
+    uint32_t cpuLimit;
+
+    if (PAL_GetCpuLimit(&cpuLimit) && cpuLimit < processorCount)
+        processorCount = cpuLimit;
+#endif
+
     END_QCALL;
 
     return processorCount;

src/dlls/mscordac/mscordac_unixexports.src

@@ -22,6 +22,7 @@ PAL_GetResourceString
 PAL_get_stdout
 PAL_get_stderr
 PAL_GetCurrentThread
+PAL_GetCpuLimit
 PAL_GetSymbolModuleBase
 PAL_GetTransportPipeName
 PAL_InitializeDLL
@@ -106,6 +107,7 @@ GetLastError
 GetLongPathNameW
 GetModuleFileNameW
 GetProcAddress
+GetProcessAffinityMask
 GetProcessHeap
 GetShortPathNameW
 GetStdHandle

src/dlls/mscoree/mscorwks_unixexports.src

@@ -56,6 +56,7 @@ GetFileType
 GetFullPathNameW
 GetLongPathNameW
 GetProcAddress
+GetProcessAffinityMask
 GetStdHandle
 GetSystemInfo
 GetTempFileNameW

src/gc/unix/cgroup.cpp

@@ -9,7 +9,7 @@ Module Name:
     cgroup.cpp
 
 Abstract:
-    Read memory limits for the current process
+    Read memory and cpu limits for the current process
 --*/
 #include <cstdint>
 #include <cstddef>
@@ -26,42 +26,24 @@ Module Name:
 #define PROC_CGROUP_FILENAME "/proc/self/cgroup"
 #define PROC_STATM_FILENAME "/proc/self/statm"
 #define MEM_LIMIT_FILENAME "/memory.limit_in_bytes"
+#define CFS_QUOTA_FILENAME "/cpu.cfs_quota_us"
+#define CFS_PERIOD_FILENAME "/cpu.cfs_period_us"
 
 class CGroup
 {
     char* m_memory_cgroup_path;
+    char* m_cpu_cgroup_path;
 public:
     CGroup()
     {
-        m_memory_cgroup_path = nullptr;
-        char* memoryHierarchyMount = nullptr;
-        char *cgroup_path_relative_to_mount = nullptr;
-        size_t len;
-        memoryHierarchyMount = FindMemoryHierarchyMount();
-        if (memoryHierarchyMount == nullptr)
-            goto done;
-
-        cgroup_path_relative_to_mount = FindCGroupPathForMemorySubsystem();
-        if (cgroup_path_relative_to_mount == nullptr)
-            goto done;
-
-        len = strlen(memoryHierarchyMount);
-        len += strlen(cgroup_path_relative_to_mount);
-        m_memory_cgroup_path = (char*)malloc(len+1);
-        if (m_memory_cgroup_path == nullptr)
-           goto done;
-        
-        strcpy(m_memory_cgroup_path, memoryHierarchyMount);
-        strcat(m_memory_cgroup_path, cgroup_path_relative_to_mount);
-
-    done:
-        free(memoryHierarchyMount);
-        free(cgroup_path_relative_to_mount);
+        m_memory_cgroup_path = FindCgroupPath(&IsMemorySubsystem);
+        m_cpu_cgroup_path = FindCgroupPath(&IsCpuSubsystem);
     }
 
     ~CGroup()
     {
         free(m_memory_cgroup_path);
+        free(m_cpu_cgroup_path);
     }
 
     bool GetPhysicalMemoryLimit(size_t *val)
@@ -84,15 +66,89 @@ class CGroup
         free(mem_limit_filename);
         return result;
     }
+
+    bool GetCpuLimit(uint32_t *val)
+    {
+        long long quota;
+        long long period;
+        long long cpu_count;
+
+        quota = ReadCpuCGroupValue(CFS_QUOTA_FILENAME);
+        if (quota <= 0)
+            return false;
+
+        period = ReadCpuCGroupValue(CFS_PERIOD_FILENAME);
+        if (period <= 0)
+            return false;
+
+        // Cannot have less than 1 CPU
+        if (quota <= period)
+        {
+            *val = 1;
+            return true;
+        }
+        
+        cpu_count = quota / period;
+        if (cpu_count < UINT32_MAX)
+        {
+            *val = cpu_count;
+        }
+        else
+        {
+            *val = UINT32_MAX;
+        }
+
+        return true;
+    }
 
 private:
-    char* FindMemoryHierarchyMount()
+    static bool IsMemorySubsystem(const char *strTok){
+        return strcmp("memory", strTok) == 0;
+    }
+
+    static bool IsCpuSubsystem(const char *strTok){
+        return strcmp("cpu", strTok) == 0;
+    }
+
+    static char* FindCgroupPath(bool (*is_subsystem)(const char *)){
+        char *cgroup_path = nullptr;
+        char *hierarchy_mount = nullptr;
+        char *hierarchy_root = nullptr;
+        char *cgroup_path_relative_to_mount = nullptr;
+
+        FindHierarchyMount(is_subsystem, &hierarchy_mount, &hierarchy_root);
+        if (hierarchy_mount == nullptr || hierarchy_root == nullptr)
+            goto done;
+
+        cgroup_path_relative_to_mount = FindCGroupPathForSubsystem(is_subsystem);
+        if (cgroup_path_relative_to_mount == nullptr)
+            goto done;
+
+        cgroup_path = (char*)malloc(strlen(hierarchy_mount) + strlen(cgroup_path_relative_to_mount) + 1);
+        if (cgroup_path == nullptr)
+           goto done;
+
+        strcpy(cgroup_path, hierarchy_mount);
+        // For a host cgroup, we need to append the relative path.
+        // In a docker container, the root and relative path are the same and we don't need to append.
+        if (strcmp(hierarchy_root, cgroup_path_relative_to_mount) != 0)
+            strcat(cgroup_path, cgroup_path_relative_to_mount);
+
+    done:
+        free(hierarchy_mount);
+        free(hierarchy_root);
+        free(cgroup_path_relative_to_mount);
+        return cgroup_path;
+    }
+
+    static void FindHierarchyMount(bool (*is_subsystem)(const char *), char** pmountpath, char** pmountroot)
     {
         char *line = nullptr;
         size_t lineLen = 0, maxLineLen = 0;
         char *filesystemType = nullptr;
         char *options = nullptr;
-        char* mountpath = nullptr;
+        char *mountpath = nullptr;
+        char *mountroot = nullptr;
 
         FILE *mountinfofile = fopen(PROC_MOUNTINFO_FILENAME, "r");
         if (mountinfofile == nullptr)
@@ -113,11 +169,11 @@ class CGroup
                 maxLineLen = lineLen;
             }
 
-            char* separatorChar = strchr(line, '-');
+            char* separatorChar = strstr(line, " - ");
 
             // See man page of proc to get format for /proc/self/mountinfo file
             int sscanfRet = sscanf(separatorChar, 
-                                   "- %s %*s %s",
+                                   " - %s %*s %s",
                                    filesystemType,
                                    options);
             if (sscanfRet != 2)
@@ -132,37 +188,43 @@ class CGroup
                 char* strTok = strtok_r(options, ",", &context); 
                 while (strTok != nullptr)
                 {
-                    if (strncmp("memory", strTok, 6) == 0)
+                    if (is_subsystem(strTok))
                     {
                         mountpath = (char*)malloc(lineLen+1);
                         if (mountpath == nullptr)
                             goto done;
+                        mountroot = (char*)malloc(lineLen+1);
+                        if (mountroot == nullptr)
+                            goto done;
 
                         sscanfRet = sscanf(line,
-                                           "%*s %*s %*s %*s %s ",
+                                           "%*s %*s %*s %s %s ",
+                                           mountroot,
                                            mountpath);
-                        if (sscanfRet != 1)
-                        {
-                            free(mountpath);
-                            mountpath = nullptr;
+                        if (sscanfRet != 2)
                             assert(!"Failed to parse mount info file contents with sscanf.");
-                        }
+
+                        // assign the output arguments and clear the locals so we don't free them.
+                        *pmountpath = mountpath;
+                        *pmountroot = mountroot;
+                        mountpath = mountroot = nullptr;
                         goto done;
                     }
                     strTok = strtok_r(nullptr, ",", &context);
                 }
             }
         }
     done:
+        free(mountpath);
+        free(mountroot);
         free(filesystemType);
         free(options);
         free(line);
         if (mountinfofile)
             fclose(mountinfofile);
-        return mountpath;
     }
 
-    char* FindCGroupPathForMemorySubsystem()
+    static char* FindCGroupPathForSubsystem(bool (*is_subsystem)(const char *))
     {
         char *line = nullptr;
         size_t lineLen = 0;
@@ -205,7 +267,7 @@ class CGroup
             char* strTok = strtok_r(subsystem_list, ",", &context); 
             while (strTok != nullptr)
             {
-                if (strncmp("memory", strTok, 6) == 0)
+                if (is_subsystem(strTok))
                 {
                     result = true;
                     break;  
@@ -271,6 +333,59 @@ class CGroup
         free(line);    
         return result;
     }
+
+    long long ReadCpuCGroupValue(const char* subsystemFilename){
+        char *filename = nullptr;
+        bool result = false;
+        long long val;
+
+        if (m_cpu_cgroup_path == nullptr)
+            return -1;
+
+        filename = (char*)malloc(strlen(m_cpu_cgroup_path) + strlen(subsystemFilename) + 1);
+        if (filename == nullptr)
+            return -1;
+
+        strcpy(filename, m_cpu_cgroup_path);
+        strcat(filename, subsystemFilename);
+        result = ReadLongLongValueFromFile(filename, &val);
+        free(filename);
+        if (!result)
+             return -1;
+
+        return val;
+    }
+
+    bool ReadLongLongValueFromFile(const char* filename, long long* val)
+    {
+        bool result = false;
+        char *line = nullptr;
+        size_t lineLen = 0;
+
+        FILE* file = nullptr;
+    
+        if (val == nullptr)
+            goto done;
+    
+        file = fopen(filename, "r");
+        if (file == nullptr)
+            goto done;
+        
+        if (getline(&line, &lineLen, file) == -1)
+            goto done;
+
+        errno = 0;
+        *val = atoll(line);
+        if (errno != 0)
+            goto done;      
+
+        result = true;
+    done:
+        if (file)
+            fclose(file);
+        free(line);    
+        return result;
+    }
 };
 
 size_t GetRestrictedPhysicalMemoryLimit()
@@ -340,3 +455,13 @@ bool GetWorkingSetSize(size_t* val)
     free(line);
     return result;
 }
+
+bool GetCpuLimit(uint32_t* val)
+{
+    CGroup cgroup;
+
+    if (val == nullptr)
+        return false;
+
+    return cgroup.GetCpuLimit(val);
+}

src/gc/unix/config.h.in

@@ -12,5 +12,6 @@
 #cmakedefine01 HAVE_SCHED_GETCPU
 #cmakedefine01 HAVE_PTHREAD_CONDATTR_SETCLOCK
 #cmakedefine01 HAVE_MACH_ABSOLUTE_TIME
+#cmakedefine01 HAVE_SCHED_GETAFFINITY
 
 #endif // __CONFIG_H__

src/gc/unix/configure.cmake

@@ -52,4 +52,6 @@ check_cxx_source_runs("
     }
     " HAVE_MACH_ABSOLUTE_TIME)
 
+check_library_exists(c sched_getaffinity "" HAVE_SCHED_GETAFFINITY)
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h)