Skip to content

Commit

Permalink
mm: memcontrol: use obj_cgroup APIs to charge the LRU pages
Browse files Browse the repository at this point in the history 
We will reuse the obj_cgroup APIs to charge the LRU pages. Finally,
page->memcg_data will have 2 different meanings.

  - For the slab pages, page->memcg_data points to an object cgroups
    vector.

  - For the kmem pages (exclude the slab pages) and the LRU pages,
    page->memcg_data points to an object cgroup.

In this patch, we reuse obj_cgroup APIs to charge LRU pages. In the end,
The page cache cannot prevent long-living objects from pinning the original
memory cgroup in the memory.

At the same time we also changed the rules of page and objcg or memcg
binding stability. The new rules are as follows.

For a page any of the following ensures page and objcg binding stability:

  - the page lock
  - LRU isolation
  - lock_page_memcg()
  - exclusive reference

Based on the stable binding of page and objcg, for a page any of the
following ensures page and memcg binding stability:

  - css_set_lock
  - cgroup_mutex
  - the lruvec lock
  - the split queue lock (only THP page)

If the caller only want to ensure that the page counters of memcg are
updated correctly, ensure that the binding stability of page and objcg
is sufficient.

Signed-off-by: Muchun Song <songmuchun@bytedance.com>
  • Loading branch information
@intel-lab-lkp
Muchun Song authored and intel-lab-lkp committed Sep 16, 2021
1 parent 346d67a commit a919496
Show file tree
Hide file tree
Showing 3 changed files with 292 additions and 142 deletions.
96 changes: 36 additions & 60 deletions include/linux/memcontrol.h
View file
Expand Up @@ -376,8 +376,6 @@ enum page_memcg_data_flags {

#define MEMCG_DATA_FLAGS_MASK (__NR_MEMCG_DATA_FLAGS - 1)

static inline bool PageMemcgKmem(struct page *page);

/*
* After the initialization objcg->memcg is always pointing at
* a valid memcg, but can be atomically swapped to the parent memcg.
Expand All @@ -391,43 +389,19 @@ static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg)
}

/*
* __page_memcg - get the memory cgroup associated with a non-kmem page
* @page: a pointer to the page struct
*
* Returns a pointer to the memory cgroup associated with the page,
* or NULL. This function assumes that the page is known to have a
* proper memory cgroup pointer. It's not safe to call this function
* against some type of pages, e.g. slab pages or ex-slab pages or
* kmem pages.
*/
static inline struct mem_cgroup *__page_memcg(struct page *page)
{
unsigned long memcg_data = page->memcg_data;

VM_BUG_ON_PAGE(PageSlab(page), page);
VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_OBJCGS, page);
VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_KMEM, page);

return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
}

/*
* __page_objcg - get the object cgroup associated with a kmem page
* page_objcg - get the object cgroup associated with page
* @page: a pointer to the page struct
*
* Returns a pointer to the object cgroup associated with the page,
* or NULL. This function assumes that the page is known to have a
* proper object cgroup pointer. It's not safe to call this function
* against some type of pages, e.g. slab pages or ex-slab pages or
* LRU pages.
* proper object cgroup pointer.
*/
static inline struct obj_cgroup *__page_objcg(struct page *page)
static inline struct obj_cgroup *page_objcg(struct page *page)
{
unsigned long memcg_data = page->memcg_data;

VM_BUG_ON_PAGE(PageSlab(page), page);
VM_BUG_ON_PAGE(memcg_data & MEMCG_DATA_OBJCGS, page);
VM_BUG_ON_PAGE(!(memcg_data & MEMCG_DATA_KMEM), page);

return (struct obj_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
}
Expand All @@ -441,23 +415,35 @@ static inline struct obj_cgroup *__page_objcg(struct page *page)
* proper memory cgroup pointer. It's not safe to call this function
* against some type of pages, e.g. slab pages or ex-slab pages.
*
* For a non-kmem page any of the following ensures page and memcg binding
* stability:
* For a page any of the following ensures page and objcg binding stability:
*
* - the page lock
* - LRU isolation
* - lock_page_memcg()
* - exclusive reference
*
* For a kmem page a caller should hold an rcu read lock to protect memcg
* associated with a kmem page from being released.
* Based on the stable binding of page and objcg, for a page any of the
* following ensures page and memcg binding stability:
*
* - css_set_lock
* - cgroup_mutex
* - the lruvec lock
* - the split queue lock (only THP page)
*
* If the caller only want to ensure that the page counters of memcg are
* updated correctly, ensure that the binding stability of page and objcg
* is sufficient.
*
* A caller should hold an rcu read lock (In addition, regions of code across
* which interrupts, preemption, or softirqs have been disabled also serve as
* RCU read-side critical sections) to protect memcg associated with a page
* from being released.
*/
static inline struct mem_cgroup *page_memcg(struct page *page)
{
if (PageMemcgKmem(page))
return obj_cgroup_memcg(__page_objcg(page));
else
return __page_memcg(page);
struct obj_cgroup *objcg = page_objcg(page);

return objcg ? obj_cgroup_memcg(objcg) : NULL;
}

/*
Expand All @@ -470,6 +456,8 @@ static inline struct mem_cgroup *page_memcg(struct page *page)
* is known to have a proper memory cgroup pointer. It's not safe to call
* this function against some type of pages, e.g. slab pages or ex-slab
* pages.
*
* The page and objcg or memcg binding rules can refer to page_memcg().
*/
static inline struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
{
Expand All @@ -493,22 +481,20 @@ static inline struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
* or NULL. This function assumes that the page is known to have a
* proper memory cgroup pointer. It's not safe to call this function
* against some type of pages, e.g. slab pages or ex-slab pages.
*
* The page and objcg or memcg binding rules can refer to page_memcg().
*/
static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
{
unsigned long memcg_data = READ_ONCE(page->memcg_data);
struct obj_cgroup *objcg;

VM_BUG_ON_PAGE(PageSlab(page), page);
WARN_ON_ONCE(!rcu_read_lock_held());

if (memcg_data & MEMCG_DATA_KMEM) {
struct obj_cgroup *objcg;

objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
return obj_cgroup_memcg(objcg);
}
objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);

return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
return objcg ? obj_cgroup_memcg(objcg) : NULL;
}

/*
Expand All @@ -521,16 +507,10 @@ static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
* has an associated memory cgroup pointer or an object cgroups vector or
* an object cgroup.
*
* For a non-kmem page any of the following ensures page and memcg binding
* stability:
*
* - the page lock
* - LRU isolation
* - lock_page_memcg()
* - exclusive reference
* The page and objcg or memcg binding rules can refer to page_memcg().
*
* For a kmem page a caller should hold an rcu read lock to protect memcg
* associated with a kmem page from being released.
* A caller should hold an rcu read lock to protect memcg associated with a
* page from being released.
*/
static inline struct mem_cgroup *page_memcg_check(struct page *page)
{
Expand All @@ -539,18 +519,14 @@ static inline struct mem_cgroup *page_memcg_check(struct page *page)
* for slab pages, READ_ONCE() should be used here.
*/
unsigned long memcg_data = READ_ONCE(page->memcg_data);
struct obj_cgroup *objcg;

if (memcg_data & MEMCG_DATA_OBJCGS)
return NULL;

if (memcg_data & MEMCG_DATA_KMEM) {
struct obj_cgroup *objcg;

objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
return obj_cgroup_memcg(objcg);
}
objcg = (void *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);

return (struct mem_cgroup *)(memcg_data & ~MEMCG_DATA_FLAGS_MASK);
return objcg ? obj_cgroup_memcg(objcg) : NULL;
}

#ifdef CONFIG_MEMCG_KMEM
Expand Down
42 changes: 42 additions & 0 deletions mm/huge_memory.c
View file
Expand Up @@ -499,6 +499,48 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
}

#ifdef CONFIG_MEMCG
static struct shrinker deferred_split_shrinker;

static void memcg_reparent_split_queue_lock(struct mem_cgroup *memcg,
struct mem_cgroup *parent)
{
spin_lock(&memcg->deferred_split_queue.split_queue_lock);
spin_lock(&parent->deferred_split_queue.split_queue_lock);
}

static void memcg_reparent_split_queue_unlock(struct mem_cgroup *memcg,
struct mem_cgroup *parent)
{
spin_unlock(&parent->deferred_split_queue.split_queue_lock);
spin_unlock(&memcg->deferred_split_queue.split_queue_lock);
}

static void memcg_reparent_split_queue(struct mem_cgroup *memcg,
struct mem_cgroup *parent)
{
int nid;
struct deferred_split *src, *dst;

src = &memcg->deferred_split_queue;
dst = &parent->deferred_split_queue;

if (!src->split_queue_len)
return;

list_splice_tail_init(&src->split_queue, &dst->split_queue);
dst->split_queue_len += src->split_queue_len;
src->split_queue_len = 0;

for_each_node(nid)
set_shrinker_bit(parent, nid, deferred_split_shrinker.id);
}

const struct memcg_reparent_ops split_queue_reparent_ops = {
.lock = memcg_reparent_split_queue_lock,
.unlock = memcg_reparent_split_queue_unlock,
.reparent = memcg_reparent_split_queue,
};

static inline struct mem_cgroup *split_queue_memcg(struct deferred_split *queue)
{
if (mem_cgroup_disabled())
Expand Down

0 comments on commit a919496

Please sign in to comment.