Skip to content

Commit 4f98a2f

Browse files
Rik van Rieltorvalds
Rik van Riel
authored and
torvalds
committed
vmscan: split LRU lists into anon & file sets
Split the LRU lists in two, one set for pages that are backed by real file systems ("file") and one for pages that are backed by memory and swap ("anon"). The latter includes tmpfs. The advantage of doing this is that the VM will not have to scan over lots of anonymous pages (which we generally do not want to swap out), just to find the page cache pages that it should evict. This patch has the infrastructure and a basic policy to balance how much we scan the anon lists and how much we scan the file lists. The big policy changes are in separate patches. [lee.schermerhorn@hp.com: collect lru meminfo statistics from correct offset] [kosaki.motohiro@jp.fujitsu.com: prevent incorrect oom under split_lru] [kosaki.motohiro@jp.fujitsu.com: fix pagevec_move_tail() doesn't treat unevictable page] [hugh@veritas.com: memcg swapbacked pages active] [hugh@veritas.com: splitlru: BDI_CAP_SWAP_BACKED] [akpm@linux-foundation.org: fix /proc/vmstat units] [nishimura@mxp.nes.nec.co.jp: memcg: fix handling of shmem migration] [kosaki.motohiro@jp.fujitsu.com: adjust Quicklists field of /proc/meminfo] [kosaki.motohiro@jp.fujitsu.com: fix style issue of get_scan_ratio()] Signed-off-by: Rik van Riel <riel@redhat.com> Signed-off-by: Lee Schermerhorn <Lee.Schermerhorn@hp.com> Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Signed-off-by: Hugh Dickins <hugh@veritas.com> Signed-off-by: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent b2e1853 commit 4f98a2f

25 files changed

Lines changed: 562 additions & 367 deletions

‎drivers/base/node.c‎

Lines changed: 33 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -61,34 +61,44 @@ static ssize_t node_read_meminfo(struct sys_device * dev,
6161
si_meminfo_node(&i, nid);
6262

6363
n = sprintf(buf, "\n"
64-
"Node %d MemTotal: %8lu kB\n"
65-
"Node %d MemFree: %8lu kB\n"
66-
"Node %d MemUsed: %8lu kB\n"
67-
"Node %d Active: %8lu kB\n"
68-
"Node %d Inactive: %8lu kB\n"
64+
"Node %d MemTotal: %8lu kB\n"
65+
"Node %d MemFree: %8lu kB\n"
66+
"Node %d MemUsed: %8lu kB\n"
67+
"Node %d Active: %8lu kB\n"
68+
"Node %d Inactive: %8lu kB\n"
69+
"Node %d Active(anon): %8lu kB\n"
70+
"Node %d Inactive(anon): %8lu kB\n"
71+
"Node %d Active(file): %8lu kB\n"
72+
"Node %d Inactive(file): %8lu kB\n"
6973
#ifdef CONFIG_HIGHMEM
70-
"Node %d HighTotal: %8lu kB\n"
71-
"Node %d HighFree: %8lu kB\n"
72-
"Node %d LowTotal: %8lu kB\n"
73-
"Node %d LowFree: %8lu kB\n"
74+
"Node %d HighTotal: %8lu kB\n"
75+
"Node %d HighFree: %8lu kB\n"
76+
"Node %d LowTotal: %8lu kB\n"
77+
"Node %d LowFree: %8lu kB\n"
7478
#endif
75-
"Node %d Dirty: %8lu kB\n"
76-
"Node %d Writeback: %8lu kB\n"
77-
"Node %d FilePages: %8lu kB\n"
78-
"Node %d Mapped: %8lu kB\n"
79-
"Node %d AnonPages: %8lu kB\n"
80-
"Node %d PageTables: %8lu kB\n"
81-
"Node %d NFS_Unstable: %8lu kB\n"
82-
"Node %d Bounce: %8lu kB\n"
83-
"Node %d WritebackTmp: %8lu kB\n"
84-
"Node %d Slab: %8lu kB\n"
85-
"Node %d SReclaimable: %8lu kB\n"
86-
"Node %d SUnreclaim: %8lu kB\n",
79+
"Node %d Dirty: %8lu kB\n"
80+
"Node %d Writeback: %8lu kB\n"
81+
"Node %d FilePages: %8lu kB\n"
82+
"Node %d Mapped: %8lu kB\n"
83+
"Node %d AnonPages: %8lu kB\n"
84+
"Node %d PageTables: %8lu kB\n"
85+
"Node %d NFS_Unstable: %8lu kB\n"
86+
"Node %d Bounce: %8lu kB\n"
87+
"Node %d WritebackTmp: %8lu kB\n"
88+
"Node %d Slab: %8lu kB\n"
89+
"Node %d SReclaimable: %8lu kB\n"
90+
"Node %d SUnreclaim: %8lu kB\n",
8791
nid, K(i.totalram),
8892
nid, K(i.freeram),
8993
nid, K(i.totalram - i.freeram),
90-
nid, K(node_page_state(nid, NR_ACTIVE)),
91-
nid, K(node_page_state(nid, NR_INACTIVE)),
94+
nid, K(node_page_state(nid, NR_ACTIVE_ANON) +
95+
node_page_state(nid, NR_ACTIVE_FILE)),
96+
nid, K(node_page_state(nid, NR_INACTIVE_ANON) +
97+
node_page_state(nid, NR_INACTIVE_FILE)),
98+
nid, K(node_page_state(nid, NR_ACTIVE_ANON)),
99+
nid, K(node_page_state(nid, NR_INACTIVE_ANON)),
100+
nid, K(node_page_state(nid, NR_ACTIVE_FILE)),
101+
nid, K(node_page_state(nid, NR_INACTIVE_FILE)),
92102
#ifdef CONFIG_HIGHMEM
93103
nid, K(i.totalhigh),
94104
nid, K(i.freehigh),

‎fs/cifs/file.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1791,7 +1791,7 @@ static void cifs_copy_cache_pages(struct address_space *mapping,
17911791
SetPageUptodate(page);
17921792
unlock_page(page);
17931793
if (!pagevec_add(plru_pvec, page))
1794-
__pagevec_lru_add(plru_pvec);
1794+
__pagevec_lru_add_file(plru_pvec);
17951795
data += PAGE_CACHE_SIZE;
17961796
}
17971797
return;
@@ -1925,7 +1925,7 @@ static int cifs_readpages(struct file *file, struct address_space *mapping,
19251925
bytes_read = 0;
19261926
}
19271927

1928-
pagevec_lru_add(&lru_pvec);
1928+
pagevec_lru_add_file(&lru_pvec);
19291929

19301930
/* need to free smb_read_data buf before exit */
19311931
if (smb_read_data) {

‎fs/nfs/dir.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1517,7 +1517,7 @@ static int nfs_symlink(struct inode *dir, struct dentry *dentry, const char *sym
15171517
if (!add_to_page_cache(page, dentry->d_inode->i_mapping, 0,
15181518
GFP_KERNEL)) {
15191519
pagevec_add(&lru_pvec, page);
1520-
pagevec_lru_add(&lru_pvec);
1520+
pagevec_lru_add_file(&lru_pvec);
15211521
SetPageUptodate(page);
15221522
unlock_page(page);
15231523
} else

‎fs/ntfs/file.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -439,7 +439,7 @@ static inline int __ntfs_grab_cache_pages(struct address_space *mapping,
439439
pages[nr] = *cached_page;
440440
page_cache_get(*cached_page);
441441
if (unlikely(!pagevec_add(lru_pvec, *cached_page)))
442-
__pagevec_lru_add(lru_pvec);
442+
__pagevec_lru_add_file(lru_pvec);
443443
*cached_page = NULL;
444444
}
445445
index++;
@@ -2084,7 +2084,7 @@ static ssize_t ntfs_file_buffered_write(struct kiocb *iocb,
20842084
OSYNC_METADATA|OSYNC_DATA);
20852085
}
20862086
}
2087-
pagevec_lru_add(&lru_pvec);
2087+
pagevec_lru_add_file(&lru_pvec);
20882088
ntfs_debug("Done. Returning %s (written 0x%lx, status %li).",
20892089
written ? "written" : "status", (unsigned long)written,
20902090
(long)status);

‎fs/proc/proc_misc.c‎

Lines changed: 45 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,8 @@ static int meminfo_read_proc(char *page, char **start, off_t off,
136136
unsigned long allowed;
137137
struct vmalloc_info vmi;
138138
long cached;
139+
unsigned long pages[NR_LRU_LISTS];
140+
int lru;
139141

140142
/*
141143
* display in kilobytes.
@@ -154,51 +156,62 @@ static int meminfo_read_proc(char *page, char **start, off_t off,
154156

155157
get_vmalloc_info(&vmi);
156158

159+
for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
160+
pages[lru] = global_page_state(NR_LRU_BASE + lru);
161+
157162
/*
158163
* Tagged format, for easy grepping and expansion.
159164
*/
160165
len = sprintf(page,
161-
"MemTotal: %8lu kB\n"
162-
"MemFree: %8lu kB\n"
163-
"Buffers: %8lu kB\n"
164-
"Cached: %8lu kB\n"
165-
"SwapCached: %8lu kB\n"
166-
"Active: %8lu kB\n"
167-
"Inactive: %8lu kB\n"
166+
"MemTotal: %8lu kB\n"
167+
"MemFree: %8lu kB\n"
168+
"Buffers: %8lu kB\n"
169+
"Cached: %8lu kB\n"
170+
"SwapCached: %8lu kB\n"
171+
"Active: %8lu kB\n"
172+
"Inactive: %8lu kB\n"
173+
"Active(anon): %8lu kB\n"
174+
"Inactive(anon): %8lu kB\n"
175+
"Active(file): %8lu kB\n"
176+
"Inactive(file): %8lu kB\n"
168177
#ifdef CONFIG_HIGHMEM
169-
"HighTotal: %8lu kB\n"
170-
"HighFree: %8lu kB\n"
171-
"LowTotal: %8lu kB\n"
172-
"LowFree: %8lu kB\n"
178+
"HighTotal: %8lu kB\n"
179+
"HighFree: %8lu kB\n"
180+
"LowTotal: %8lu kB\n"
181+
"LowFree: %8lu kB\n"
173182
#endif
174-
"SwapTotal: %8lu kB\n"
175-
"SwapFree: %8lu kB\n"
176-
"Dirty: %8lu kB\n"
177-
"Writeback: %8lu kB\n"
178-
"AnonPages: %8lu kB\n"
179-
"Mapped: %8lu kB\n"
180-
"Slab: %8lu kB\n"
181-
"SReclaimable: %8lu kB\n"
182-
"SUnreclaim: %8lu kB\n"
183-
"PageTables: %8lu kB\n"
183+
"SwapTotal: %8lu kB\n"
184+
"SwapFree: %8lu kB\n"
185+
"Dirty: %8lu kB\n"
186+
"Writeback: %8lu kB\n"
187+
"AnonPages: %8lu kB\n"
188+
"Mapped: %8lu kB\n"
189+
"Slab: %8lu kB\n"
190+
"SReclaimable: %8lu kB\n"
191+
"SUnreclaim: %8lu kB\n"
192+
"PageTables: %8lu kB\n"
184193
#ifdef CONFIG_QUICKLIST
185-
"Quicklists: %8lu kB\n"
194+
"Quicklists: %8lu kB\n"
186195
#endif
187-
"NFS_Unstable: %8lu kB\n"
188-
"Bounce: %8lu kB\n"
189-
"WritebackTmp: %8lu kB\n"
190-
"CommitLimit: %8lu kB\n"
191-
"Committed_AS: %8lu kB\n"
192-
"VmallocTotal: %8lu kB\n"
193-
"VmallocUsed: %8lu kB\n"
194-
"VmallocChunk: %8lu kB\n",
196+
"NFS_Unstable: %8lu kB\n"
197+
"Bounce: %8lu kB\n"
198+
"WritebackTmp: %8lu kB\n"
199+
"CommitLimit: %8lu kB\n"
200+
"Committed_AS: %8lu kB\n"
201+
"VmallocTotal: %8lu kB\n"
202+
"VmallocUsed: %8lu kB\n"
203+
"VmallocChunk: %8lu kB\n",
195204
K(i.totalram),
196205
K(i.freeram),
197206
K(i.bufferram),
198207
K(cached),
199208
K(total_swapcache_pages),
200-
K(global_page_state(NR_ACTIVE)),
201-
K(global_page_state(NR_INACTIVE)),
209+
K(pages[LRU_ACTIVE_ANON] + pages[LRU_ACTIVE_FILE]),
210+
K(pages[LRU_INACTIVE_ANON] + pages[LRU_INACTIVE_FILE]),
211+
K(pages[LRU_ACTIVE_ANON]),
212+
K(pages[LRU_INACTIVE_ANON]),
213+
K(pages[LRU_ACTIVE_FILE]),
214+
K(pages[LRU_INACTIVE_FILE]),
202215
#ifdef CONFIG_HIGHMEM
203216
K(i.totalhigh),
204217
K(i.freehigh),

‎fs/ramfs/file-nommu.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,12 +112,12 @@ int ramfs_nommu_expand_for_mapping(struct inode *inode, size_t newsize)
112112
goto add_error;
113113

114114
if (!pagevec_add(&lru_pvec, page))
115-
__pagevec_lru_add(&lru_pvec);
115+
__pagevec_lru_add_file(&lru_pvec);
116116

117117
unlock_page(page);
118118
}
119119

120-
pagevec_lru_add(&lru_pvec);
120+
pagevec_lru_add_file(&lru_pvec);
121121
return 0;
122122

123123
fsize_exceeded:

‎include/linux/backing-dev.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,8 @@ int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio);
175175
* BDI_CAP_READ_MAP: Can be mapped for reading
176176
* BDI_CAP_WRITE_MAP: Can be mapped for writing
177177
* BDI_CAP_EXEC_MAP: Can be mapped for execution
178+
*
179+
* BDI_CAP_SWAP_BACKED: Count shmem/tmpfs objects as swap-backed.
178180
*/
179181
#define BDI_CAP_NO_ACCT_DIRTY 0x00000001
180182
#define BDI_CAP_NO_WRITEBACK 0x00000002
@@ -184,6 +186,7 @@ int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio);
184186
#define BDI_CAP_WRITE_MAP 0x00000020
185187
#define BDI_CAP_EXEC_MAP 0x00000040
186188
#define BDI_CAP_NO_ACCT_WB 0x00000080
189+
#define BDI_CAP_SWAP_BACKED 0x00000100
187190

188191
#define BDI_CAP_VMFLAGS \
189192
(BDI_CAP_READ_MAP | BDI_CAP_WRITE_MAP | BDI_CAP_EXEC_MAP)
@@ -248,6 +251,11 @@ static inline bool bdi_cap_account_writeback(struct backing_dev_info *bdi)
248251
BDI_CAP_NO_WRITEBACK));
249252
}
250253

254+
static inline bool bdi_cap_swap_backed(struct backing_dev_info *bdi)
255+
{
256+
return bdi->capabilities & BDI_CAP_SWAP_BACKED;
257+
}
258+
251259
static inline bool mapping_cap_writeback_dirty(struct address_space *mapping)
252260
{
253261
return bdi_cap_writeback_dirty(mapping->backing_dev_info);
@@ -258,4 +266,9 @@ static inline bool mapping_cap_account_dirty(struct address_space *mapping)
258266
return bdi_cap_account_dirty(mapping->backing_dev_info);
259267
}
260268

269+
static inline bool mapping_cap_swap_backed(struct address_space *mapping)
270+
{
271+
return bdi_cap_swap_backed(mapping->backing_dev_info);
272+
}
273+
261274
#endif /* _LINUX_BACKING_DEV_H */

‎include/linux/memcontrol.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ extern unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
4444
unsigned long *scanned, int order,
4545
int mode, struct zone *z,
4646
struct mem_cgroup *mem_cont,
47-
int active);
47+
int active, int file);
4848
extern void mem_cgroup_out_of_memory(struct mem_cgroup *mem, gfp_t gfp_mask);
4949
int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem);
5050

‎include/linux/mm_inline.h‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
* page_is_file_cache - should the page be on a file LRU or anon LRU?
66
* @page: the page to test
77
*
8-
* Returns !0 if @page is page cache page backed by a regular filesystem,
8+
* Returns LRU_FILE if @page is page cache page backed by a regular filesystem,
99
* or 0 if @page is anonymous, tmpfs or otherwise ram or swap backed.
1010
* Used by functions that manipulate the LRU lists, to sort a page
1111
* onto the right LRU list.
@@ -20,7 +20,7 @@ static inline int page_is_file_cache(struct page *page)
2020
return 0;
2121

2222
/* The page is page cache backed by a normal filesystem. */
23-
return 1;
23+
return LRU_FILE;
2424
}
2525

2626
static inline void
@@ -38,39 +38,64 @@ del_page_from_lru_list(struct zone *zone, struct page *page, enum lru_list l)
3838
}
3939

4040
static inline void
41-
add_page_to_active_list(struct zone *zone, struct page *page)
41+
add_page_to_inactive_anon_list(struct zone *zone, struct page *page)
4242
{
43-
add_page_to_lru_list(zone, page, LRU_ACTIVE);
43+
add_page_to_lru_list(zone, page, LRU_INACTIVE_ANON);
4444
}
4545

4646
static inline void
47-
add_page_to_inactive_list(struct zone *zone, struct page *page)
47+
add_page_to_active_anon_list(struct zone *zone, struct page *page)
4848
{
49-
add_page_to_lru_list(zone, page, LRU_INACTIVE);
49+
add_page_to_lru_list(zone, page, LRU_ACTIVE_ANON);
5050
}
5151

5252
static inline void
53-
del_page_from_active_list(struct zone *zone, struct page *page)
53+
add_page_to_inactive_file_list(struct zone *zone, struct page *page)
5454
{
55-
del_page_from_lru_list(zone, page, LRU_ACTIVE);
55+
add_page_to_lru_list(zone, page, LRU_INACTIVE_FILE);
5656
}
5757

5858
static inline void
59-
del_page_from_inactive_list(struct zone *zone, struct page *page)
59+
add_page_to_active_file_list(struct zone *zone, struct page *page)
6060
{
61-
del_page_from_lru_list(zone, page, LRU_INACTIVE);
61+
add_page_to_lru_list(zone, page, LRU_ACTIVE_FILE);
62+
}
63+
64+
static inline void
65+
del_page_from_inactive_anon_list(struct zone *zone, struct page *page)
66+
{
67+
del_page_from_lru_list(zone, page, LRU_INACTIVE_ANON);
68+
}
69+
70+
static inline void
71+
del_page_from_active_anon_list(struct zone *zone, struct page *page)
72+
{
73+
del_page_from_lru_list(zone, page, LRU_ACTIVE_ANON);
74+
}
75+
76+
static inline void
77+
del_page_from_inactive_file_list(struct zone *zone, struct page *page)
78+
{
79+
del_page_from_lru_list(zone, page, LRU_INACTIVE_FILE);
80+
}
81+
82+
static inline void
83+
del_page_from_active_file_list(struct zone *zone, struct page *page)
84+
{
85+
del_page_from_lru_list(zone, page, LRU_INACTIVE_FILE);
6286
}
6387

6488
static inline void
6589
del_page_from_lru(struct zone *zone, struct page *page)
6690
{
67-
enum lru_list l = LRU_INACTIVE;
91+
enum lru_list l = LRU_BASE;
6892

6993
list_del(&page->lru);
7094
if (PageActive(page)) {
7195
__ClearPageActive(page);
72-
l = LRU_ACTIVE;
96+
l += LRU_ACTIVE;
7397
}
98+
l += page_is_file_cache(page);
7499
__dec_zone_state(zone, NR_LRU_BASE + l);
75100
}
76101

@@ -87,6 +112,7 @@ static inline enum lru_list page_lru(struct page *page)
87112

88113
if (PageActive(page))
89114
lru += LRU_ACTIVE;
115+
lru += page_is_file_cache(page);
90116

91117
return lru;
92118
}

0 commit comments

Comments
 (0)