Skip to content

Commit

Permalink
brd: convert to folios
Browse files Browse the repository at this point in the history 
Convert the driver to work on folios instead of pages.

Signed-off-by: Hannes Reinecke <hare@suse.de>
  • Loading branch information
@hreinecke @intel-lab-lkp
hreinecke authored and intel-lab-lkp committed Mar 6, 2023
1 parent db7183e commit b29fb98
Showing 1 changed file with 85 additions and 86 deletions.
171 changes: 85 additions & 86 deletions drivers/block/brd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
#include <linux/uaccess.h>

/*
* Each block ramdisk device has a radix_tree brd_pages of pages that stores
* the pages containing the block device's contents. A brd page's ->index is
* Each block ramdisk device has a radix_tree brd_folios of folios that stores
* the folios containing the block device's contents. A brd folio's ->index is
* its offset in PAGE_SIZE units. This is similar to, but in no way connected
* with, the kernel's pagecache or buffer cache (which sit above our block
* device).
Expand All @@ -40,25 +40,25 @@ struct brd_device {
struct list_head brd_list;

/*
* Backing store of pages and lock to protect it. This is the contents
* of the block device.
* Backing store of folios and lock to protect it.
* This is the contents of the block device.
*/
spinlock_t brd_lock;
struct radix_tree_root brd_pages;
u64 brd_nr_pages;
struct radix_tree_root brd_folios;
u64 brd_nr_folios;
};

/*
* Look up and return a brd's page for a given sector.
* Look up and return a brd's folio for a given sector.
*/
static struct page *brd_lookup_page(struct brd_device *brd, sector_t sector)
static struct folio *brd_lookup_folio(struct brd_device *brd, sector_t sector)
{
pgoff_t idx;
struct page *page;
struct folio *folio;

/*
* The page lifetime is protected by the fact that we have opened the
* device node -- brd pages will never be deleted under us, so we
* The folio lifetime is protected by the fact that we have opened the
* device node -- brd folios will never be deleted under us, so we
* don't need any further locking or refcounting.
*
* This is strictly true for the radix-tree nodes as well (ie. we
Expand All @@ -68,49 +68,49 @@ static struct page *brd_lookup_page(struct brd_device *brd, sector_t sector)
* here, only deletes).
*/
rcu_read_lock();
idx = sector >> PAGE_SECTORS_SHIFT; /* sector to page index */
page = radix_tree_lookup(&brd->brd_pages, idx);
idx = sector >> PAGE_SECTORS_SHIFT; /* sector to folio index */
folio = radix_tree_lookup(&brd->brd_folios, idx);
rcu_read_unlock();

BUG_ON(page && page->index != idx);
BUG_ON(folio && folio->index != idx);

return page;
return folio;
}

/*
* Insert a new page for a given sector, if one does not already exist.
* Insert a new folio for a given sector, if one does not already exist.
*/
static int brd_insert_page(struct brd_device *brd, sector_t sector, gfp_t gfp)
static int brd_insert_folio(struct brd_device *brd, sector_t sector, gfp_t gfp)
{
pgoff_t idx;
struct page *page;
struct folio *folio;
int ret = 0;

page = brd_lookup_page(brd, sector);
if (page)
folio = brd_lookup_folio(brd, sector);
if (folio)
return 0;

page = alloc_page(gfp | __GFP_ZERO | __GFP_HIGHMEM);
if (!page)
folio = folio_alloc(gfp | __GFP_ZERO, 0);
if (!folio)
return -ENOMEM;

if (radix_tree_maybe_preload(gfp)) {
__free_page(page);
folio_put(folio);
return -ENOMEM;
}

spin_lock(&brd->brd_lock);
idx = sector >> PAGE_SECTORS_SHIFT;
page->index = idx;
if (radix_tree_insert(&brd->brd_pages, idx, page)) {
__free_page(page);
page = radix_tree_lookup(&brd->brd_pages, idx);
if (!page)
folio->index = idx;
if (radix_tree_insert(&brd->brd_folios, idx, folio)) {
folio_put(folio);
folio = radix_tree_lookup(&brd->brd_folios, idx);
if (!folio)
ret = -ENOMEM;
else if (page->index != idx)
else if (folio->index != idx)
ret = -EIO;
} else {
brd->brd_nr_pages++;
brd->brd_nr_folios++;
}
spin_unlock(&brd->brd_lock);

Expand All @@ -119,30 +119,30 @@ static int brd_insert_page(struct brd_device *brd, sector_t sector, gfp_t gfp)
}

/*
* Free all backing store pages and radix tree. This must only be called when
* Free all backing store folios and radix tree. This must only be called when
* there are no other users of the device.
*/
#define FREE_BATCH 16
static void brd_free_pages(struct brd_device *brd)
static void brd_free_folios(struct brd_device *brd)
{
unsigned long pos = 0;
struct page *pages[FREE_BATCH];
int nr_pages;
struct folio *folios[FREE_BATCH];
int nr_folios;

do {
int i;

nr_pages = radix_tree_gang_lookup(&brd->brd_pages,
(void **)pages, pos, FREE_BATCH);
nr_folios = radix_tree_gang_lookup(&brd->brd_folios,
(void **)folios, pos, FREE_BATCH);

for (i = 0; i < nr_pages; i++) {
for (i = 0; i < nr_folios; i++) {
void *ret;

BUG_ON(pages[i]->index < pos);
pos = pages[i]->index;
ret = radix_tree_delete(&brd->brd_pages, pos);
BUG_ON(!ret || ret != pages[i]);
__free_page(pages[i]);
BUG_ON(folios[i]->index < pos);
pos = folios[i]->index;
ret = radix_tree_delete(&brd->brd_folios, pos);
BUG_ON(!ret || ret != folios[i]);
folio_put(folios[i]);
}

pos++;
Expand All @@ -155,10 +155,10 @@ static void brd_free_pages(struct brd_device *brd)

/*
* This assumes radix_tree_gang_lookup always returns as
* many pages as possible. If the radix-tree code changes,
* many folios as possible. If the radix-tree code changes,
* so will this have to.
*/
} while (nr_pages == FREE_BATCH);
} while (nr_folios == FREE_BATCH);
}

/*
Expand All @@ -172,12 +172,12 @@ static int copy_to_brd_setup(struct brd_device *brd, sector_t sector, size_t n,
int ret;

copy = min_t(size_t, n, PAGE_SIZE - offset);
ret = brd_insert_page(brd, sector, gfp);
ret = brd_insert_folio(brd, sector, gfp);
if (ret)
return ret;
if (copy < n) {
sector += copy >> SECTOR_SHIFT;
ret = brd_insert_page(brd, sector, gfp);
ret = brd_insert_folio(brd, sector, gfp);
}
return ret;
}
Expand All @@ -188,29 +188,29 @@ static int copy_to_brd_setup(struct brd_device *brd, sector_t sector, size_t n,
static void copy_to_brd(struct brd_device *brd, const void *src,
sector_t sector, size_t n)
{
struct page *page;
struct folio *folio;
void *dst;
unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
size_t copy;

copy = min_t(size_t, n, PAGE_SIZE - offset);
page = brd_lookup_page(brd, sector);
BUG_ON(!page);
folio = brd_lookup_folio(brd, sector);
BUG_ON(!folio);

dst = kmap_atomic(page);
memcpy(dst + offset, src, copy);
kunmap_atomic(dst);
dst = kmap_local_folio(folio, offset);
memcpy(dst, src, copy);
kunmap_local(dst);

if (copy < n) {
src += copy;
sector += copy >> SECTOR_SHIFT;
copy = n - copy;
page = brd_lookup_page(brd, sector);
BUG_ON(!page);
folio = brd_lookup_folio(brd, sector);
BUG_ON(!folio);

dst = kmap_atomic(page);
dst = kmap_local_folio(folio, 0);
memcpy(dst, src, copy);
kunmap_atomic(dst);
kunmap_local(dst);
}
}

Expand All @@ -220,38 +220,38 @@ static void copy_to_brd(struct brd_device *brd, const void *src,
static void copy_from_brd(void *dst, struct brd_device *brd,
sector_t sector, size_t n)
{
struct page *page;
struct folio *folio;
void *src;
unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
size_t copy;

copy = min_t(size_t, n, PAGE_SIZE - offset);
page = brd_lookup_page(brd, sector);
if (page) {
src = kmap_atomic(page);
memcpy(dst, src + offset, copy);
kunmap_atomic(src);
folio = brd_lookup_folio(brd, sector);
if (folio) {
src = kmap_local_folio(folio, offset);
memcpy(dst, src, copy);
kunmap_local(src);
} else
memset(dst, 0, copy);

if (copy < n) {
dst += copy;
sector += copy >> SECTOR_SHIFT;
copy = n - copy;
page = brd_lookup_page(brd, sector);
if (page) {
src = kmap_atomic(page);
folio = brd_lookup_folio(brd, sector);
if (folio) {
src = kmap_local_folio(folio, 0);
memcpy(dst, src, copy);
kunmap_atomic(src);
kunmap_local(src);
} else
memset(dst, 0, copy);
}
}

/*
* Process a single bvec of a bio.
* Process a single folio of a bio.
*/
static int brd_do_bvec(struct brd_device *brd, struct page *page,
static int brd_do_folio(struct brd_device *brd, struct folio *folio,
unsigned int len, unsigned int off, blk_opf_t opf,
sector_t sector)
{
Expand All @@ -261,7 +261,7 @@ static int brd_do_bvec(struct brd_device *brd, struct page *page,
if (op_is_write(opf)) {
/*
* Must use NOIO because we don't want to recurse back into the
* block or filesystem layers from page reclaim.
* block or filesystem layers from folio reclaim.
*/
gfp_t gfp = opf & REQ_NOWAIT ? GFP_NOWAIT : GFP_NOIO;

Expand All @@ -270,15 +270,15 @@ static int brd_do_bvec(struct brd_device *brd, struct page *page,
goto out;
}

mem = kmap_atomic(page);
mem = kmap_local_folio(folio, off);
if (!op_is_write(opf)) {
copy_from_brd(mem + off, brd, sector, len);
flush_dcache_page(page);
copy_from_brd(mem, brd, sector, len);
flush_dcache_folio(folio);
} else {
flush_dcache_page(page);
copy_to_brd(brd, mem + off, sector, len);
flush_dcache_folio(folio);
copy_to_brd(brd, mem, sector, len);
}
kunmap_atomic(mem);
kunmap_local(mem);

out:
return err;
Expand All @@ -288,19 +288,18 @@ static void brd_submit_bio(struct bio *bio)
{
struct brd_device *brd = bio->bi_bdev->bd_disk->private_data;
sector_t sector = bio->bi_iter.bi_sector;
struct bio_vec bvec;
struct bvec_iter iter;
struct folio_iter iter;

bio_for_each_segment(bvec, bio, iter) {
unsigned int len = bvec.bv_len;
bio_for_each_folio_all(iter, bio) {
unsigned int len = iter.length;
int err;

/* Don't support un-aligned buffer */
WARN_ON_ONCE((bvec.bv_offset & (SECTOR_SIZE - 1)) ||
WARN_ON_ONCE((iter.offset & (SECTOR_SIZE - 1)) ||
(len & (SECTOR_SIZE - 1)));

err = brd_do_bvec(brd, bvec.bv_page, len, bvec.bv_offset,
bio->bi_opf, sector);
err = brd_do_folio(brd, iter.folio, len, iter.offset,
bio->bi_opf, sector);
if (err) {
if (err == -ENOMEM && bio->bi_opf & REQ_NOWAIT) {
bio_wouldblock_error(bio);
Expand Down Expand Up @@ -387,12 +386,12 @@ static int brd_alloc(int i)
list_add_tail(&brd->brd_list, &brd_devices);

spin_lock_init(&brd->brd_lock);
INIT_RADIX_TREE(&brd->brd_pages, GFP_ATOMIC);
INIT_RADIX_TREE(&brd->brd_folios, GFP_ATOMIC);

snprintf(buf, DISK_NAME_LEN, "ram%d", i);
if (!IS_ERR_OR_NULL(brd_debugfs_dir))
debugfs_create_u64(buf, 0444, brd_debugfs_dir,
&brd->brd_nr_pages);
&brd->brd_nr_folios);

disk = brd->brd_disk = blk_alloc_disk(NUMA_NO_NODE);
if (!disk)
Expand Down Expand Up @@ -447,7 +446,7 @@ static void brd_cleanup(void)
list_for_each_entry_safe(brd, next, &brd_devices, brd_list) {
del_gendisk(brd->brd_disk);
put_disk(brd->brd_disk);
brd_free_pages(brd);
brd_free_folios(brd);
list_del(&brd->brd_list);
kfree(brd);
}
Expand Down Expand Up @@ -478,7 +477,7 @@ static int __init brd_init(void)

brd_check_and_reset_par();

brd_debugfs_dir = debugfs_create_dir("ramdisk_pages", NULL);
brd_debugfs_dir = debugfs_create_dir("ramdisk_folios", NULL);

for (i = 0; i < rd_nr; i++) {
err = brd_alloc(i);
Expand Down

0 comments on commit b29fb98

Please sign in to comment.