Skip to content

Commit

Permalink
dax: Use percpu rwsem for dax_{read,write}_lock()
Browse files Browse the repository at this point in the history 
In order to introduce dax holder registration, we need a write lock for
dax.  Change the current lock to percpu_rw_semaphore and introduce a
dax write lock for registration.

Signed-off-by: Shiyang Ruan <ruansy.fnst@fujitsu.com>
  • Loading branch information
@irides @intel-lab-lkp
irides authored and intel-lab-lkp committed Oct 31, 2021
1 parent 2f111a6 commit c300bbf
Show file tree
Hide file tree
Showing 5 changed files with 55 additions and 49 deletions.
11 changes: 5 additions & 6 deletions drivers/dax/device.c
View file
Expand Up @@ -198,15 +198,14 @@ static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf,
struct file *filp = vmf->vma->vm_file;
unsigned long fault_size;
vm_fault_t rc = VM_FAULT_SIGBUS;
int id;
pfn_t pfn;
struct dev_dax *dev_dax = filp->private_data;

dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm,
(vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read",
vmf->vma->vm_start, vmf->vma->vm_end, pe_size);

id = dax_read_lock();
dax_read_lock(dev_dax->dax_dev);
switch (pe_size) {
case PE_SIZE_PTE:
fault_size = PAGE_SIZE;
Expand Down Expand Up @@ -246,7 +245,7 @@ static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf,
page->index = pgoff + i;
}
}
dax_read_unlock(id);
dax_read_unlock(dev_dax->dax_dev);

return rc;
}
Expand Down Expand Up @@ -284,17 +283,17 @@ static const struct vm_operations_struct dax_vm_ops = {
static int dax_mmap(struct file *filp, struct vm_area_struct *vma)
{
struct dev_dax *dev_dax = filp->private_data;
int rc, id;
int rc;

dev_dbg(&dev_dax->dev, "trace\n");

/*
* We lock to check dax_dev liveness and will re-check at
* fault time.
*/
id = dax_read_lock();
dax_read_lock(dev_dax->dax_dev);
rc = check_vma(dev_dax, vma, __func__);
dax_read_unlock(id);
dax_read_unlock(dev_dax->dax_dev);
if (rc)
return rc;

Expand Down
51 changes: 31 additions & 20 deletions drivers/dax/super.c
View file
Expand Up @@ -31,12 +31,12 @@ struct dax_device {
struct cdev cdev;
const char *host;
void *private;
struct percpu_rw_semaphore rwsem;
unsigned long flags;
const struct dax_operations *ops;
};

static dev_t dax_devt;
DEFINE_STATIC_SRCU(dax_srcu);
static struct vfsmount *dax_mnt;
static DEFINE_IDA(dax_minor_ida);
static struct kmem_cache *dax_cache __read_mostly;
Expand All @@ -46,18 +46,28 @@ static struct super_block *dax_superblock __read_mostly;
static struct hlist_head dax_host_list[DAX_HASH_SIZE];
static DEFINE_SPINLOCK(dax_host_lock);

int dax_read_lock(void)
void dax_read_lock(struct dax_device *dax_dev)
{
return srcu_read_lock(&dax_srcu);
percpu_down_read(&dax_dev->rwsem);
}
EXPORT_SYMBOL_GPL(dax_read_lock);

void dax_read_unlock(int id)
void dax_read_unlock(struct dax_device *dax_dev)
{
srcu_read_unlock(&dax_srcu, id);
percpu_up_read(&dax_dev->rwsem);
}
EXPORT_SYMBOL_GPL(dax_read_unlock);

void dax_write_lock(struct dax_device *dax_dev)
{
percpu_down_write(&dax_dev->rwsem);
}

void dax_write_unlock(struct dax_device *dax_dev)
{
percpu_up_write(&dax_dev->rwsem);
}

static int dax_host_hash(const char *host)
{
return hashlen_hash(hashlen_string("DAX", host)) % DAX_HASH_SIZE;
Expand All @@ -70,26 +80,28 @@ static int dax_host_hash(const char *host)
static struct dax_device *dax_get_by_host(const char *host)
{
struct dax_device *dax_dev, *found = NULL;
int hash, id;
int hash;

if (!host)
return NULL;

hash = dax_host_hash(host);

id = dax_read_lock();
spin_lock(&dax_host_lock);
hlist_for_each_entry(dax_dev, &dax_host_list[hash], list) {
dax_read_lock(dax_dev);
if (!dax_alive(dax_dev)
|| strcmp(host, dax_dev->host) != 0)
|| strcmp(host, dax_dev->host) != 0) {
dax_read_unlock(dax_dev);
continue;
}

if (igrab(&dax_dev->inode))
found = dax_dev;
dax_read_unlock(dax_dev);
break;
}
spin_unlock(&dax_host_lock);
dax_read_unlock(id);

return found;
}
Expand Down Expand Up @@ -130,7 +142,7 @@ bool generic_fsdax_supported(struct dax_device *dax_dev,
pfn_t pfn, end_pfn;
sector_t last_page;
long len, len2;
int err, id;
int err;

if (blocksize != PAGE_SIZE) {
pr_info("%pg: error: unsupported blocksize for dax\n", bdev);
Expand All @@ -155,14 +167,14 @@ bool generic_fsdax_supported(struct dax_device *dax_dev,
return false;
}

id = dax_read_lock();
dax_read_lock(dax_dev);
len = dax_direct_access(dax_dev, pgoff, 1, &kaddr, &pfn);
len2 = dax_direct_access(dax_dev, pgoff_end, 1, &end_kaddr, &end_pfn);

if (len < 1 || len2 < 1) {
pr_info("%pg: error: dax access failed (%ld)\n",
bdev, len < 1 ? len : len2);
dax_read_unlock(id);
dax_read_unlock(dax_dev);
return false;
}

Expand Down Expand Up @@ -192,7 +204,7 @@ bool generic_fsdax_supported(struct dax_device *dax_dev,
put_dev_pagemap(end_pgmap);

}
dax_read_unlock(id);
dax_read_unlock(dax_dev);

if (!dax_enabled) {
pr_info("%pg: error: dax support not enabled\n", bdev);
Expand All @@ -206,16 +218,15 @@ bool dax_supported(struct dax_device *dax_dev, struct block_device *bdev,
int blocksize, sector_t start, sector_t len)
{
bool ret = false;
int id;

if (!dax_dev)
return false;

id = dax_read_lock();
dax_read_lock(dax_dev);
if (dax_alive(dax_dev) && dax_dev->ops->dax_supported)
ret = dax_dev->ops->dax_supported(dax_dev, bdev, blocksize,
start, len);
dax_read_unlock(id);
dax_read_unlock(dax_dev);
return ret;
}
EXPORT_SYMBOL_GPL(dax_supported);
Expand Down Expand Up @@ -410,7 +421,7 @@ EXPORT_SYMBOL_GPL(__set_dax_synchronous);

bool dax_alive(struct dax_device *dax_dev)
{
lockdep_assert_held(&dax_srcu);
lockdep_assert_held(&dax_dev->rwsem);
return test_bit(DAXDEV_ALIVE, &dax_dev->flags);
}
EXPORT_SYMBOL_GPL(dax_alive);
Expand All @@ -425,10 +436,9 @@ void kill_dax(struct dax_device *dax_dev)
{
if (!dax_dev)
return;

dax_write_lock(dax_dev);
clear_bit(DAXDEV_ALIVE, &dax_dev->flags);

synchronize_srcu(&dax_srcu);
dax_write_unlock(dax_dev);

spin_lock(&dax_host_lock);
hlist_del_init(&dax_dev->list);
Expand Down Expand Up @@ -590,6 +600,7 @@ struct dax_device *alloc_dax(void *private, const char *__host,
dax_add_host(dax_dev, host);
dax_dev->ops = ops;
dax_dev->private = private;
percpu_init_rwsem(&dax_dev->rwsem);
if (flags & DAXDEV_F_SYNC)
set_dax_synchronous(dax_dev);

Expand Down
7 changes: 3 additions & 4 deletions drivers/md/dm-writecache.c
View file
Expand Up @@ -260,7 +260,6 @@ static int persistent_memory_claim(struct dm_writecache *wc)
loff_t s;
long p, da;
pfn_t pfn;
int id;
struct page **pages;
sector_t offset;

Expand All @@ -284,7 +283,7 @@ static int persistent_memory_claim(struct dm_writecache *wc)
}
offset >>= PAGE_SHIFT - 9;

id = dax_read_lock();
dax_read_lock(wc->ssd_dev->dax_dev);

da = dax_direct_access(wc->ssd_dev->dax_dev, offset, p, &wc->memory_map, &pfn);
if (da < 0) {
Expand Down Expand Up @@ -334,7 +333,7 @@ static int persistent_memory_claim(struct dm_writecache *wc)
wc->memory_vmapped = true;
}

dax_read_unlock(id);
dax_read_unlock(wc->ssd_dev->dax_dev);

wc->memory_map += (size_t)wc->start_sector << SECTOR_SHIFT;
wc->memory_map_size -= (size_t)wc->start_sector << SECTOR_SHIFT;
Expand All @@ -343,7 +342,7 @@ static int persistent_memory_claim(struct dm_writecache *wc)
err3:
kvfree(pages);
err2:
dax_read_unlock(id);
dax_read_unlock(wc->ssd_dev->dax_dev);
err1:
return r;
}
Expand Down
26 changes: 12 additions & 14 deletions fs/dax.c
View file
Expand Up @@ -715,22 +715,21 @@ static int copy_cow_page_dax(struct block_device *bdev, struct dax_device *dax_d
void *vto, *kaddr;
pgoff_t pgoff;
long rc;
int id;

rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
if (rc)
return rc;

id = dax_read_lock();
dax_read_lock(dax_dev);
rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, NULL);
if (rc < 0) {
dax_read_unlock(id);
dax_read_unlock(dax_dev);
return rc;
}
vto = kmap_atomic(to);
copy_user_page(vto, (void __force *)kaddr, vaddr, to);
kunmap_atomic(vto);
dax_read_unlock(id);
dax_read_unlock(dax_dev);
return 0;
}

Expand Down Expand Up @@ -1015,13 +1014,13 @@ static int dax_iomap_pfn(const struct iomap *iomap, loff_t pos, size_t size,
{
const sector_t sector = dax_iomap_sector(iomap, pos);
pgoff_t pgoff;
int id, rc;
int rc;
long length;

rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
if (rc)
return rc;
id = dax_read_lock();
dax_read_lock(iomap->dax_dev);
length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
NULL, pfnp);
if (length < 0) {
Expand All @@ -1038,7 +1037,7 @@ static int dax_iomap_pfn(const struct iomap *iomap, loff_t pos, size_t size,
goto out;
rc = 0;
out:
dax_read_unlock(id);
dax_read_unlock(iomap->dax_dev);
return rc;
}

Expand Down Expand Up @@ -1130,7 +1129,7 @@ s64 dax_iomap_zero(loff_t pos, u64 length, struct iomap *iomap)
{
sector_t sector = iomap_sector(iomap, pos & PAGE_MASK);
pgoff_t pgoff;
long rc, id;
long rc;
void *kaddr;
bool page_aligned = false;
unsigned offset = offset_in_page(pos);
Expand All @@ -1144,22 +1143,22 @@ s64 dax_iomap_zero(loff_t pos, u64 length, struct iomap *iomap)
if (rc)
return rc;

id = dax_read_lock();
dax_read_lock(iomap->dax_dev);

if (page_aligned)
rc = dax_zero_page_range(iomap->dax_dev, pgoff, 1);
else
rc = dax_direct_access(iomap->dax_dev, pgoff, 1, &kaddr, NULL);
if (rc < 0) {
dax_read_unlock(id);
dax_read_unlock(iomap->dax_dev);
return rc;
}

if (!page_aligned) {
memset(kaddr + offset, 0, size);
dax_flush(iomap->dax_dev, kaddr + offset, size);
}
dax_read_unlock(id);
dax_read_unlock(iomap->dax_dev);
return size;
}

Expand All @@ -1174,7 +1173,6 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi,
loff_t end = pos + length, done = 0;
ssize_t ret = 0;
size_t xfer;
int id;

if (iov_iter_rw(iter) == READ) {
end = min(end, i_size_read(iomi->inode));
Expand All @@ -1199,7 +1197,7 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi,
(end - 1) >> PAGE_SHIFT);
}

id = dax_read_lock();
dax_read_lock(dax_dev);
while (pos < end) {
unsigned offset = pos & (PAGE_SIZE - 1);
const size_t size = ALIGN(length + offset, PAGE_SIZE);
Expand Down Expand Up @@ -1251,7 +1249,7 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi,
if (xfer < map_len)
break;
}
dax_read_unlock(id);
dax_read_unlock(dax_dev);

return done ? done : ret;
}
Expand Down
9 changes: 4 additions & 5 deletions include/linux/dax.h
View file
Expand Up @@ -177,15 +177,14 @@ static inline void dax_unlock_page(struct page *page, dax_entry_t cookie)
#endif

#if IS_ENABLED(CONFIG_DAX)
int dax_read_lock(void);
void dax_read_unlock(int id);
void dax_read_lock(struct dax_device *dax_dev);
void dax_read_unlock(struct dax_device *dax_dev);
#else
static inline int dax_read_lock(void)
static inline void dax_read_lock(struct dax_device *dax_dev)
{
return 0;
}

static inline void dax_read_unlock(int id)
static inline void dax_read_unlock(struct dax_device *dax_dev)
{
}
#endif /* CONFIG_DAX */
Expand Down

0 comments on commit c300bbf

Please sign in to comment.