block: don't hide inode from block_device users

Instead of tricks with struct bdev_inode, just openly place the inode
inside struct block_device. First, it allows us to inline I_BDEV, which
is simple but non-inline nature of it impacts performance. Also, make it
possible to get rid of ->bd_inode pointer and hooping with extra
indirection, the amount of which became a noticeable problem for the
block layer.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>

block/bdev.c

@@ -29,22 +29,6 @@
 #include "../fs/internal.h"
 #include "blk.h"
 
-struct bdev_inode {
-	struct block_device bdev;
-	struct inode vfs_inode;
-};
-
-static inline struct bdev_inode *BDEV_I(struct inode *inode)
-{
-	return container_of(inode, struct bdev_inode, vfs_inode);
-}
-
-struct block_device *I_BDEV(struct inode *inode)
-{
-	return &BDEV_I(inode)->bdev;
-}
-EXPORT_SYMBOL(I_BDEV);
-
 static void bdev_write_inode(struct block_device *bdev)
 {
 	struct inode *inode = bdev->bd_inode;
@@ -388,12 +372,13 @@ static struct kmem_cache * bdev_cachep __read_mostly;
 
 static struct inode *bdev_alloc_inode(struct super_block *sb)
 {
-	struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL);
+	struct block_device *bdev = kmem_cache_alloc(bdev_cachep, GFP_KERNEL);
 
-	if (!ei)
+	if (!bdev)
 		return NULL;
-	memset(&ei->bdev, 0, sizeof(ei->bdev));
-	return &ei->vfs_inode;
+	memset(bdev, 0, sizeof(*bdev));
+	inode_init_once(&bdev->inode);
+	return &bdev->inode;
 }
 
 static void bdev_free_inode(struct inode *inode)
@@ -412,14 +397,7 @@ static void bdev_free_inode(struct inode *inode)
 	if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
 		blk_free_ext_minor(MINOR(bdev->bd_dev));
 
-	kmem_cache_free(bdev_cachep, BDEV_I(inode));
-}
-
-static void init_once(void *data)
-{
-	struct bdev_inode *ei = data;
-
-	inode_init_once(&ei->vfs_inode);
+	kmem_cache_free(bdev_cachep, bdev);
 }
 
 static void bdev_evict_inode(struct inode *inode)
@@ -461,10 +439,10 @@ void __init bdev_cache_init(void)
 	int err;
 	static struct vfsmount *bd_mnt;
 
-	bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
-			0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
-				SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC),
-			init_once);
+	bdev_cachep = kmem_cache_create("bdev_cache",
+			sizeof(struct block_device), 0,
+			(SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
+				SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC), NULL);
 	err = register_filesystem(&bd_type);
 	if (err)
 		panic("Cannot register bdev pseudo-fs");
@@ -743,7 +721,7 @@ struct block_device *blkdev_get_no_open(dev_t dev)
 	}
 
 	/* switch from the inode reference to a device mode one: */
-	bdev = &BDEV_I(inode)->bdev;
+	bdev = I_BDEV(inode);
 	if (!kobject_get_unless_zero(&bdev->bd_device.kobj))
 		bdev = NULL;
 	iput(inode);

block/fops.c

@@ -17,11 +17,6 @@
 #include <linux/fs.h>
 #include "blk.h"
 
-static inline struct inode *bdev_file_inode(struct file *file)
-{
-	return file->f_mapping->host;
-}
-
 static int blkdev_get_block(struct inode *inode, sector_t iblock,
 		struct buffer_head *bh, int create)
 {
@@ -390,7 +385,8 @@ const struct address_space_operations def_blk_aops = {
  */
 static loff_t blkdev_llseek(struct file *file, loff_t offset, int whence)
 {
-	struct inode *bd_inode = bdev_file_inode(file);
+	struct block_device *bdev = file->private_data;
+	struct inode *bd_inode = &bdev->inode;
 	loff_t retval;
 
 	inode_lock(bd_inode);
@@ -446,7 +442,7 @@ static int blkdev_open(struct inode *inode, struct file *filp)
 		return PTR_ERR(bdev);
 
 	filp->private_data = bdev;
-	filp->f_mapping = bdev->bd_inode->i_mapping;
+	filp->f_mapping = bdev->inode.i_mapping;
 	filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
 	return 0;
 }
@@ -486,7 +482,7 @@ static long block_ioctl(struct file *file, unsigned cmd, unsigned long arg)
 static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
 	struct block_device *bdev = iocb->ki_filp->private_data;
-	struct inode *bd_inode = bdev->bd_inode;
+	struct inode *bd_inode = &bdev->inode;
 	loff_t size = i_size_read(bd_inode);
 	struct blk_plug plug;
 	size_t shorted = 0;
@@ -525,7 +521,7 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
 static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
 	struct block_device *bdev = iocb->ki_filp->private_data;
-	loff_t size = i_size_read(bdev->bd_inode);
+	loff_t size = i_size_read(&bdev->inode);
 	loff_t pos = iocb->ki_pos;
 	size_t shorted = 0;
 	ssize_t ret;
@@ -551,8 +547,8 @@ static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
 static long blkdev_fallocate(struct file *file, int mode, loff_t start,
 			     loff_t len)
 {
-	struct inode *inode = bdev_file_inode(file);
-	struct block_device *bdev = I_BDEV(inode);
+	struct block_device *bdev = file->private_data;
+	struct inode *inode = &bdev->inode;
 	loff_t end = start + len - 1;
 	loff_t isize;
 	int error;
@@ -562,7 +558,7 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start,
 		return -EOPNOTSUPP;
 
 	/* Don't go off the end of the device. */
-	isize = i_size_read(bdev->bd_inode);
+	isize = i_size_read(inode);
 	if (start >= isize)
 		return -EINVAL;
 	if (end >= isize) {

include/linux/blk_types.h

@@ -49,6 +49,7 @@ struct block_device {
 #ifdef CONFIG_FAIL_MAKE_REQUEST
 	bool			bd_make_it_fail;
 #endif
+	struct inode		inode;
 } __randomize_layout;
 
 #define bdev_whole(_bdev) \

include/linux/blkdev.h

@@ -1617,7 +1617,7 @@ static inline unsigned int blksize_bits(unsigned int size)
 
 static inline unsigned int block_size(struct block_device *bdev)
 {
-	return 1 << bdev->bd_inode->i_blkbits;
+	return 1 << bdev->inode.i_blkbits;
 }
 
 int kblockd_schedule_work(struct work_struct *work);
@@ -1984,10 +1984,14 @@ void blkdev_put_no_open(struct block_device *bdev);
 
 struct block_device *bdev_alloc(struct gendisk *disk, u8 partno);
 void bdev_add(struct block_device *bdev, dev_t dev);
-struct block_device *I_BDEV(struct inode *inode);
 int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
 		loff_t lend);
 
+static inline struct block_device *I_BDEV(struct inode *inode)
+{
+	return container_of(inode, struct block_device, inode);
+}
+
 #ifdef CONFIG_BLOCK
 void invalidate_bdev(struct block_device *bdev);
 int sync_blockdev(struct block_device *bdev);