Skip to content

Commit

Permalink
block: fix diskstats access
Browse files Browse the repository at this point in the history 
There are two variants of stat functions - ones prefixed with double
underbars which don't care about preemption and ones without which
disable preemption before manipulating per-cpu counters.  It's unclear
whether the underbarred ones assume that preemtion is disabled on
entry as some callers don't do that.

This patch unifies diskstats access by implementing disk_stat_lock()
and disk_stat_unlock() which take care of both RCU (for partition
access) and preemption (for per-cpu counter access).  diskstats access
should always be enclosed between the two functions.  As such, there's
no need for the versions which disables preemption.  They're removed
and double underbars ones are renamed to drop the underbars.  As an
extra argument is added, there's no danger of using the old version
unconverted.

disk_stat_lock() uses get_cpu() and returns the cpu index and all
diskstat functions which access per-cpu counters now has @cpu
argument to help RT.

This change adds RCU or preemption operations at some places but also
collapses several preemption ops into one at others.  Overall, the
performance difference should be negligible as all involved ops are
very lightweight per-cpu ones.

Signed-off-by: Tejun Heo <tj@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
  • Loading branch information
@htejun
htejun authored and Jens Axboe committed Oct 9, 2008
1 parent e71bf0d commit c995905
Show file tree
Hide file tree
Showing 13 changed files with 158 additions and 156 deletions.
52 changes: 27 additions & 25 deletions block/blk-core.c
View file
Expand Up @@ -56,25 +56,26 @@ static void drive_stat_acct(struct request *rq, int new_io)
{
struct hd_struct *part;
int rw = rq_data_dir(rq);
int cpu;

if (!blk_fs_request(rq) || !rq->rq_disk)
return;

rcu_read_lock();

cpu = disk_stat_lock();
part = disk_map_sector_rcu(rq->rq_disk, rq->sector);

if (!new_io)
__all_stat_inc(rq->rq_disk, part, merges[rw], rq->sector);
all_stat_inc(cpu, rq->rq_disk, part, merges[rw], rq->sector);
else {
disk_round_stats(rq->rq_disk);
disk_round_stats(cpu, rq->rq_disk);
rq->rq_disk->in_flight++;
if (part) {
part_round_stats(part);
part_round_stats(cpu, part);
part->in_flight++;
}
}

rcu_read_unlock();
disk_stat_unlock();
}

void blk_queue_congestion_threshold(struct request_queue *q)
Expand Down Expand Up @@ -997,33 +998,33 @@ static inline void add_request(struct request_queue *q, struct request *req)
* /proc/diskstats. This accounts immediately for all queue usage up to
* the current jiffies and restarts the counters again.
*/
void disk_round_stats(struct gendisk *disk)
void disk_round_stats(int cpu, struct gendisk *disk)
{
unsigned long now = jiffies;

if (now == disk->stamp)
return;

if (disk->in_flight) {
__disk_stat_add(disk, time_in_queue,
disk->in_flight * (now - disk->stamp));
__disk_stat_add(disk, io_ticks, (now - disk->stamp));
disk_stat_add(cpu, disk, time_in_queue,
disk->in_flight * (now - disk->stamp));
disk_stat_add(cpu, disk, io_ticks, (now - disk->stamp));
}
disk->stamp = now;
}
EXPORT_SYMBOL_GPL(disk_round_stats);

void part_round_stats(struct hd_struct *part)
void part_round_stats(int cpu, struct hd_struct *part)
{
unsigned long now = jiffies;

if (now == part->stamp)
return;

if (part->in_flight) {
__part_stat_add(part, time_in_queue,
part->in_flight * (now - part->stamp));
__part_stat_add(part, io_ticks, (now - part->stamp));
part_stat_add(cpu, part, time_in_queue,
part->in_flight * (now - part->stamp));
part_stat_add(cpu, part, io_ticks, (now - part->stamp));
}
part->stamp = now;
}
Expand Down Expand Up @@ -1563,12 +1564,13 @@ static int __end_that_request_first(struct request *req, int error,
if (blk_fs_request(req) && req->rq_disk) {
const int rw = rq_data_dir(req);
struct hd_struct *part;
int cpu;

rcu_read_lock();
cpu = disk_stat_lock();
part = disk_map_sector_rcu(req->rq_disk, req->sector);
all_stat_add(req->rq_disk, part, sectors[rw],
nr_bytes >> 9, req->sector);
rcu_read_unlock();
all_stat_add(cpu, req->rq_disk, part, sectors[rw],
nr_bytes >> 9, req->sector);
disk_stat_unlock();
}

total_bytes = bio_nbytes = 0;
Expand Down Expand Up @@ -1753,21 +1755,21 @@ static void end_that_request_last(struct request *req, int error)
unsigned long duration = jiffies - req->start_time;
const int rw = rq_data_dir(req);
struct hd_struct *part;
int cpu;

rcu_read_lock();

cpu = disk_stat_lock();
part = disk_map_sector_rcu(disk, req->sector);

__all_stat_inc(disk, part, ios[rw], req->sector);
__all_stat_add(disk, part, ticks[rw], duration, req->sector);
disk_round_stats(disk);
all_stat_inc(cpu, disk, part, ios[rw], req->sector);
all_stat_add(cpu, disk, part, ticks[rw], duration, req->sector);
disk_round_stats(cpu, disk);
disk->in_flight--;
if (part) {
part_round_stats(part);
part_round_stats(cpu, part);
part->in_flight--;
}

rcu_read_unlock();
disk_stat_unlock();
}

if (req->end_io)
Expand Down
11 changes: 6 additions & 5 deletions block/blk-merge.c
View file
Expand Up @@ -388,18 +388,19 @@ static int attempt_merge(struct request_queue *q, struct request *req,

if (req->rq_disk) {
struct hd_struct *part;
int cpu;

rcu_read_lock();

cpu = disk_stat_lock();
part = disk_map_sector_rcu(req->rq_disk, req->sector);
disk_round_stats(req->rq_disk);

disk_round_stats(cpu, req->rq_disk);
req->rq_disk->in_flight--;
if (part) {
part_round_stats(part);
part_round_stats(cpu, part);
part->in_flight--;
}

rcu_read_unlock();
disk_stat_unlock();
}

req->ioprio = ioprio_best(req->ioprio, next->ioprio);
Expand Down
20 changes: 11 additions & 9 deletions block/genhd.c
View file
Expand Up @@ -633,10 +633,11 @@ static ssize_t disk_stat_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct gendisk *disk = dev_to_disk(dev);
int cpu;

preempt_disable();
disk_round_stats(disk);
preempt_enable();
cpu = disk_stat_lock();
disk_round_stats(cpu, disk);
disk_stat_unlock();
return sprintf(buf,
"%8lu %8lu %8llu %8u "
"%8lu %8lu %8llu %8u "
Expand Down Expand Up @@ -749,6 +750,7 @@ static int diskstats_show(struct seq_file *seqf, void *v)
struct disk_part_iter piter;
struct hd_struct *hd;
char buf[BDEVNAME_SIZE];
int cpu;

/*
if (&gp->dev.kobj.entry == block_class.devices.next)
Expand All @@ -758,9 +760,9 @@ static int diskstats_show(struct seq_file *seqf, void *v)
"\n\n");
*/

preempt_disable();
disk_round_stats(gp);
preempt_enable();
cpu = disk_stat_lock();
disk_round_stats(cpu, gp);
disk_stat_unlock();
seq_printf(seqf, "%4d %4d %s %lu %lu %llu %u %lu %lu %llu %u %u %u %u\n",
MAJOR(disk_devt(gp)), MINOR(disk_devt(gp)),
disk_name(gp, 0, buf),
Expand All @@ -777,9 +779,9 @@ static int diskstats_show(struct seq_file *seqf, void *v)
/* now show all non-0 size partitions of it */
disk_part_iter_init(&piter, gp, 0);
while ((hd = disk_part_iter_next(&piter))) {
preempt_disable();
part_round_stats(hd);
preempt_enable();
cpu = disk_stat_lock();
part_round_stats(cpu, hd);
disk_stat_unlock();
seq_printf(seqf, "%4d %4d %s %lu %lu %llu "
"%u %lu %lu %llu %u %u %u %u\n",
MAJOR(part_devt(hd)), MINOR(part_devt(hd)),
Expand Down
15 changes: 8 additions & 7 deletions drivers/block/aoe/aoecmd.c
View file
Expand Up @@ -756,16 +756,17 @@ diskstats(struct gendisk *disk, struct bio *bio, ulong duration, sector_t sector
unsigned long n_sect = bio->bi_size >> 9;
const int rw = bio_data_dir(bio);
struct hd_struct *part;
int cpu;

rcu_read_lock();

cpu = disk_stat_lock();
part = disk_map_sector_rcu(disk, sector);
all_stat_inc(disk, part, ios[rw], sector);
all_stat_add(disk, part, ticks[rw], duration, sector);
all_stat_add(disk, part, sectors[rw], n_sect, sector);
all_stat_add(disk, part, io_ticks, duration, sector);

rcu_read_unlock();
all_stat_inc(cpu, disk, part, ios[rw], sector);
all_stat_add(cpu, disk, part, ticks[rw], duration, sector);
all_stat_add(cpu, disk, part, sectors[rw], n_sect, sector);
all_stat_add(cpu, disk, part, io_ticks, duration, sector);

disk_stat_unlock();
}

void
Expand Down
26 changes: 15 additions & 11 deletions drivers/md/dm.c
View file
Expand Up @@ -377,12 +377,13 @@ static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
static void start_io_acct(struct dm_io *io)
{
struct mapped_device *md = io->md;
int cpu;

io->start_time = jiffies;

preempt_disable();
disk_round_stats(dm_disk(md));
preempt_enable();
cpu = disk_stat_lock();
disk_round_stats(cpu, dm_disk(md));
disk_stat_unlock();
dm_disk(md)->in_flight = atomic_inc_return(&md->pending);
}

Expand All @@ -391,15 +392,15 @@ static int end_io_acct(struct dm_io *io)
struct mapped_device *md = io->md;
struct bio *bio = io->bio;
unsigned long duration = jiffies - io->start_time;
int pending;
int pending, cpu;
int rw = bio_data_dir(bio);

preempt_disable();
disk_round_stats(dm_disk(md));
preempt_enable();
dm_disk(md)->in_flight = pending = atomic_dec_return(&md->pending);
cpu = disk_stat_lock();
disk_round_stats(cpu, dm_disk(md));
disk_stat_add(cpu, dm_disk(md), ticks[rw], duration);
disk_stat_unlock();

disk_stat_add(dm_disk(md), ticks[rw], duration);
dm_disk(md)->in_flight = pending = atomic_dec_return(&md->pending);

return !pending;
}
Expand Down Expand Up @@ -885,6 +886,7 @@ static int dm_request(struct request_queue *q, struct bio *bio)
int r = -EIO;
int rw = bio_data_dir(bio);
struct mapped_device *md = q->queuedata;
int cpu;

/*
* There is no use in forwarding any barrier request since we can't
Expand All @@ -897,8 +899,10 @@ static int dm_request(struct request_queue *q, struct bio *bio)

down_read(&md->io_lock);

disk_stat_inc(dm_disk(md), ios[rw]);
disk_stat_add(dm_disk(md), sectors[rw], bio_sectors(bio));
cpu = disk_stat_lock();
disk_stat_inc(cpu, dm_disk(md), ios[rw]);
disk_stat_add(cpu, dm_disk(md), sectors[rw], bio_sectors(bio));
disk_stat_unlock();

/*
* If we're suspended we have to queue
Expand Down
7 changes: 5 additions & 2 deletions drivers/md/linear.c
View file
Expand Up @@ -318,14 +318,17 @@ static int linear_make_request (struct request_queue *q, struct bio *bio)
mddev_t *mddev = q->queuedata;
dev_info_t *tmp_dev;
sector_t block;
int cpu;

if (unlikely(bio_barrier(bio))) {
bio_endio(bio, -EOPNOTSUPP);
return 0;
}

disk_stat_inc(mddev->gendisk, ios[rw]);
disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
cpu = disk_stat_lock();
disk_stat_inc(cpu, mddev->gendisk, ios[rw]);
disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bio));
disk_stat_unlock();

tmp_dev = which_dev(mddev, bio->bi_sector);
block = bio->bi_sector >> 1;
Expand Down
7 changes: 5 additions & 2 deletions drivers/md/multipath.c
View file
Expand Up @@ -147,6 +147,7 @@ static int multipath_make_request (struct request_queue *q, struct bio * bio)
struct multipath_bh * mp_bh;
struct multipath_info *multipath;
const int rw = bio_data_dir(bio);
int cpu;

if (unlikely(bio_barrier(bio))) {
bio_endio(bio, -EOPNOTSUPP);
Expand All @@ -158,8 +159,10 @@ static int multipath_make_request (struct request_queue *q, struct bio * bio)
mp_bh->master_bio = bio;
mp_bh->mddev = mddev;

disk_stat_inc(mddev->gendisk, ios[rw]);
disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
cpu = disk_stat_lock();
disk_stat_inc(cpu, mddev->gendisk, ios[rw]);
disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bio));
disk_stat_unlock();

mp_bh->path = multipath_map(conf);
if (mp_bh->path < 0) {
Expand Down
7 changes: 5 additions & 2 deletions drivers/md/raid0.c
View file
Expand Up @@ -399,14 +399,17 @@ static int raid0_make_request (struct request_queue *q, struct bio *bio)
sector_t chunk;
sector_t block, rsect;
const int rw = bio_data_dir(bio);
int cpu;

if (unlikely(bio_barrier(bio))) {
bio_endio(bio, -EOPNOTSUPP);
return 0;
}

disk_stat_inc(mddev->gendisk, ios[rw]);
disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
cpu = disk_stat_lock();
disk_stat_inc(cpu, mddev->gendisk, ios[rw]);
disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bio));
disk_stat_unlock();

chunk_size = mddev->chunk_size >> 10;
chunk_sects = mddev->chunk_size >> 9;
Expand Down
8 changes: 5 additions & 3 deletions drivers/md/raid1.c
View file
Expand Up @@ -779,7 +779,7 @@ static int make_request(struct request_queue *q, struct bio * bio)
struct page **behind_pages = NULL;
const int rw = bio_data_dir(bio);
const int do_sync = bio_sync(bio);
int do_barriers;
int cpu, do_barriers;
mdk_rdev_t *blocked_rdev;

/*
Expand All @@ -804,8 +804,10 @@ static int make_request(struct request_queue *q, struct bio * bio)

bitmap = mddev->bitmap;

disk_stat_inc(mddev->gendisk, ios[rw]);
disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
cpu = disk_stat_lock();
disk_stat_inc(cpu, mddev->gendisk, ios[rw]);
disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bio));
disk_stat_unlock();

/*
* make_request() can abort the operation when READA is being
Expand Down
7 changes: 5 additions & 2 deletions drivers/md/raid10.c
View file
Expand Up @@ -789,6 +789,7 @@ static int make_request(struct request_queue *q, struct bio * bio)
mirror_info_t *mirror;
r10bio_t *r10_bio;
struct bio *read_bio;
int cpu;
int i;
int chunk_sects = conf->chunk_mask + 1;
const int rw = bio_data_dir(bio);
Expand Down Expand Up @@ -843,8 +844,10 @@ static int make_request(struct request_queue *q, struct bio * bio)
*/
wait_barrier(conf);

disk_stat_inc(mddev->gendisk, ios[rw]);
disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
cpu = disk_stat_lock();
disk_stat_inc(cpu, mddev->gendisk, ios[rw]);
disk_stat_add(cpu, mddev->gendisk, sectors[rw], bio_sectors(bio));
disk_stat_unlock();

r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

Expand Down

0 comments on commit c995905

Please sign in to comment.