Skip to content

Commit 7519134

Browse files
axboeaxboe
committed
io_uring/net: add iovec recycling
Right now the io_async_msghdr is recycled to avoid the overhead of allocating+freeing it for every request. But the iovec is not included, hence that will be allocated and freed for each transfer regardless. This commit enables recyling of the iovec between io_async_msghdr recycles. This avoids alloc+free for each one if an iovec is used, and on top of that, it extends the cache hot nature of msg to the iovec as well. Also enables KASAN for the iovec entries, so that reuse can be detected even while they are in the cache. The io_async_msghdr also shrinks from 376 -> 288 bytes, an 88 byte saving (or ~23% smaller), as the fast_iovec entry is dropped from 8 entries to a single entry. There's no point keeping a big fast iovec entry, if iovecs aren't being allocated and freed continually. Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent 9f8539f commit 7519134

File tree

2 files changed

+91
-53
lines changed

2 files changed

+91
-53
lines changed

‎io_uring/net.c‎

Lines changed: 84 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -115,15 +115,31 @@ static bool io_net_retry(struct socket *sock, int flags)
115115
return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
116116
}
117117

118+
static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg)
119+
{
120+
if (kmsg->free_iov) {
121+
kfree(kmsg->free_iov);
122+
kmsg->free_iov_nr = 0;
123+
kmsg->free_iov = NULL;
124+
}
125+
}
126+
118127
static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
119128
{
120129
struct io_async_msghdr *hdr = req->async_data;
130+
struct iovec *iov;
121131

122-
if (unlikely(issue_flags & IO_URING_F_UNLOCKED))
132+
/* can't recycle, ensure we free the iovec if we have one */
133+
if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) {
134+
io_netmsg_iovec_free(hdr);
123135
return;
136+
}
124137

125138
/* Let normal cleanup path reap it if we fail adding to the cache */
139+
iov = hdr->free_iov;
126140
if (io_alloc_cache_put(&req->ctx->netmsg_cache, &hdr->cache)) {
141+
if (iov)
142+
kasan_mempool_poison_object(iov);
127143
req->async_data = NULL;
128144
req->flags &= ~REQ_F_ASYNC_DATA;
129145
}
@@ -138,28 +154,56 @@ static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)
138154
entry = io_alloc_cache_get(&ctx->netmsg_cache);
139155
if (entry) {
140156
hdr = container_of(entry, struct io_async_msghdr, cache);
141-
hdr->free_iov = NULL;
157+
if (hdr->free_iov) {
158+
kasan_mempool_unpoison_object(hdr->free_iov,
159+
hdr->free_iov_nr * sizeof(struct iovec));
160+
req->flags |= REQ_F_NEED_CLEANUP;
161+
}
142162
req->flags |= REQ_F_ASYNC_DATA;
143163
req->async_data = hdr;
144164
return hdr;
145165
}
146166

147167
if (!io_alloc_async_data(req)) {
148168
hdr = req->async_data;
169+
hdr->free_iov_nr = 0;
149170
hdr->free_iov = NULL;
150171
return hdr;
151172
}
152173
return NULL;
153174
}
154175

176+
/* assign new iovec to kmsg, if we need to */
177+
static int io_net_vec_assign(struct io_kiocb *req, struct io_async_msghdr *kmsg,
178+
struct iovec *iov)
179+
{
180+
if (iov) {
181+
req->flags |= REQ_F_NEED_CLEANUP;
182+
kmsg->free_iov_nr = kmsg->msg.msg_iter.nr_segs;
183+
if (kmsg->free_iov)
184+
kfree(kmsg->free_iov);
185+
kmsg->free_iov = iov;
186+
}
187+
return 0;
188+
}
189+
155190
#ifdef CONFIG_COMPAT
156191
static int io_compat_msg_copy_hdr(struct io_kiocb *req,
157192
struct io_async_msghdr *iomsg,
158193
struct compat_msghdr *msg, int ddir)
159194
{
160195
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
161196
struct compat_iovec __user *uiov;
162-
int ret;
197+
struct iovec *iov;
198+
int ret, nr_segs;
199+
200+
if (iomsg->free_iov) {
201+
nr_segs = iomsg->free_iov_nr;
202+
iov = iomsg->free_iov;
203+
} else {
204+
iov = &iomsg->fast_iov;
205+
nr_segs = 1;
206+
}
163207

164208
if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg)))
165209
return -EFAULT;
@@ -168,9 +212,9 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,
168212
if (req->flags & REQ_F_BUFFER_SELECT) {
169213
compat_ssize_t clen;
170214

171-
iomsg->free_iov = NULL;
172215
if (msg->msg_iovlen == 0) {
173-
sr->len = 0;
216+
sr->len = iov->iov_len = 0;
217+
iov->iov_base = NULL;
174218
} else if (msg->msg_iovlen > 1) {
175219
return -EINVAL;
176220
} else {
@@ -186,22 +230,29 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,
186230
return 0;
187231
}
188232

189-
iomsg->free_iov = iomsg->fast_iov;
190233
ret = __import_iovec(ddir, (struct iovec __user *)uiov, msg->msg_iovlen,
191-
UIO_FASTIOV, &iomsg->free_iov,
192-
&iomsg->msg.msg_iter, true);
234+
nr_segs, &iov, &iomsg->msg.msg_iter, true);
193235
if (unlikely(ret < 0))
194236
return ret;
195237

196-
return 0;
238+
return io_net_vec_assign(req, iomsg, iov);
197239
}
198240
#endif
199241

200242
static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
201243
struct user_msghdr *msg, int ddir)
202244
{
203245
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
204-
int ret;
246+
struct iovec *iov;
247+
int ret, nr_segs;
248+
249+
if (iomsg->free_iov) {
250+
nr_segs = iomsg->free_iov_nr;
251+
iov = iomsg->free_iov;
252+
} else {
253+
iov = &iomsg->fast_iov;
254+
nr_segs = 1;
255+
}
205256

206257
if (!user_access_begin(sr->umsg, sizeof(*sr->umsg)))
207258
return -EFAULT;
@@ -217,20 +268,18 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
217268

218269
if (req->flags & REQ_F_BUFFER_SELECT) {
219270
if (msg->msg_iovlen == 0) {
220-
sr->len = iomsg->fast_iov[0].iov_len = 0;
221-
iomsg->fast_iov[0].iov_base = NULL;
222-
iomsg->free_iov = NULL;
271+
sr->len = iov->iov_len = 0;
272+
iov->iov_base = NULL;
223273
} else if (msg->msg_iovlen > 1) {
224274
ret = -EINVAL;
225275
goto ua_end;
226276
} else {
227277
/* we only need the length for provided buffers */
228278
if (!access_ok(&msg->msg_iov[0].iov_len, sizeof(__kernel_size_t)))
229279
goto ua_end;
230-
unsafe_get_user(iomsg->fast_iov[0].iov_len,
231-
&msg->msg_iov[0].iov_len, ua_end);
232-
sr->len = iomsg->fast_iov[0].iov_len;
233-
iomsg->free_iov = NULL;
280+
unsafe_get_user(iov->iov_len, &msg->msg_iov[0].iov_len,
281+
ua_end);
282+
sr->len = iov->iov_len;
234283
}
235284
ret = 0;
236285
ua_end:
@@ -239,13 +288,12 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
239288
}
240289

241290
user_access_end();
242-
iomsg->free_iov = iomsg->fast_iov;
243-
ret = __import_iovec(ddir, msg->msg_iov, msg->msg_iovlen, UIO_FASTIOV,
244-
&iomsg->free_iov, &iomsg->msg.msg_iter, false);
291+
ret = __import_iovec(ddir, msg->msg_iov, msg->msg_iovlen, nr_segs,
292+
&iov, &iomsg->msg.msg_iter, false);
245293
if (unlikely(ret < 0))
246294
return ret;
247295

248-
return 0;
296+
return io_net_vec_assign(req, iomsg, iov);
249297
}
250298

251299
static int io_sendmsg_copy_hdr(struct io_kiocb *req,
@@ -285,7 +333,7 @@ void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
285333
{
286334
struct io_async_msghdr *io = req->async_data;
287335

288-
kfree(io->free_iov);
336+
io_netmsg_iovec_free(io);
289337
}
290338

291339
static int io_send_setup(struct io_kiocb *req)
@@ -366,9 +414,6 @@ static void io_req_msg_cleanup(struct io_kiocb *req,
366414
unsigned int issue_flags)
367415
{
368416
req->flags &= ~REQ_F_NEED_CLEANUP;
369-
/* fast path, check for non-NULL to avoid function call */
370-
if (kmsg->free_iov)
371-
kfree(kmsg->free_iov);
372417
io_netmsg_recycle(req, issue_flags);
373418
}
374419

@@ -621,11 +666,6 @@ static inline void io_recv_prep_retry(struct io_kiocb *req,
621666
{
622667
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
623668

624-
if (kmsg->free_iov) {
625-
kfree(kmsg->free_iov);
626-
kmsg->free_iov = NULL;
627-
}
628-
629669
req->flags &= ~REQ_F_BL_EMPTY;
630670
sr->done_io = 0;
631671
sr->len = 0; /* get from the provided buffer */
@@ -941,14 +981,10 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
941981
void io_send_zc_cleanup(struct io_kiocb *req)
942982
{
943983
struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
944-
struct io_async_msghdr *io;
984+
struct io_async_msghdr *io = req->async_data;
945985

946-
if (req_has_async_data(req)) {
947-
io = req->async_data;
948-
/* might be ->fast_iov if *msg_copy_hdr failed */
949-
if (io->free_iov != io->fast_iov)
950-
kfree(io->free_iov);
951-
}
986+
if (req_has_async_data(req))
987+
io_netmsg_iovec_free(io);
952988
if (zc->notif) {
953989
io_notif_flush(zc->notif);
954990
zc->notif = NULL;
@@ -1170,8 +1206,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
11701206
*/
11711207
if (!(issue_flags & IO_URING_F_UNLOCKED)) {
11721208
io_notif_flush(zc->notif);
1173-
io_netmsg_recycle(req, issue_flags);
1174-
req->flags &= ~REQ_F_NEED_CLEANUP;
1209+
io_req_msg_cleanup(req, kmsg, 0);
11751210
}
11761211
io_req_set_res(req, ret, IORING_CQE_F_MORE);
11771212
return IOU_OK;
@@ -1221,13 +1256,7 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
12211256
ret = -EINTR;
12221257
req_set_fail(req);
12231258
}
1224-
/* fast path, check for non-NULL to avoid function call */
1225-
if (kmsg->free_iov) {
1226-
kfree(kmsg->free_iov);
1227-
kmsg->free_iov = NULL;
1228-
}
12291259

1230-
io_netmsg_recycle(req, issue_flags);
12311260
if (ret >= 0)
12321261
ret += sr->done_io;
12331262
else if (sr->done_io)
@@ -1239,7 +1268,7 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
12391268
*/
12401269
if (!(issue_flags & IO_URING_F_UNLOCKED)) {
12411270
io_notif_flush(sr->notif);
1242-
req->flags &= ~REQ_F_NEED_CLEANUP;
1271+
io_req_msg_cleanup(req, kmsg, 0);
12431272
}
12441273
io_req_set_res(req, ret, IORING_CQE_F_MORE);
12451274
return IOU_OK;
@@ -1483,6 +1512,14 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags)
14831512

14841513
void io_netmsg_cache_free(struct io_cache_entry *entry)
14851514
{
1486-
kfree(container_of(entry, struct io_async_msghdr, cache));
1515+
struct io_async_msghdr *kmsg;
1516+
1517+
kmsg = container_of(entry, struct io_async_msghdr, cache);
1518+
if (kmsg->free_iov) {
1519+
kasan_mempool_unpoison_object(kmsg->free_iov,
1520+
kmsg->free_iov_nr * sizeof(struct iovec));
1521+
io_netmsg_iovec_free(kmsg);
1522+
}
1523+
kfree(kmsg);
14871524
}
14881525
#endif

‎io_uring/net.h‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,18 @@
88
struct io_async_msghdr {
99
#if defined(CONFIG_NET)
1010
union {
11-
struct iovec fast_iov[UIO_FASTIOV];
11+
struct iovec fast_iov;
1212
struct {
13-
struct iovec fast_iov_one;
14-
__kernel_size_t controllen;
15-
int namelen;
16-
__kernel_size_t payloadlen;
13+
struct io_cache_entry cache;
14+
/* entry size of ->free_iov, if valid */
15+
int free_iov_nr;
1716
};
18-
struct io_cache_entry cache;
1917
};
2018
/* points to an allocated iov, if NULL we use fast_iov instead */
2119
struct iovec *free_iov;
20+
__kernel_size_t controllen;
21+
__kernel_size_t payloadlen;
22+
int namelen;
2223
struct sockaddr __user *uaddr;
2324
struct msghdr msg;
2425
struct sockaddr_storage addr;

0 commit comments

Comments
 (0)