-
Notifications
You must be signed in to change notification settings - Fork 751
/
fio.c
1845 lines (1651 loc) · 46.1 KB
/
fio.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright 2015, Joyent Inc.
*/
/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
/* All Rights Reserved */
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/signal.h>
#include <sys/cred.h>
#include <sys/user.h>
#include <sys/conf.h>
#include <sys/vfs.h>
#include <sys/vnode.h>
#include <sys/pathname.h>
#include <sys/file.h>
#include <sys/flock.h>
#include <sys/proc.h>
#include <sys/var.h>
#include <sys/cpuvar.h>
#include <sys/open.h>
#include <sys/cmn_err.h>
#include <sys/priocntl.h>
#include <sys/procset.h>
#include <sys/prsystm.h>
#include <sys/debug.h>
#include <sys/kmem.h>
#include <sys/atomic.h>
#include <sys/fcntl.h>
#include <sys/poll.h>
#include <sys/rctl.h>
#include <sys/port_impl.h>
#include <sys/dtrace.h>
#include <c2/audit.h>
#include <sys/nbmlock.h>
#ifdef DEBUG
static uint32_t afd_maxfd; /* # of entries in maximum allocated array */
static uint32_t afd_alloc; /* count of kmem_alloc()s */
static uint32_t afd_free; /* count of kmem_free()s */
static uint32_t afd_wait; /* count of waits on non-zero ref count */
#define MAXFD(x) (afd_maxfd = ((afd_maxfd >= (x))? afd_maxfd : (x)))
#define COUNT(x) atomic_inc_32(&x)
#else /* DEBUG */
#define MAXFD(x)
#define COUNT(x)
#endif /* DEBUG */
kmem_cache_t *file_cache;
static void port_close_fd(portfd_t *);
/*
* File descriptor allocation.
*
* fd_find(fip, minfd) finds the first available descriptor >= minfd.
* The most common case is open(2), in which minfd = 0, but we must also
* support fcntl(fd, F_DUPFD, minfd).
*
* The algorithm is as follows: we keep all file descriptors in an infix
* binary tree in which each node records the number of descriptors
* allocated in its right subtree, including itself. Starting at minfd,
* we ascend the tree until we find a non-fully allocated right subtree.
* We then descend that subtree in a binary search for the smallest fd.
* Finally, we ascend the tree again to increment the allocation count
* of every subtree containing the newly-allocated fd. Freeing an fd
* requires only the last step: we ascend the tree to decrement allocation
* counts. Each of these three steps (ascent to find non-full subtree,
* descent to find lowest fd, ascent to update allocation counts) is
* O(log n), thus the algorithm as a whole is O(log n).
*
* We don't implement the fd tree using the customary left/right/parent
* pointers, but instead take advantage of the glorious mathematics of
* full infix binary trees. For reference, here's an illustration of the
* logical structure of such a tree, rooted at 4 (binary 100), covering
* the range 1-7 (binary 001-111). Our canonical trees do not include
* fd 0; we'll deal with that later.
*
* 100
* / \
* / \
* 010 110
* / \ / \
* 001 011 101 111
*
* We make the following observations, all of which are easily proven by
* induction on the depth of the tree:
*
* (T1) The least-significant bit (LSB) of any node is equal to its level
* in the tree. In our example, nodes 001, 011, 101 and 111 are at
* level 0; nodes 010 and 110 are at level 1; and node 100 is at level 2.
*
* (T2) The child size (CSIZE) of node N -- that is, the total number of
* right-branch descendants in a child of node N, including itself -- is
* given by clearing all but the least significant bit of N. This
* follows immediately from (T1). Applying this rule to our example, we
* see that CSIZE(100) = 100, CSIZE(x10) = 10, and CSIZE(xx1) = 1.
*
* (T3) The nearest left ancestor (LPARENT) of node N -- that is, the nearest
* ancestor containing node N in its right child -- is given by clearing
* the LSB of N. For example, LPARENT(111) = 110 and LPARENT(110) = 100.
* Clearing the LSB of nodes 001, 010 or 100 yields zero, reflecting
* the fact that these are leftmost nodes. Note that this algorithm
* automatically skips generations as necessary. For example, the parent
* of node 101 is 110, which is a *right* ancestor (not what we want);
* but its grandparent is 100, which is a left ancestor. Clearing the LSB
* of 101 gets us to 100 directly, skipping right past the uninteresting
* generation (110).
*
* Note that since LPARENT clears the LSB, whereas CSIZE clears all *but*
* the LSB, we can express LPARENT() nicely in terms of CSIZE():
*
* LPARENT(N) = N - CSIZE(N)
*
* (T4) The nearest right ancestor (RPARENT) of node N is given by:
*
* RPARENT(N) = N + CSIZE(N)
*
* (T5) For every interior node, the children differ from their parent by
* CSIZE(parent) / 2. In our example, CSIZE(100) / 2 = 2 = 10 binary,
* and indeed, the children of 100 are 100 +/- 10 = 010 and 110.
*
* Next, we'll need a few two's-complement math tricks. Suppose a number,
* N, has the following form:
*
* N = xxxx10...0
*
* That is, the binary representation of N consists of some string of bits,
* then a 1, then all zeroes. This amounts to nothing more than saying that
* N has a least-significant bit, which is true for any N != 0. If we look
* at N and N - 1 together, we see that we can combine them in useful ways:
*
* N = xxxx10...0
* N - 1 = xxxx01...1
* ------------------------
* N & (N - 1) = xxxx000000
* N | (N - 1) = xxxx111111
* N ^ (N - 1) = 111111
*
* In particular, this suggests several easy ways to clear all but the LSB,
* which by (T2) is exactly what we need to determine CSIZE(N) = 10...0.
* We'll opt for this formulation:
*
* (C1) CSIZE(N) = (N - 1) ^ (N | (N - 1))
*
* Similarly, we have an easy way to determine LPARENT(N), which requires
* that we clear the LSB of N:
*
* (L1) LPARENT(N) = N & (N - 1)
*
* We note in the above relations that (N | (N - 1)) - N = CSIZE(N) - 1.
* When combined with (T4), this yields an easy way to compute RPARENT(N):
*
* (R1) RPARENT(N) = (N | (N - 1)) + 1
*
* Finally, to accommodate fd 0 we must adjust all of our results by +/-1 to
* move the fd range from [1, 2^n) to [0, 2^n - 1). This is straightforward,
* so there's no need to belabor the algebra; the revised relations become:
*
* (C1a) CSIZE(N) = N ^ (N | (N + 1))
*
* (L1a) LPARENT(N) = (N & (N + 1)) - 1
*
* (R1a) RPARENT(N) = N | (N + 1)
*
* This completes the mathematical framework. We now have all the tools
* we need to implement fd_find() and fd_reserve().
*
* fd_find(fip, minfd) finds the smallest available file descriptor >= minfd.
* It does not actually allocate the descriptor; that's done by fd_reserve().
* fd_find() proceeds in two steps:
*
* (1) Find the leftmost subtree that contains a descriptor >= minfd.
* We start at the right subtree rooted at minfd. If this subtree is
* not full -- if fip->fi_list[minfd].uf_alloc != CSIZE(minfd) -- then
* step 1 is done. Otherwise, we know that all fds in this subtree
* are taken, so we ascend to RPARENT(minfd) using (R1a). We repeat
* this process until we either find a candidate subtree or exceed
* fip->fi_nfiles. We use (C1a) to compute CSIZE().
*
* (2) Find the smallest fd in the subtree discovered by step 1.
* Starting at the root of this subtree, we descend to find the
* smallest available fd. Since the left children have the smaller
* fds, we will descend rightward only when the left child is full.
*
* We begin by comparing the number of allocated fds in the root
* to the number of allocated fds in its right child; if they differ
* by exactly CSIZE(child), we know the left subtree is full, so we
* descend right; that is, the right child becomes the search root.
* Otherwise we leave the root alone and start following the right
* child's left children. As fortune would have it, this is very
* simple computationally: by (T5), the right child of fd is just
* fd + size, where size = CSIZE(fd) / 2. Applying (T5) again,
* we find that the right child's left child is fd + size - (size / 2) =
* fd + (size / 2); *its* left child is fd + (size / 2) - (size / 4) =
* fd + (size / 4), and so on. In general, fd's right child's
* leftmost nth descendant is fd + (size >> n). Thus, to follow
* the right child's left descendants, we just halve the size in
* each iteration of the search.
*
* When we descend leftward, we must keep track of the number of fds
* that were allocated in all the right subtrees we rejected, so we
* know how many of the root fd's allocations are in the remaining
* (as yet unexplored) leftmost part of its right subtree. When we
* encounter a fully-allocated left child -- that is, when we find
* that fip->fi_list[fd].uf_alloc == ralloc + size -- we descend right
* (as described earlier), resetting ralloc to zero.
*
* fd_reserve(fip, fd, incr) either allocates or frees fd, depending
* on whether incr is 1 or -1. Starting at fd, fd_reserve() ascends
* the leftmost ancestors (see (T3)) and updates the allocation counts.
* At each step we use (L1a) to compute LPARENT(), the next left ancestor.
*
* flist_minsize() finds the minimal tree that still covers all
* used fds; as long as the allocation count of a root node is zero, we
* don't need that node or its right subtree.
*
* flist_nalloc() counts the number of allocated fds in the tree, by starting
* at the top of the tree and summing the right-subtree allocation counts as
* it descends leftwards.
*
* Note: we assume that flist_grow() will keep fip->fi_nfiles of the form
* 2^n - 1. This ensures that the fd trees are always full, which saves
* quite a bit of boundary checking.
*/
static int
fd_find(uf_info_t *fip, int minfd)
{
int size, ralloc, fd;
ASSERT(MUTEX_HELD(&fip->fi_lock));
ASSERT((fip->fi_nfiles & (fip->fi_nfiles + 1)) == 0);
for (fd = minfd; (uint_t)fd < fip->fi_nfiles; fd |= fd + 1) {
size = fd ^ (fd | (fd + 1));
if (fip->fi_list[fd].uf_alloc == size)
continue;
for (ralloc = 0, size >>= 1; size != 0; size >>= 1) {
ralloc += fip->fi_list[fd + size].uf_alloc;
if (fip->fi_list[fd].uf_alloc == ralloc + size) {
fd += size;
ralloc = 0;
}
}
return (fd);
}
return (-1);
}
static void
fd_reserve(uf_info_t *fip, int fd, int incr)
{
int pfd;
uf_entry_t *ufp = &fip->fi_list[fd];
ASSERT((uint_t)fd < fip->fi_nfiles);
ASSERT((ufp->uf_busy == 0 && incr == 1) ||
(ufp->uf_busy == 1 && incr == -1));
ASSERT(MUTEX_HELD(&ufp->uf_lock));
ASSERT(MUTEX_HELD(&fip->fi_lock));
for (pfd = fd; pfd >= 0; pfd = (pfd & (pfd + 1)) - 1)
fip->fi_list[pfd].uf_alloc += incr;
ufp->uf_busy += incr;
}
static int
flist_minsize(uf_info_t *fip)
{
int fd;
/*
* We'd like to ASSERT(MUTEX_HELD(&fip->fi_lock)), but we're called
* by flist_fork(), which relies on other mechanisms for mutual
* exclusion.
*/
ASSERT((fip->fi_nfiles & (fip->fi_nfiles + 1)) == 0);
for (fd = fip->fi_nfiles; fd != 0; fd >>= 1)
if (fip->fi_list[fd >> 1].uf_alloc != 0)
break;
return (fd);
}
static int
flist_nalloc(uf_info_t *fip)
{
int fd;
int nalloc = 0;
ASSERT(MUTEX_HELD(&fip->fi_lock));
ASSERT((fip->fi_nfiles & (fip->fi_nfiles + 1)) == 0);
for (fd = fip->fi_nfiles; fd != 0; fd >>= 1)
nalloc += fip->fi_list[fd >> 1].uf_alloc;
return (nalloc);
}
/*
* Increase size of the fi_list array to accommodate at least maxfd.
* We keep the size of the form 2^n - 1 for benefit of fd_find().
*/
static void
flist_grow(int maxfd)
{
uf_info_t *fip = P_FINFO(curproc);
int newcnt, oldcnt;
uf_entry_t *src, *dst, *newlist, *oldlist, *newend, *oldend;
uf_rlist_t *urp;
for (newcnt = 1; newcnt <= maxfd; newcnt = (newcnt << 1) | 1)
continue;
newlist = kmem_zalloc(newcnt * sizeof (uf_entry_t), KM_SLEEP);
mutex_enter(&fip->fi_lock);
oldcnt = fip->fi_nfiles;
if (newcnt <= oldcnt) {
mutex_exit(&fip->fi_lock);
kmem_free(newlist, newcnt * sizeof (uf_entry_t));
return;
}
ASSERT((newcnt & (newcnt + 1)) == 0);
oldlist = fip->fi_list;
oldend = oldlist + oldcnt;
newend = newlist + oldcnt; /* no need to lock beyond old end */
/*
* fi_list and fi_nfiles cannot change while any uf_lock is held,
* so we must grab all the old locks *and* the new locks up to oldcnt.
* (Locks beyond the end of oldcnt aren't visible until we store
* the new fi_nfiles, which is the last thing we do before dropping
* all the locks, so there's no need to acquire these locks).
* Holding the new locks is necessary because when fi_list changes
* to point to the new list, fi_nfiles won't have been stored yet.
* If we *didn't* hold the new locks, someone doing a UF_ENTER()
* could see the new fi_list, grab the new uf_lock, and then see
* fi_nfiles change while the lock is held -- in violation of
* UF_ENTER() semantics.
*/
for (src = oldlist; src < oldend; src++)
mutex_enter(&src->uf_lock);
for (dst = newlist; dst < newend; dst++)
mutex_enter(&dst->uf_lock);
for (src = oldlist, dst = newlist; src < oldend; src++, dst++) {
dst->uf_file = src->uf_file;
dst->uf_fpollinfo = src->uf_fpollinfo;
dst->uf_refcnt = src->uf_refcnt;
dst->uf_alloc = src->uf_alloc;
dst->uf_flag = src->uf_flag;
dst->uf_busy = src->uf_busy;
dst->uf_portfd = src->uf_portfd;
dst->uf_gen = src->uf_gen;
}
/*
* As soon as we store the new flist, future locking operations
* will use it. Therefore, we must ensure that all the state
* we've just established reaches global visibility before the
* new flist does.
*/
membar_producer();
fip->fi_list = newlist;
/*
* Routines like getf() make an optimistic check on the validity
* of the supplied file descriptor: if it's less than the current
* value of fi_nfiles -- examined without any locks -- then it's
* safe to attempt a UF_ENTER() on that fd (which is a valid
* assumption because fi_nfiles only increases). Therefore, it
* is critical that the new value of fi_nfiles not reach global
* visibility until after the new fi_list: if it happened the
* other way around, getf() could see the new fi_nfiles and attempt
* a UF_ENTER() on the old fi_list, which would write beyond its
* end if the fd exceeded the old fi_nfiles.
*/
membar_producer();
fip->fi_nfiles = newcnt;
/*
* The new state is consistent now, so we can drop all the locks.
*/
for (dst = newlist; dst < newend; dst++)
mutex_exit(&dst->uf_lock);
for (src = oldlist; src < oldend; src++) {
/*
* If any threads are blocked on the old cvs, wake them.
* This will force them to wake up, discover that fi_list
* has changed, and go back to sleep on the new cvs.
*/
cv_broadcast(&src->uf_wanted_cv);
cv_broadcast(&src->uf_closing_cv);
mutex_exit(&src->uf_lock);
}
mutex_exit(&fip->fi_lock);
/*
* Retire the old flist. We can't actually kmem_free() it now
* because someone may still have a pointer to it. Instead,
* we link it onto a list of retired flists. The new flist
* is at least double the size of the previous flist, so the
* total size of all retired flists will be less than the size
* of the current one (to prove, consider the sum of a geometric
* series in powers of 2). exit() frees the retired flists.
*/
urp = kmem_zalloc(sizeof (uf_rlist_t), KM_SLEEP);
urp->ur_list = oldlist;
urp->ur_nfiles = oldcnt;
mutex_enter(&fip->fi_lock);
urp->ur_next = fip->fi_rlist;
fip->fi_rlist = urp;
mutex_exit(&fip->fi_lock);
}
/*
* Utility functions for keeping track of the active file descriptors.
*/
void
clear_stale_fd() /* called from post_syscall() */
{
afd_t *afd = &curthread->t_activefd;
int i;
/* uninitialized is ok here, a_nfd is then zero */
for (i = 0; i < afd->a_nfd; i++) {
/* assert that this should not be necessary */
ASSERT(afd->a_fd[i] == -1);
afd->a_fd[i] = -1;
}
afd->a_stale = 0;
}
void
free_afd(afd_t *afd) /* called below and from thread_free() */
{
int i;
/* free the buffer if it was kmem_alloc()ed */
if (afd->a_nfd > sizeof (afd->a_buf) / sizeof (afd->a_buf[0])) {
COUNT(afd_free);
kmem_free(afd->a_fd, afd->a_nfd * sizeof (afd->a_fd[0]));
}
/* (re)initialize the structure */
afd->a_fd = &afd->a_buf[0];
afd->a_nfd = sizeof (afd->a_buf) / sizeof (afd->a_buf[0]);
afd->a_stale = 0;
for (i = 0; i < afd->a_nfd; i++)
afd->a_fd[i] = -1;
}
static void
set_active_fd(int fd)
{
afd_t *afd = &curthread->t_activefd;
int i;
int *old_fd;
int old_nfd;
int *new_fd;
int new_nfd;
if (afd->a_nfd == 0) { /* first time initialization */
ASSERT(fd == -1);
mutex_enter(&afd->a_fdlock);
free_afd(afd);
mutex_exit(&afd->a_fdlock);
}
/* insert fd into vacant slot, if any */
for (i = 0; i < afd->a_nfd; i++) {
if (afd->a_fd[i] == -1) {
afd->a_fd[i] = fd;
return;
}
}
/*
* Reallocate the a_fd[] array to add one more slot.
*/
ASSERT(fd == -1);
old_nfd = afd->a_nfd;
old_fd = afd->a_fd;
new_nfd = old_nfd + 1;
new_fd = kmem_alloc(new_nfd * sizeof (afd->a_fd[0]), KM_SLEEP);
MAXFD(new_nfd);
COUNT(afd_alloc);
mutex_enter(&afd->a_fdlock);
afd->a_fd = new_fd;
afd->a_nfd = new_nfd;
for (i = 0; i < old_nfd; i++)
afd->a_fd[i] = old_fd[i];
afd->a_fd[i] = fd;
mutex_exit(&afd->a_fdlock);
if (old_nfd > sizeof (afd->a_buf) / sizeof (afd->a_buf[0])) {
COUNT(afd_free);
kmem_free(old_fd, old_nfd * sizeof (afd->a_fd[0]));
}
}
void
clear_active_fd(int fd) /* called below and from aio.c */
{
afd_t *afd = &curthread->t_activefd;
int i;
for (i = 0; i < afd->a_nfd; i++) {
if (afd->a_fd[i] == fd) {
afd->a_fd[i] = -1;
break;
}
}
ASSERT(i < afd->a_nfd); /* not found is not ok */
}
/*
* Does this thread have this fd active?
*/
static int
is_active_fd(kthread_t *t, int fd)
{
afd_t *afd = &t->t_activefd;
int i;
ASSERT(t != curthread);
mutex_enter(&afd->a_fdlock);
/* uninitialized is ok here, a_nfd is then zero */
for (i = 0; i < afd->a_nfd; i++) {
if (afd->a_fd[i] == fd) {
mutex_exit(&afd->a_fdlock);
return (1);
}
}
mutex_exit(&afd->a_fdlock);
return (0);
}
/*
* Convert a user supplied file descriptor into a pointer to a file structure.
* Only task is to check range of the descriptor (soft resource limit was
* enforced at open time and shouldn't be checked here).
*/
file_t *
getf_gen(int fd, uf_entry_gen_t *genp)
{
uf_info_t *fip = P_FINFO(curproc);
uf_entry_t *ufp;
file_t *fp;
if ((uint_t)fd >= fip->fi_nfiles)
return (NULL);
/*
* Reserve a slot in the active fd array now so we can call
* set_active_fd(fd) for real below, while still inside UF_ENTER().
*/
set_active_fd(-1);
UF_ENTER(ufp, fip, fd);
if ((fp = ufp->uf_file) == NULL) {
UF_EXIT(ufp);
if (fd == fip->fi_badfd && fip->fi_action > 0)
tsignal(curthread, fip->fi_action);
return (NULL);
}
ufp->uf_refcnt++;
if (genp != NULL) {
*genp = ufp->uf_gen;
}
set_active_fd(fd); /* record the active file descriptor */
UF_EXIT(ufp);
return (fp);
}
file_t *
getf(int fd)
{
return (getf_gen(fd, NULL));
}
/*
* Close whatever file currently occupies the file descriptor slot
* and install the new file, usually NULL, in the file descriptor slot.
* The close must complete before we release the file descriptor slot.
* If newfp != NULL we only return an error if we can't allocate the
* slot so the caller knows that it needs to free the filep;
* in the other cases we return the error number from closef().
*/
int
closeandsetf(int fd, file_t *newfp)
{
proc_t *p = curproc;
uf_info_t *fip = P_FINFO(p);
uf_entry_t *ufp;
file_t *fp;
fpollinfo_t *fpip;
portfd_t *pfd;
int error;
if ((uint_t)fd >= fip->fi_nfiles) {
if (newfp == NULL)
return (EBADF);
flist_grow(fd);
}
if (newfp != NULL) {
/*
* If ufp is reserved but has no file pointer, it's in the
* transition between ufalloc() and setf(). We must wait
* for this transition to complete before assigning the
* new non-NULL file pointer.
*/
mutex_enter(&fip->fi_lock);
if (fd == fip->fi_badfd) {
mutex_exit(&fip->fi_lock);
if (fip->fi_action > 0)
tsignal(curthread, fip->fi_action);
return (EBADF);
}
UF_ENTER(ufp, fip, fd);
while (ufp->uf_busy && ufp->uf_file == NULL) {
mutex_exit(&fip->fi_lock);
cv_wait_stop(&ufp->uf_wanted_cv, &ufp->uf_lock, 250);
UF_EXIT(ufp);
mutex_enter(&fip->fi_lock);
UF_ENTER(ufp, fip, fd);
}
if ((fp = ufp->uf_file) == NULL) {
ASSERT(ufp->uf_fpollinfo == NULL);
ASSERT(ufp->uf_flag == 0);
fd_reserve(fip, fd, 1);
ufp->uf_file = newfp;
ufp->uf_gen++;
UF_EXIT(ufp);
mutex_exit(&fip->fi_lock);
return (0);
}
mutex_exit(&fip->fi_lock);
} else {
UF_ENTER(ufp, fip, fd);
if ((fp = ufp->uf_file) == NULL) {
UF_EXIT(ufp);
return (EBADF);
}
}
ASSERT(ufp->uf_busy);
ufp->uf_file = NULL;
ufp->uf_flag = 0;
/*
* If the file descriptor reference count is non-zero, then
* some other lwp in the process is performing system call
* activity on the file. To avoid blocking here for a long
* time (the other lwp might be in a long term sleep in its
* system call), we scan all other lwps in the process to
* find the ones with this fd as one of their active fds,
* set their a_stale flag, and set them running if they
* are in an interruptible sleep so they will emerge from
* their system calls immediately. post_syscall() will
* test the a_stale flag and set errno to EBADF.
*/
ASSERT(ufp->uf_refcnt == 0 || p->p_lwpcnt > 1);
if (ufp->uf_refcnt > 0) {
kthread_t *t;
/*
* We call sprlock_proc(p) to ensure that the thread
* list will not change while we are scanning it.
* To do this, we must drop ufp->uf_lock and then
* reacquire it (so we are not holding both p->p_lock
* and ufp->uf_lock at the same time). ufp->uf_lock
* must be held for is_active_fd() to be correct
* (set_active_fd() is called while holding ufp->uf_lock).
*
* This is a convoluted dance, but it is better than
* the old brute-force method of stopping every thread
* in the process by calling holdlwps(SHOLDFORK1).
*/
UF_EXIT(ufp);
COUNT(afd_wait);
mutex_enter(&p->p_lock);
sprlock_proc(p);
mutex_exit(&p->p_lock);
UF_ENTER(ufp, fip, fd);
ASSERT(ufp->uf_file == NULL);
if (ufp->uf_refcnt > 0) {
for (t = curthread->t_forw;
t != curthread;
t = t->t_forw) {
if (is_active_fd(t, fd)) {
thread_lock(t);
t->t_activefd.a_stale = 1;
t->t_post_sys = 1;
if (ISWAKEABLE(t))
setrun_locked(t);
thread_unlock(t);
}
}
}
UF_EXIT(ufp);
mutex_enter(&p->p_lock);
sprunlock(p);
UF_ENTER(ufp, fip, fd);
ASSERT(ufp->uf_file == NULL);
}
/*
* Wait for other lwps to stop using this file descriptor.
*/
while (ufp->uf_refcnt > 0) {
cv_wait_stop(&ufp->uf_closing_cv, &ufp->uf_lock, 250);
/*
* cv_wait_stop() drops ufp->uf_lock, so the file list
* can change. Drop the lock on our (possibly) stale
* ufp and let UF_ENTER() find and lock the current ufp.
*/
UF_EXIT(ufp);
UF_ENTER(ufp, fip, fd);
}
#ifdef DEBUG
/*
* catch a watchfd on device's pollhead list but not on fpollinfo list
*/
if (ufp->uf_fpollinfo != NULL)
checkwfdlist(fp->f_vnode, ufp->uf_fpollinfo);
#endif /* DEBUG */
/*
* We may need to cleanup some cached poll states in t_pollstate
* before the fd can be reused. It is important that we don't
* access a stale thread structure. We will do the cleanup in two
* phases to avoid deadlock and holding uf_lock for too long.
* In phase 1, hold the uf_lock and call pollblockexit() to set
* state in t_pollstate struct so that a thread does not exit on
* us. In phase 2, we drop the uf_lock and call pollcacheclean().
*/
pfd = ufp->uf_portfd;
ufp->uf_portfd = NULL;
fpip = ufp->uf_fpollinfo;
ufp->uf_fpollinfo = NULL;
if (fpip != NULL)
pollblockexit(fpip);
UF_EXIT(ufp);
if (fpip != NULL)
pollcacheclean(fpip, fd);
if (pfd)
port_close_fd(pfd);
/*
* Keep the file descriptor entry reserved across the closef().
*/
error = closef(fp);
setf(fd, newfp);
/* Only return closef() error when closing is all we do */
return (newfp == NULL ? error : 0);
}
/*
* Decrement uf_refcnt; wakeup anyone waiting to close the file.
*/
void
releasef(int fd)
{
uf_info_t *fip = P_FINFO(curproc);
uf_entry_t *ufp;
UF_ENTER(ufp, fip, fd);
ASSERT(ufp->uf_refcnt > 0);
clear_active_fd(fd); /* clear the active file descriptor */
if (--ufp->uf_refcnt == 0)
cv_broadcast(&ufp->uf_closing_cv);
UF_EXIT(ufp);
}
/*
* Identical to releasef() but can be called from another process.
*/
void
areleasef(int fd, uf_info_t *fip)
{
uf_entry_t *ufp;
UF_ENTER(ufp, fip, fd);
ASSERT(ufp->uf_refcnt > 0);
if (--ufp->uf_refcnt == 0)
cv_broadcast(&ufp->uf_closing_cv);
UF_EXIT(ufp);
}
/*
* Duplicate all file descriptors across a fork.
*/
void
flist_fork(uf_info_t *pfip, uf_info_t *cfip)
{
int fd, nfiles;
uf_entry_t *pufp, *cufp;
mutex_init(&cfip->fi_lock, NULL, MUTEX_DEFAULT, NULL);
cfip->fi_rlist = NULL;
/*
* We don't need to hold fi_lock because all other lwp's in the
* parent have been held.
*/
cfip->fi_nfiles = nfiles = flist_minsize(pfip);
cfip->fi_list = nfiles == 0 ? NULL :
kmem_zalloc(nfiles * sizeof (uf_entry_t), KM_SLEEP);
for (fd = 0, pufp = pfip->fi_list, cufp = cfip->fi_list; fd < nfiles;
fd++, pufp++, cufp++) {
cufp->uf_file = pufp->uf_file;
cufp->uf_alloc = pufp->uf_alloc;
cufp->uf_flag = pufp->uf_flag;
cufp->uf_busy = pufp->uf_busy;
cufp->uf_gen = pufp->uf_gen;
if (pufp->uf_file == NULL) {
ASSERT(pufp->uf_flag == 0);
if (pufp->uf_busy) {
/*
* Grab locks to appease ASSERTs in fd_reserve
*/
mutex_enter(&cfip->fi_lock);
mutex_enter(&cufp->uf_lock);
fd_reserve(cfip, fd, -1);
mutex_exit(&cufp->uf_lock);
mutex_exit(&cfip->fi_lock);
}
}
}
}
/*
* Close all open file descriptors for the current process.
* This is only called from exit(), which is single-threaded,
* so we don't need any locking.
*/
void
closeall(uf_info_t *fip)
{
int fd;
file_t *fp;
uf_entry_t *ufp;
ufp = fip->fi_list;
for (fd = 0; fd < fip->fi_nfiles; fd++, ufp++) {
if ((fp = ufp->uf_file) != NULL) {
ufp->uf_file = NULL;
if (ufp->uf_portfd != NULL) {
portfd_t *pfd;
/* remove event port association */
pfd = ufp->uf_portfd;
ufp->uf_portfd = NULL;
port_close_fd(pfd);
}
ASSERT(ufp->uf_fpollinfo == NULL);
(void) closef(fp);
}
}
kmem_free(fip->fi_list, fip->fi_nfiles * sizeof (uf_entry_t));
fip->fi_list = NULL;
fip->fi_nfiles = 0;
while (fip->fi_rlist != NULL) {
uf_rlist_t *urp = fip->fi_rlist;
fip->fi_rlist = urp->ur_next;
kmem_free(urp->ur_list, urp->ur_nfiles * sizeof (uf_entry_t));
kmem_free(urp, sizeof (uf_rlist_t));
}
}
/*
* Internal form of close. Decrement reference count on file
* structure. Decrement reference count on the vnode following
* removal of the referencing file structure.
*/
int
closef(file_t *fp)
{
vnode_t *vp;
int error;
int count;
int flag;
offset_t offset;
/*
* audit close of file (may be exit)
*/
if (AU_AUDITING())
audit_closef(fp);
ASSERT(MUTEX_NOT_HELD(&P_FINFO(curproc)->fi_lock));
mutex_enter(&fp->f_tlock);
ASSERT(fp->f_count > 0);
count = fp->f_count--;
flag = fp->f_flag;
offset = fp->f_offset;
vp = fp->f_vnode;
error = VOP_CLOSE(vp, flag, count, offset, fp->f_cred, NULL);
if (count > 1) {
mutex_exit(&fp->f_tlock);
return (error);
}
ASSERT(fp->f_count == 0);
/* Last reference, remove any OFD style lock for the file_t */
ofdcleanlock(fp);
mutex_exit(&fp->f_tlock);
/*
* If DTrace has getf() subroutines active, it will set dtrace_closef
* to point to code that implements a barrier with respect to probe
* context. This must be called before the file_t is freed (and the
* vnode that it refers to is released) -- but it must be after the
* file_t has been removed from the uf_entry_t. That is, there must
* be no way for a racing getf() in probe context to yield the fp that
* we're operating upon.
*/
if (dtrace_closef != NULL)
(*dtrace_closef)();
VN_RELE(vp);
/*
* deallocate resources to audit_data
*/
if (audit_active)
audit_unfalloc(fp);
crfree(fp->f_cred);
kmem_cache_free(file_cache, fp);
return (error);
}
/*
* This is a combination of ufalloc() and setf().
*/
int
ufalloc_file(int start, file_t *fp)
{