@@ -23,157 +23,152 @@ package runtime
2323typedef struct Sema Sema;
2424struct Sema
2525{
26- uint32 *addr;
26+ uint32 volatile *addr;
2727 G *g;
2828 Sema *prev;
2929 Sema *next;
3030};
3131
32- // TODO: For now, a linked list; maybe a hash table of linked lists later.
33- static Sema *semfirst, *semlast;
34- static Lock semlock;
32+ typedef struct SemaRoot SemaRoot;
33+ struct SemaRoot
34+ {
35+ Lock;
36+ Sema *head;
37+ Sema *tail;
38+ // Number of waiters. Read w/o the lock.
39+ uint32 volatile nwait;
40+ };
41+
42+ // Prime to not correlate with any user patterns.
43+ #define SEMTABLESZ 251
44+
45+ static union
46+ {
47+ SemaRoot;
48+ // Modern processors tend to have 64-byte cache lines,
49+ // potentially with 128-byte effective cache line size for reading.
50+ // While there are hypothetical architectures
51+ // with 16-4096 byte cache lines, 128 looks like a good compromise.
52+ uint8 pad[128];
53+ } semtable[SEMTABLESZ];
54+
55+ static SemaRoot*
56+ semroot(uint32 *addr)
57+ {
58+ return &semtable[((uintptr)addr >> 3) % SEMTABLESZ];
59+ }
3560
3661static void
37- semqueue(uint32 *addr, Sema *s)
62+ semqueue(SemaRoot *root, uint32 volatile *addr, Sema *s)
3863{
64+ s->g = g;
3965 s->addr = addr;
40- s->g = nil;
41-
42- runtime·lock(&semlock);
43- s->prev = semlast;
4466 s->next = nil;
45- if(semlast)
46- semlast->next = s;
67+ s->prev = root->tail;
68+ if(root->tail)
69+ root->tail->next = s;
4770 else
48- semfirst = s;
49- semlast = s;
50- runtime·unlock(&semlock);
71+ root->head = s;
72+ root->tail = s;
5173}
5274
5375static void
54- semdequeue(Sema *s)
76+ semdequeue(SemaRoot *root, Sema *s)
5577{
56- runtime·lock(&semlock);
5778 if(s->next)
5879 s->next->prev = s->prev;
5980 else
60- semlast = s->prev;
81+ root->tail = s->prev;
6182 if(s->prev)
6283 s->prev->next = s->next;
6384 else
64- semfirst = s->next;
85+ root->head = s->next;
6586 s->prev = nil;
6687 s->next = nil;
67- runtime·unlock(&semlock);
68- }
69-
70- static void
71- semwakeup(uint32 *addr)
72- {
73- Sema *s;
74-
75- runtime·lock(&semlock);
76- for(s=semfirst; s; s=s->next) {
77- if(s->addr == addr && s->g) {
78- runtime·ready(s->g);
79- s->g = nil;
80- break;
81- }
82- }
83- runtime·unlock(&semlock);
84- }
85-
86- // Step 1 of sleep: make ourselves available for wakeup.
87- // TODO(rsc): Maybe we can write a version without
88- // locks by using cas on s->g. Maybe not: I need to
89- // think more about whether it would be correct.
90- static void
91- semsleep1(Sema *s)
92- {
93- runtime·lock(&semlock);
94- s->g = g;
95- runtime·unlock(&semlock);
96- }
97-
98- // Decided not to go through with it: undo step 1.
99- static void
100- semsleepundo1(Sema *s)
101- {
102- runtime·lock(&semlock);
103- if(s->g != nil) {
104- s->g = nil; // back ourselves out
105- } else {
106- // If s->g == nil already, semwakeup
107- // already readied us. Since we never stopped
108- // running, readying us just set g->readyonstop.
109- // Clear it.
110- if(g->readyonstop == 0)
111- *(int32*)0x555 = 555;
112- g->readyonstop = 0;
113- }
114- runtime·unlock(&semlock);
115- }
116-
117- // Step 2: wait for the wakeup.
118- static void
119- semsleep2(Sema *s)
120- {
121- USED(s);
122- g->status = Gwaiting;
123- runtime·gosched();
12488}
12589
12690static int32
12791cansemacquire(uint32 *addr)
12892{
12993 uint32 v;
13094
131- while((v = * addr) > 0)
95+ while((v = runtime·atomicload( addr) ) > 0)
13296 if(runtime·cas(addr, v, v-1))
13397 return 1;
13498 return 0;
13599}
136100
137- // For now has no return value.
138- // Might return an ok (not interrupted) bool in the future?
139101void
140- runtime·semacquire(uint32 *addr)
102+ runtime·semacquire(uint32 volatile *addr)
141103{
142104 Sema s;
105+ SemaRoot *root;
143106
144107 // Easy case.
145108 if(cansemacquire(addr))
146109 return;
147110
148111 // Harder case:
149- // queue
150- // try semacquire one more time, sleep if failed
151- // dequeue
152- // wake up one more guy to avoid races (TODO(rsc): maybe unnecessary?)
153- semqueue(addr, &s);
112+ // increment waiter count
113+ // try cansemacquire one more time, return if succeeded
114+ // enqueue itself as a waiter
115+ // sleep
116+ // (waiter descriptor is dequeued by signaler)
117+ root = semroot(addr);
154118 for(;;) {
155- semsleep1(&s);
119+ runtime·lock(root);
120+ // Add ourselves to nwait to disable "easy case" in semrelease.
121+ runtime·xadd(&root->nwait, 1);
122+ // Check cansemacquire to avoid missed wakeup.
156123 if(cansemacquire(addr)) {
157- semsleepundo1(&s);
158- break;
124+ runtime·xadd(&root->nwait, -1);
125+ runtime·unlock(root);
126+ return;
159127 }
160- semsleep2(&s);
128+ // Any semrelease after the cansemacquire knows we're waiting
129+ // (we set nwait above), so go to sleep.
130+ semqueue(root, addr, &s);
131+ g->status = Gwaiting;
132+ runtime·unlock(root);
133+ runtime·gosched();
134+ if(cansemacquire(addr))
135+ return;
161136 }
162- semdequeue(&s);
163- semwakeup(addr);
164137}
165138
166139void
167- runtime·semrelease(uint32 *addr)
140+ runtime·semrelease(uint32 volatile *addr)
168141{
169- uint32 v;
142+ Sema *s;
143+ SemaRoot *root;
170144
171- for(;;) {
172- v = *addr;
173- if(runtime·cas(addr, v, v+1))
145+ root = semroot(addr);
146+ runtime·xadd(addr, 1);
147+
148+ // Easy case: no waiters?
149+ // This check must happen after the xadd, to avoid a missed wakeup
150+ // (see loop in semacquire).
151+ if(runtime·atomicload(&root->nwait) == 0)
152+ return;
153+
154+ // Harder case: search for a waiter and wake it.
155+ runtime·lock(root);
156+ if(runtime·atomicload(&root->nwait) == 0) {
157+ // The count is already consumed by another goroutine,
158+ // so no need to wake up another goroutine.
159+ runtime·unlock(root);
160+ return;
161+ }
162+ for(s = root->head; s; s = s->next) {
163+ if(s->addr == addr) {
164+ runtime·xadd(&root->nwait, -1);
165+ semdequeue(root, s);
174166 break;
167+ }
175168 }
176- semwakeup(addr);
169+ runtime·unlock(root);
170+ if(s)
171+ runtime·ready(s->g);
177172}
178173
179174func Semacquire(addr *uint32) {
0 commit comments