Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

kernel - usched_dfly revamp (2), reduce token collisions

* Add wakeup_start_delayed() and wakeup_end_delayed().  These functions
  will attempt to delay any wakeup() calls made between them.

  Use the functions in the unix domain socket send code.

* This removes a lot of volatility from monster's 48:48 pgbench tests by
  delaying the wakeup()s related to a unix domain socket write until after
  the pool token has been released.

* Adjust usched_dfly parameters.  In particular, weight2 can be higher now.
  • Loading branch information...
commit b0da0c887b6297fd4334569862260a8c0fee1ec6 1 parent 574339e
Matthew Dillon authored
52 sys/kern/kern_synch.c
@@ -463,6 +463,14 @@ tsleep(const volatile void *ident, int flags, const char *wmesg, int timo)
463 463 struct callout thandle;
464 464
465 465 /*
  466 + * Currently a severe hack. Make sure any delayed wakeups
  467 + * are flushed before we sleep or we might deadlock on whatever
  468 + * event we are sleeping on.
  469 + */
  470 + if (td->td_flags & TDF_DELAYED_WAKEUP)
  471 + wakeup_end_delayed();
  472 +
  473 + /*
466 474 * NOTE: removed KTRPOINT, it could cause races due to blocking
467 475 * even in stable. Just scrap it for now.
468 476 */
@@ -943,7 +951,17 @@ _wakeup(void *ident, int domain)
943 951 void
944 952 wakeup(const volatile void *ident)
945 953 {
946   - _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, mycpu->gd_cpuid));
  954 + globaldata_t gd = mycpu;
  955 + thread_t td = gd->gd_curthread;
  956 +
  957 + if (td && (td->td_flags & TDF_DELAYED_WAKEUP)) {
  958 + if (!atomic_cmpset_ptr(&gd->gd_delayed_wakeup[0], NULL, ident)) {
  959 + if (!atomic_cmpset_ptr(&gd->gd_delayed_wakeup[1], NULL, ident))
  960 + _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, gd->gd_cpuid));
  961 + }
  962 + return;
  963 + }
  964 + _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, gd->gd_cpuid));
947 965 }
948 966
949 967 /*
@@ -1046,6 +1064,38 @@ wakeup_domain_one(const volatile void *ident, int domain)
1046 1064 PWAKEUP_ENCODE(domain, mycpu->gd_cpuid) | PWAKEUP_ONE);
1047 1065 }
1048 1066
  1067 +void
  1068 +wakeup_start_delayed(void)
  1069 +{
  1070 + globaldata_t gd = mycpu;
  1071 +
  1072 + crit_enter();
  1073 + gd->gd_curthread->td_flags |= TDF_DELAYED_WAKEUP;
  1074 + crit_exit();
  1075 +}
  1076 +
  1077 +void
  1078 +wakeup_end_delayed(void)
  1079 +{
  1080 + globaldata_t gd = mycpu;
  1081 +
  1082 + if (gd->gd_curthread->td_flags & TDF_DELAYED_WAKEUP) {
  1083 + crit_enter();
  1084 + gd->gd_curthread->td_flags &= ~TDF_DELAYED_WAKEUP;
  1085 + if (gd->gd_delayed_wakeup[0] || gd->gd_delayed_wakeup[1]) {
  1086 + if (gd->gd_delayed_wakeup[0]) {
  1087 + wakeup(gd->gd_delayed_wakeup[0]);
  1088 + gd->gd_delayed_wakeup[0] = NULL;
  1089 + }
  1090 + if (gd->gd_delayed_wakeup[1]) {
  1091 + wakeup(gd->gd_delayed_wakeup[1]);
  1092 + gd->gd_delayed_wakeup[1] = NULL;
  1093 + }
  1094 + }
  1095 + crit_exit();
  1096 + }
  1097 +}
  1098 +
1049 1099 /*
1050 1100 * setrunnable()
1051 1101 *
4 sys/kern/uipc_usrreq.c
@@ -453,6 +453,8 @@ uipc_send(netmsg_t msg)
453 453 goto release;
454 454 }
455 455
  456 + wakeup_start_delayed();
  457 +
456 458 if (control && (error = unp_internalize(control, msg->send.nm_td)))
457 459 goto release;
458 460
@@ -582,9 +584,9 @@ uipc_send(netmsg_t msg)
582 584
583 585 if (control && error != 0)
584 586 unp_dispose(control);
585   -
586 587 release:
587 588 lwkt_relpooltoken(unp);
  589 + wakeup_end_delayed();
588 590 done:
589 591
590 592 if (control)
6 sys/kern/usched_dfly.c
@@ -252,10 +252,10 @@ SYSCTL_INT(_debug, OID_AUTO, dfly_chooser, CTLFLAG_RW,
252 252 #ifdef SMP
253 253 static int usched_dfly_smt = 0;
254 254 static int usched_dfly_cache_coherent = 0;
255   -static int usched_dfly_weight1 = 30; /* keep thread on current cpu */
256   -static int usched_dfly_weight2 = 15; /* synchronous peer's current cpu */
  255 +static int usched_dfly_weight1 = 50; /* keep thread on current cpu */
  256 +static int usched_dfly_weight2 = 30; /* synchronous peer's current cpu */
257 257 static int usched_dfly_weight3 = 10; /* number of threads on queue */
258   -static int usched_dfly_weight4 = 50; /* availability of idle cores */
  258 +static int usched_dfly_weight4 = 40; /* availability of idle cores */
259 259 static int usched_dfly_features = 0x8F; /* allow pulls */
260 260 #endif
261 261 static int usched_dfly_rrinterval = (ESTCPUFREQ + 9) / 10;
3  sys/sys/globaldata.h
@@ -166,7 +166,8 @@ struct globaldata {
166 166 int gd_ireserved[7];
167 167 const char *gd_infomsg; /* debugging */
168 168 struct lwkt_tokref gd_handoff; /* hand-off tokref */
169   - void *gd_preserved[8]; /* future fields */
  169 + void *gd_delayed_wakeup[2];
  170 + void *gd_preserved[6]; /* future fields */
170 171 /* extended by <machine/globaldata.h> */
171 172 };
172 173
2  sys/sys/systm.h
@@ -370,6 +370,8 @@ void wakeup_oncpu (struct globaldata *gd, const volatile void *chan);
370 370 void wakeup_oncpu_one (struct globaldata *gd, const volatile void *chan);
371 371 void wakeup_domain (const volatile void *chan, int domain);
372 372 void wakeup_domain_one (const volatile void *chan, int domain);
  373 +void wakeup_start_delayed(void);
  374 +void wakeup_end_delayed(void);
373 375
374 376 /*
375 377 * Common `cdev_t' stuff are declared here to avoid #include poisoning
2  sys/sys/thread.h
@@ -371,7 +371,7 @@ struct thread {
371 371 #define TDF_EXITING 0x00400000 /* thread exiting */
372 372 #define TDF_USINGFP 0x00800000 /* thread using fp coproc */
373 373 #define TDF_KERNELFP 0x01000000 /* kernel using fp coproc */
374   -#define TDF_UNUSED02000000 0x02000000
  374 +#define TDF_DELAYED_WAKEUP 0x02000000
375 375 #define TDF_CRYPTO 0x04000000 /* crypto thread */
376 376
377 377 #define TDF_MP_STOPREQ 0x00000001 /* suspend_kproc */

0 comments on commit b0da0c8

Please sign in to comment.
Something went wrong with that request. Please try again.