Skip to content
Browse files

kernel - usched_dfly revamp (2), reduce token collisions

* Add wakeup_start_delayed() and wakeup_end_delayed().  These functions
  will attempt to delay any wakeup() calls made between them.

  Use the functions in the unix domain socket send code.

* This removes a lot of volatility from monster's 48:48 pgbench tests by
  delaying the wakeup()s related to a unix domain socket write until after
  the pool token has been released.

* Adjust usched_dfly parameters.  In particular, weight2 can be higher now.
  • Loading branch information...
1 parent 574339e commit b0da0c887b6297fd4334569862260a8c0fee1ec6 Matthew Dillon committed Sep 21, 2012
Showing with 62 additions and 7 deletions.
  1. +51 −1 sys/kern/kern_synch.c
  2. +3 −1 sys/kern/uipc_usrreq.c
  3. +3 −3 sys/kern/usched_dfly.c
  4. +2 −1 sys/sys/globaldata.h
  5. +2 −0 sys/sys/systm.h
  6. +1 −1 sys/sys/thread.h
View
52 sys/kern/kern_synch.c
@@ -463,6 +463,14 @@ tsleep(const volatile void *ident, int flags, const char *wmesg, int timo)
struct callout thandle;
/*
+ * Currently a severe hack. Make sure any delayed wakeups
+ * are flushed before we sleep or we might deadlock on whatever
+ * event we are sleeping on.
+ */
+ if (td->td_flags & TDF_DELAYED_WAKEUP)
+ wakeup_end_delayed();
+
+ /*
* NOTE: removed KTRPOINT, it could cause races due to blocking
* even in stable. Just scrap it for now.
*/
@@ -943,7 +951,17 @@ _wakeup(void *ident, int domain)
void
wakeup(const volatile void *ident)
{
- _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, mycpu->gd_cpuid));
+ globaldata_t gd = mycpu;
+ thread_t td = gd->gd_curthread;
+
+ if (td && (td->td_flags & TDF_DELAYED_WAKEUP)) {
+ if (!atomic_cmpset_ptr(&gd->gd_delayed_wakeup[0], NULL, ident)) {
+ if (!atomic_cmpset_ptr(&gd->gd_delayed_wakeup[1], NULL, ident))
+ _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, gd->gd_cpuid));
+ }
+ return;
+ }
+ _wakeup(__DEALL(ident), PWAKEUP_ENCODE(0, gd->gd_cpuid));
}
/*
@@ -1046,6 +1064,38 @@ wakeup_domain_one(const volatile void *ident, int domain)
PWAKEUP_ENCODE(domain, mycpu->gd_cpuid) | PWAKEUP_ONE);
}
+void
+wakeup_start_delayed(void)
+{
+ globaldata_t gd = mycpu;
+
+ crit_enter();
+ gd->gd_curthread->td_flags |= TDF_DELAYED_WAKEUP;
+ crit_exit();
+}
+
+void
+wakeup_end_delayed(void)
+{
+ globaldata_t gd = mycpu;
+
+ if (gd->gd_curthread->td_flags & TDF_DELAYED_WAKEUP) {
+ crit_enter();
+ gd->gd_curthread->td_flags &= ~TDF_DELAYED_WAKEUP;
+ if (gd->gd_delayed_wakeup[0] || gd->gd_delayed_wakeup[1]) {
+ if (gd->gd_delayed_wakeup[0]) {
+ wakeup(gd->gd_delayed_wakeup[0]);
+ gd->gd_delayed_wakeup[0] = NULL;
+ }
+ if (gd->gd_delayed_wakeup[1]) {
+ wakeup(gd->gd_delayed_wakeup[1]);
+ gd->gd_delayed_wakeup[1] = NULL;
+ }
+ }
+ crit_exit();
+ }
+}
+
/*
* setrunnable()
*
View
4 sys/kern/uipc_usrreq.c
@@ -453,6 +453,8 @@ uipc_send(netmsg_t msg)
goto release;
}
+ wakeup_start_delayed();
+
if (control && (error = unp_internalize(control, msg->send.nm_td)))
goto release;
@@ -582,9 +584,9 @@ uipc_send(netmsg_t msg)
if (control && error != 0)
unp_dispose(control);
-
release:
lwkt_relpooltoken(unp);
+ wakeup_end_delayed();
done:
if (control)
View
6 sys/kern/usched_dfly.c
@@ -252,10 +252,10 @@ SYSCTL_INT(_debug, OID_AUTO, dfly_chooser, CTLFLAG_RW,
#ifdef SMP
static int usched_dfly_smt = 0;
static int usched_dfly_cache_coherent = 0;
-static int usched_dfly_weight1 = 30; /* keep thread on current cpu */
-static int usched_dfly_weight2 = 15; /* synchronous peer's current cpu */
+static int usched_dfly_weight1 = 50; /* keep thread on current cpu */
+static int usched_dfly_weight2 = 30; /* synchronous peer's current cpu */
static int usched_dfly_weight3 = 10; /* number of threads on queue */
-static int usched_dfly_weight4 = 50; /* availability of idle cores */
+static int usched_dfly_weight4 = 40; /* availability of idle cores */
static int usched_dfly_features = 0x8F; /* allow pulls */
#endif
static int usched_dfly_rrinterval = (ESTCPUFREQ + 9) / 10;
View
3 sys/sys/globaldata.h
@@ -166,7 +166,8 @@ struct globaldata {
int gd_ireserved[7];
const char *gd_infomsg; /* debugging */
struct lwkt_tokref gd_handoff; /* hand-off tokref */
- void *gd_preserved[8]; /* future fields */
+ void *gd_delayed_wakeup[2];
+ void *gd_preserved[6]; /* future fields */
/* extended by <machine/globaldata.h> */
};
View
2 sys/sys/systm.h
@@ -370,6 +370,8 @@ void wakeup_oncpu (struct globaldata *gd, const volatile void *chan);
void wakeup_oncpu_one (struct globaldata *gd, const volatile void *chan);
void wakeup_domain (const volatile void *chan, int domain);
void wakeup_domain_one (const volatile void *chan, int domain);
+void wakeup_start_delayed(void);
+void wakeup_end_delayed(void);
/*
* Common `cdev_t' stuff are declared here to avoid #include poisoning
View
2 sys/sys/thread.h
@@ -371,7 +371,7 @@ struct thread {
#define TDF_EXITING 0x00400000 /* thread exiting */
#define TDF_USINGFP 0x00800000 /* thread using fp coproc */
#define TDF_KERNELFP 0x01000000 /* kernel using fp coproc */
-#define TDF_UNUSED02000000 0x02000000
+#define TDF_DELAYED_WAKEUP 0x02000000
#define TDF_CRYPTO 0x04000000 /* crypto thread */
#define TDF_MP_STOPREQ 0x00000001 /* suspend_kproc */

0 comments on commit b0da0c8

Please sign in to comment.
Something went wrong with that request. Please try again.