Per-thread current time caching.

This patch fixes the problem when stored TSC is shared by
multiple cores (TSC can be different for each core).
It also prevents multiple cores accessing the same cacheline.

Signed-off-by: Martin Sustrik <sustrik@250bpm.com>

cr.c

@@ -88,7 +88,8 @@ int dill_ctx_cr_init(struct dill_ctx_cr *ctx) {
     ctx->r = &ctx->main;
     dill_qlist_init(&ctx->ready);
     dill_rbtree_init(&ctx->timers);
-    ctx->last_poll = now();
+    /* We can't use now() here as the context is still being intialized. */
+    ctx->last_poll = mnow();
     /* Initialize the main coroutine. */
     memset(&ctx->main, 0, sizeof(ctx->main));
     ctx->main.ready.next = NULL;

ctx.c

@@ -31,10 +31,13 @@ static void dill_ctx_atexit(void) {
     dill_ctx_stack_term(&dill_ctx_.stack);
     dill_ctx_handle_term(&dill_ctx_.handle);
     dill_ctx_cr_term(&dill_ctx_.cr);
+    dill_ctx_now_term(&dill_ctx_.now);
 }
 
 struct dill_ctx *dill_ctx_init(void) {
-    int rc = dill_ctx_cr_init(&dill_ctx_.cr);
+    int rc = dill_ctx_now_init(&dill_ctx_.now);
+    dill_assert(rc == 0);
+    rc = dill_ctx_cr_init(&dill_ctx_.cr);
     dill_assert(rc == 0);
     rc = dill_ctx_handle_init(&dill_ctx_.handle);
     dill_assert(rc == 0);
@@ -95,6 +98,7 @@ static void dill_ctx_term(void *ptr) {
     dill_ctx_stack_term(&ctx->stack);
     dill_ctx_handle_term(&ctx->handle);
     dill_ctx_cr_term(&ctx->cr);
+    dill_ctx_now_term(&ctx->now);
     if(dill_ismain()) dill_main = NULL;
 }
 
@@ -108,7 +112,9 @@ static void dill_makekey(void) {
 }
 
 struct dill_ctx *dill_ctx_init(void) {
-    int rc = dill_ctx_cr_init(&dill_ctx_.cr);
+    int rc = dill_ctx_now_init(&dill_ctx_.now);
+    dill_assert(rc == 0);
+    rc = dill_ctx_cr_init(&dill_ctx_.cr);
     dill_assert(rc == 0);
     rc = dill_ctx_handle_init(&dill_ctx_.handle);
     dill_assert(rc == 0);
@@ -141,6 +147,7 @@ static void dill_ctx_term(void *ptr) {
     dill_ctx_stack_term(&ctx->stack);
     dill_ctx_handle_term(&ctx->handle);
     dill_ctx_cr_term(&ctx->cr);
+    dill_ctx_now_term(&ctx->now);
     free(ctx);
     if(dill_ismain()) dill_main = NULL;
 }
@@ -161,6 +168,8 @@ struct dill_ctx *dill_getctx_(void) {
     if(dill_fast(ctx)) return ctx;
     ctx = malloc(sizeof(struct dill_ctx));
     dill_assert(ctx);
+    rc = dill_ctx_now_init(&ctx->now);
+    dill_assert(rc == 0);
     rc = dill_ctx_cr_init(&ctx->cr);
     dill_assert(rc == 0);
     rc = dill_ctx_handle_init(&ctx->handle);

ctx.h

@@ -25,13 +25,15 @@
 
 #include "cr.h"
 #include "handle.h"
+#include "now.h"
 #include "pollset.h"
 #include "stack.h"
 
 struct dill_ctx {
 #if !defined DILL_THREAD_FALLBACK
     int initialized;
 #endif
+    struct dill_ctx_now now;
     struct dill_ctx_cr cr;
     struct dill_ctx_handle handle;
     struct dill_ctx_stack stack;

now.c

@@ -30,15 +30,13 @@
 
 #if defined(__x86_64__) || defined(__i386__)
 #include <x86intrin.h>
-#define DILL_RDTSC_DIFF 1000000ULL
 #endif
 
-#include "cr.h"
-#include "libdill.h"
-#include "pollset.h"
-#include "utils.h"
+#include "ctx.h"
 
-static int64_t mnow(void) {
+int64_t mnow(void) {
+
+/* Implementation using Mach timers. */
 #if defined __APPLE__
     static mach_timebase_info_data_t dill_mtid = {0};
     if (dill_slow(!dill_mtid.denom))
@@ -47,6 +45,7 @@ static int64_t mnow(void) {
     return (int64_t)(ticks * dill_mtid.numer / dill_mtid.denom / 1000000);
 #else
 
+/* Implementation using clock_gettime(). */
 #if defined CLOCK_MONOTONIC_COARSE
     clock_t id = CLOCK_MONOTONIC_COARSE;
 #elif defined CLOCK_MONOTONIC_FAST
@@ -56,15 +55,15 @@ static int64_t mnow(void) {
 #else
 #define DILL_NOW_FALLBACK
 #endif
-
 #if !defined DILL_NOW_FALLBACK
     struct timespec ts;
     int rc = clock_gettime(id, &ts);
     dill_assert (rc == 0);
     return ((int64_t)ts.tv_sec) * 1000 + (((int64_t)ts.tv_nsec) / 1000000);
+
+/* Implementation using gettimeofday(). This is slow and error-prone
+   (the time can jump backwards!), but it's just a last resort option. */
 #else
-    /* This is slow and error-prone (the time can jump backwards!), but it's just
-       a last resort option. */
     struct timeval tv;
     int rc = gettimeofday(&tv, NULL);
     dill_assert (rc == 0);
@@ -76,18 +75,37 @@ static int64_t mnow(void) {
 
 int64_t now(void) {
 #if defined(__x86_64__) || defined(__i386__)
-    static int64_t last_tick = 0;
-    static uint64_t last_rdtsc = 0;
-    uint64_t rdtsc = __rdtsc();
-    int64_t diff = rdtsc - last_rdtsc;
+    /* On x86 platforms, rdtsc instruction can be used to quickly check time
+       in form of CPU cycles. If less than 1M cycles have elapsed since the
+       last mnow() call we assume it's still the same millisecond and return
+       cached time. This optimization can give a huge speedup with old systems.
+       1M number is chosen is such a way that it results in getting time every
+       millisecond on 1GHz processors. On faster processors we'll query time
+       somewhat more often but the number of queries should still be
+       statistically insignificant. On slower processors we'll start losing
+       precision, e.g. on 500MHz processor we can diverge by 1ms. */
+    struct dill_ctx_now *ctx = &dill_getctx->now;
+    uint64_t tsc = __rdtsc();
+    int64_t diff = tsc - ctx->last_tsc;
     if(diff < 0) diff = -diff;
-    if(dill_fast(diff < DILL_RDTSC_DIFF))
-        return last_tick;
+    if(dill_fast(diff < 1000000ULL))
+        return ctx->last_time;
     else
-        last_rdtsc = rdtsc;
-    return (last_tick = mnow());
+        ctx->last_tsc = tsc;
+    return ctx->last_time = mnow();
 #else
     return mnow();
 #endif
 }
 
+int dill_ctx_now_init(struct dill_ctx_now *ctx) {
+#if defined(__x86_64__) || defined(__i386__)
+    ctx->last_time = mnow();
+    ctx->last_tsc = __rdtsc();
+#endif
+    return 0;
+}
+
+void dill_ctx_now_term(struct dill_ctx_now *ctx) {
+}
+

now.h

@@ -25,5 +25,21 @@
 #ifndef DILL_NOW_INCLUDED
 #define DILL_NOW_INCLUDED
 
+#include <stdint.h>
+
+struct dill_ctx_now {
+#if defined(__x86_64__) || defined(__i386__)
+    int64_t last_time;
+    uint64_t last_tsc;
+#endif
+};
+
+int dill_ctx_now_init(struct dill_ctx_now *ctx);
+void dill_ctx_now_term(struct dill_ctx_now *ctx);
+
+/* Same as now() except that it doesn't use the context.
+   I.e. it can be called before calling dill_ctx_now_init(). */
+int64_t mnow(void);
+
 #endif