Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

file 153 lines (122 sloc) 5.529 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
#include <unistd.h>
#include <pthread.h>
#include "whistlepig.h"

wp_error* wp_lock_setup(pthread_rwlock_t* lock) {
  pthread_rwlockattr_t attr;
  if(pthread_rwlockattr_init(&attr) != 0) RAISE_SYSERROR("cannot initialize pthreads rwlock attr");
  if(pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) != 0) RAISE_SYSERROR("cannot set pthreads rwlockattr to PTHREAD_PROCESS_SHARED");
  if(pthread_rwlock_init(lock, &attr) != 0) RAISE_SYSERROR("cannot initialize pthreads rwlock");
  if(pthread_rwlockattr_destroy(&attr) != 0) RAISE_SYSERROR("cannot destroy pthreads rwlock attr");

  return NO_ERROR;
}

/*

alernative implementation that uses rdlock. doesn't allow us to detect
and break stale locks, but gives us a good sense of what the timing should
be like.

wp_error* wp_segment_grab_lock2(wp_segment* seg, int lock_type) {
segment_info* si = MMAP_OBJ(seg->seginfo, segment_info);
const char* lock_name = (lock_type == WP_LOCK_READLOCK ? "read" : "write");
DEBUG("grabbing %slock for segment %p", lock_name, seg);

struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);

int ret = 0;
switch(lock_type) {
case WP_LOCK_READLOCK: ret = pthread_rwlock_rdlock(&si->lock); break;
case WP_LOCK_WRITELOCK: ret = pthread_rwlock_wrlock(&si->lock); break;
}

if(ret != 0) RAISE_SYSERROR("grabbing %slock", lock_name);

clock_gettime(CLOCK_MONOTONIC, &end);


uint64_t diff_in_ns = ((end.tv_sec * 1000000000) + end.tv_nsec) -
((start.tv_sec * 1000000000) + start.tv_nsec);

uint32_t total_delay_ms = diff_in_ns / 1000000;
if(total_delay_ms > 0) printf("XXX acquired %slock for segment %p after %ums\n", lock_name, seg, total_delay_ms);

return NO_ERROR;
}
*/

// we will wait this many milliseconds before assuming the lock
// is stale and breaking it.
#define LOCK_STALE_TIME_MS 2500

/* here's the best implementation i can find, empirically, of being
able to grab pthread read and write locks, while still being able
to detect stale locks and repair them.

it involves a busyloop, which is lame.
*/
wp_error* wp_lock_grab(pthread_rwlock_t* lock, int lock_type) {
  const char* lock_name = (lock_type == WP_LOCK_READLOCK ? "read" : "write");
  DEBUG("grabbing %slock at %p", lock_name, lock);

  unsigned int delay_ms = 1;
  uint32_t total_delay_ms = 0;

  while(1) {
    int ret = 0;

    switch(lock_type) {
      case WP_LOCK_READLOCK: ret = pthread_rwlock_tryrdlock(lock); break;
      case WP_LOCK_WRITELOCK: ret = pthread_rwlock_trywrlock(lock); break;
      default: RAISE_ERROR("invalid lock type");
    }

    if(ret == 0) break; // acquired!
    // we get EAGAINs here if the writer died before closing the lock.
    if((ret != EBUSY) && (ret != EAGAIN)) RAISE_SYSERROR("acquiring %slock", lock_name);

    if(total_delay_ms >= LOCK_STALE_TIME_MS) {
      //RAISE_ERROR("timeout acquiring %slock: %ums", lock_name, total_delay_ms);
      DEBUG("assuming lock is stale and breaking it!");
      RELAY_ERROR(wp_lock_setup(lock));
    }
    if(delay_ms > 1000) sleep(delay_ms / 1000);
    usleep(1000 * (delay_ms % 1000));
    total_delay_ms += delay_ms;
  }

  if(total_delay_ms > 0) DEBUG(":( acquired %slock for after %ums\n", lock_name, total_delay_ms);
  return NO_ERROR;
}

/* an alternative implementation that uses the _timed pthread operations.
although this should be the best version, i had many problems with it. the
timeout didn't seem to ever trigger. i would also see an EINVAL whenever a
writer had a readlock. in the case of a stale lock, i would just get EINVALS
forever rather than a proper ETIMEDOUT.

since using this would require implementing my own stale lock detection
anyways, so i'm just going to use the simpler version above instead.
*/
/*
wp_error* wp_segment_grab_lock3(wp_segment* seg, int lock_type) {
segment_info* si = MMAP_OBJ(seg->seginfo, segment_info);
const char* lock_name = (lock_type == WP_LOCK_READLOCK ? "read" : "write");
DEBUG("grabbing %slock for segment %p", lock_name, seg);

struct timespec timeout;
timeout.tv_sec = 3;//LOCK_STALE_TIME_MS / 1000;
timeout.tv_nsec = 0;//(LOCK_STALE_TIME_MS % 1000) * 1000000;

struct timeval startt, endt;
gettimeofday(&startt, NULL);

int acquired = 0;
while(!acquired) {
int ret = 0;

switch(lock_type) {
case WP_LOCK_READLOCK: ret = pthread_rwlock_timedrdlock(&si->lock, &timeout); break;
case WP_LOCK_WRITELOCK: ret = pthread_rwlock_timedwrlock(&si->lock, &timeout); break;
default: RAISE_ERROR("invalid lock type");
}

switch(ret) {
case 0: acquired = 1; break;
case ETIMEDOUT:
DEBUG("assuming lock is stale and breaking it!");
RELAY_ERROR(setup_lock(&si->lock));
break;
case EAGAIN:
// despite the documentation, this seems to happen every time we request a readlock and
// the lock is already held by the writer. so we will just busyloop here. this happens
// fairly frequently, so this is lame.
usleep(1000);
break;
default:
RAISE_SYSERROR("acquiring %slock", lock_name);
}
}
gettimeofday(&endt, NULL);
long elapsed = ((endt.tv_sec - startt.tv_sec) * 1000) + ((endt.tv_usec - startt.tv_usec) / 1000);

if(elapsed > 0) printf(":( acquired %slock for segment %p after %ldms\n", lock_name, seg, elapsed);
return NO_ERROR;
}
*/

wp_error* wp_lock_release(pthread_rwlock_t* lock) {
  if(pthread_rwlock_unlock(lock) != 0) RAISE_SYSERROR("releasing lock");
  return NO_ERROR;
}
Something went wrong with that request. Please try again.