Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

avva's bucket/generation patch

git-svn-id: http://code.sixapart.com/svn/memcached/trunk@252 b0b603af-a30f-0410-a34e-baf09ae79d0b
  • Loading branch information...
commit 7a308025661a49a5e19f98d2c5b8df04d96b4642 1 parent c252f6e
authored March 05, 2006
4  ChangeLog
... ...
@@ -1,3 +1,7 @@
  1
+2006-03-04
  2
+	* avva: bucket/generation patch (old, but Brad's just finally
  3
+	committing it)
  4
+
1 5
 2006-01-01
2 6
 	* Brad Fitzpatrick <brad@danga.com>:  allocate 1 slab per class
3 7
 	on start-up, to avoid confusing users with out-of-memory errors
2  configure.ac
... ...
@@ -1,5 +1,5 @@
1 1
 AC_PREREQ(2.52)
2  
-AC_INIT(memcached, 1.1.12, brad@danga.com)
  2
+AC_INIT(memcached, 1.1.13-cvs, brad@danga.com)
3 3
 AC_CANONICAL_SYSTEM
4 4
 AC_CONFIG_SRCDIR(memcached.c)
5 5
 AM_INIT_AUTOMAKE(AC_PACKAGE_NAME, AC_PACKAGE_VERSION)
194  memcached.c
@@ -56,6 +56,8 @@ static item **todelete = 0;
56 56
 static int delcurr;
57 57
 static int deltotal;
58 58
 
  59
+int *buckets = 0; /* bucket->generation array for a managed instance */
  60
+
59 61
 time_t realtime(time_t exptime) {
60 62
     time_t now;
61 63
 
@@ -93,6 +95,7 @@ void settings_init(void) {
93 95
     settings.verbose = 0;
94 96
     settings.oldest_live = 0;
95 97
     settings.evict_to_free = 1;       /* push old items out of cache when memory runs out */
  98
+    settings.managed = 0;
96 99
 }
97 100
 
98 101
 conn **freeconns;
@@ -141,6 +144,7 @@ conn *conn_new(int sfd, int init_state, int event_flags) {
141 144
             return 0;
142 145
         }
143 146
         c->rsize = c->wsize = DATA_BUFFER_SIZE;
  147
+        c->rcurr = c->rbuf;
144 148
         c->isize = 200;
145 149
         stats.conn_structs++;
146 150
     }
@@ -157,7 +161,7 @@ conn *conn_new(int sfd, int init_state, int event_flags) {
157 161
     c->rlbytes = 0;
158 162
     c->rbytes = c->wbytes = 0;
159 163
     c->wcurr = c->wbuf;
160  
-    c->rcurr = c->rbuf;
  164
+    c->ritem = 0;
161 165
     c->icurr = c->ilist; 
162 166
     c->ileft = 0;
163 167
     c->iptr = c->ibuf;
@@ -166,6 +170,8 @@ conn *conn_new(int sfd, int init_state, int event_flags) {
166 170
     c->write_and_go = conn_read;
167 171
     c->write_and_free = 0;
168 172
     c->item = 0;
  173
+    c->bucket = -1;
  174
+    c->gen = 0;
169 175
 
170 176
     c->is_corked = 0;
171 177
 
@@ -522,6 +528,20 @@ void process_command(conn *c, char *command) {
522 528
             out_string(c, "CLIENT_ERROR bad command line format");
523 529
             return;
524 530
         }
  531
+
  532
+        if (settings.managed) {
  533
+            int bucket = c->bucket;
  534
+            if (bucket == -1) {
  535
+                out_string(c, "CLIENT_ERROR no BG data in managed mode");
  536
+                return;
  537
+            }
  538
+            c->bucket = -1;
  539
+            if (buckets[bucket] != c->gen) {
  540
+                out_string(c, "ERROR_NOT_OWNER");
  541
+                return;
  542
+            }
  543
+        }
  544
+
525 545
         expire = realtime(expire);
526 546
         it = item_alloc(key, flags, expire, len+2);
527 547
         if (it == 0) {
@@ -534,7 +554,7 @@ void process_command(conn *c, char *command) {
534 554
 
535 555
         c->item_comm = comm;
536 556
         c->item = it;
537  
-        c->rcurr = ITEM_data(it);
  557
+        c->ritem = ITEM_data(it);
538 558
         c->rlbytes = it->nbytes;
539 559
         c->state = conn_nread;
540 560
         return;
@@ -556,7 +576,20 @@ void process_command(conn *c, char *command) {
556 576
             out_string(c, "CLIENT_ERROR bad command line format");
557 577
             return;
558 578
         }
559  
-        
  579
+
  580
+        if (settings.managed) {
  581
+            int bucket = c->bucket;
  582
+            if (bucket == -1) {
  583
+                out_string(c, "CLIENT_ERROR no BG data in managed mode");
  584
+                return;
  585
+            }
  586
+            c->bucket = -1;
  587
+            if (buckets[bucket] != c->gen) {
  588
+                out_string(c, "ERROR_NOT_OWNER");
  589
+                return;
  590
+            }
  591
+        }
  592
+
560 593
         it = assoc_find(key);
561 594
         if (it && (it->it_flags & ITEM_DELETED)) {
562 595
             it = 0;
@@ -612,6 +645,19 @@ void process_command(conn *c, char *command) {
612 645
         item *it;
613 646
         time_t now = time(0);
614 647
 
  648
+        if (settings.managed) {
  649
+            int bucket = c->bucket;
  650
+            if (bucket == -1) {
  651
+                out_string(c, "CLIENT_ERROR no BG data in managed mode");
  652
+                return;
  653
+            }
  654
+            c->bucket = -1;
  655
+            if (buckets[bucket] != c->gen) {
  656
+                out_string(c, "ERROR_NOT_OWNER");
  657
+                return;
  658
+            }
  659
+        }
  660
+
615 661
         while(sscanf(start, " %250s%n", key, &next) >= 1) {
616 662
             start+=next;
617 663
             stats.get_cmds++;
@@ -663,6 +709,19 @@ void process_command(conn *c, char *command) {
663 709
         int res;
664 710
         time_t exptime = 0;
665 711
 
  712
+        if (settings.managed) {
  713
+            int bucket = c->bucket;
  714
+            if (bucket == -1) {
  715
+                out_string(c, "CLIENT_ERROR no BG data in managed mode");
  716
+                return;
  717
+            }
  718
+            c->bucket = -1;
  719
+            if (buckets[bucket] != c->gen) {
  720
+                out_string(c, "ERROR_NOT_OWNER");
  721
+                return;
  722
+            }
  723
+        }
  724
+
666 725
         res = sscanf(command, "%*s %250s %ld", key, &exptime);
667 726
         it = assoc_find(key);
668 727
         if (!it) {
@@ -701,7 +760,75 @@ void process_command(conn *c, char *command) {
701 760
         out_string(c, "DELETED");
702 761
         return;
703 762
     }
704  
-        
  763
+
  764
+    if (strncmp(command, "own ", 4) == 0) {
  765
+        int bucket, gen;
  766
+        char *start = command+4;
  767
+        if (!settings.managed) {
  768
+            out_string(c, "CLIENT_ERROR not a managed instance");
  769
+            return;
  770
+        }
  771
+        if (sscanf(start, "%u:%u\r\n", &bucket,&gen) == 2) {
  772
+            if ((bucket < 0) || (bucket >= MAX_BUCKETS)) {
  773
+                out_string(c, "CLIENT_ERROR bucket number out of range");
  774
+                return;
  775
+            }
  776
+            buckets[bucket] = gen;
  777
+            out_string(c, "OWNED");
  778
+            return;
  779
+        } else {
  780
+            out_string(c, "CLIENT_ERROR bad format");
  781
+            return;
  782
+        }
  783
+    }
  784
+
  785
+    if (strncmp(command, "disown ", 7) == 0) {
  786
+        int bucket;
  787
+        char *start = command+7;
  788
+        if (!settings.managed) {
  789
+            out_string(c, "CLIENT_ERROR not a managed instance");
  790
+            return;
  791
+        }
  792
+        if (sscanf(start, "%u\r\n", &bucket) == 1) {
  793
+            if ((bucket < 0) || (bucket >= MAX_BUCKETS)) {
  794
+                out_string(c, "CLIENT_ERROR bucket number out of range");
  795
+                return;
  796
+            }
  797
+            buckets[bucket] = 0;
  798
+            out_string(c, "DISOWNED");
  799
+            return;
  800
+        } else {
  801
+            out_string(c, "CLIENT_ERROR bad format");
  802
+            return;
  803
+        }
  804
+    }
  805
+
  806
+    if (strncmp(command, "bg ", 3) == 0) {
  807
+        int bucket, gen;
  808
+        char *start = command+3;
  809
+        if (!settings.managed) {
  810
+            out_string(c, "CLIENT_ERROR not a managed instance");
  811
+            return;
  812
+        }
  813
+        if (sscanf(start, "%u:%u\r\n", &bucket,&gen) == 2) {
  814
+            /* we never write anything back, even if input's wrong */
  815
+            if ((bucket < 0) || (bucket >= MAX_BUCKETS) || (gen<=0)) {
  816
+                /* do nothing, bad input */
  817
+            } else {
  818
+                c->bucket = bucket;
  819
+                c->gen = gen;
  820
+            }
  821
+            c->state = conn_read;
  822
+            /* normally conn_write uncorks the connection, but this
  823
+               is the only time we accept a command w/o writing anything */
  824
+            set_cork(c,0); 
  825
+            return;
  826
+        } else {
  827
+            out_string(c, "CLIENT_ERROR bad format");
  828
+            return;
  829
+        }
  830
+    }
  831
+
705 832
     if (strncmp(command, "stats", 5) == 0) {
706 833
         process_stat(c, command);
707 834
         return;
@@ -765,40 +892,47 @@ void process_command(conn *c, char *command) {
765 892
 }
766 893
 
767 894
 /* 
768  
- * if we have a complete line in the buffer, process it and move whatever
769  
- * remains in the buffer to its beginning.
  895
+ * if we have a complete line in the buffer, process it.
770 896
  */
771 897
 int try_read_command(conn *c) {
772 898
     char *el, *cont;
773 899
 
774 900
     if (!c->rbytes)
775 901
         return 0;
776  
-    el = memchr(c->rbuf, '\n', c->rbytes);
  902
+    el = memchr(c->rcurr, '\n', c->rbytes);
777 903
     if (!el)
778 904
         return 0;
779 905
     cont = el + 1;
780  
-    if (el - c->rbuf > 1 && *(el - 1) == '\r') {
  906
+    if (el - c->rcurr > 1 && *(el - 1) == '\r') {
781 907
         el--;
782 908
     }
783 909
     *el = '\0';
784 910
 
785  
-    process_command(c, c->rbuf);
  911
+    process_command(c, c->rcurr);
  912
+
  913
+    c->rbytes -= (cont - c->rcurr);
  914
+    c->rcurr = cont;
786 915
 
787  
-    if (cont - c->rbuf < c->rbytes) { /* more stuff in the buffer */
788  
-        memmove(c->rbuf, cont, c->rbytes - (cont - c->rbuf));
789  
-    }
790  
-    c->rbytes -= (cont - c->rbuf);
791 916
     return 1;
792 917
 }
793 918
 
794 919
 /*
795 920
  * read from network as much as we can, handle buffer overflow and connection
796 921
  * close. 
  922
+ * before reading, move the remaining incomplete fragment of a command
  923
+ * (if any) to the beginning of the buffer.
797 924
  * return 0 if there's nothing to read on the first read.
798 925
  */
799 926
 int try_read_network(conn *c) {
800 927
     int gotdata = 0;
801 928
     int res;
  929
+
  930
+    if (c->rcurr != c->rbuf) {
  931
+        if (c->rbytes != 0) /* otherwise there's nothing to copy */
  932
+            memmove(c->rbuf, c->rcurr, c->rbytes);
  933
+        c->rcurr = c->rbuf;
  934
+    }
  935
+
802 936
     while (1) {
803 937
         if (c->rbytes >= c->rsize) {
804 938
             char *new_rbuf = realloc(c->rbuf, c->rsize*2);
@@ -899,7 +1033,7 @@ void drive_machine(conn *c) {
899 1033
             break;
900 1034
 
901 1035
         case conn_nread:
902  
-            /* we are reading rlbytes into rcurr; */
  1036
+            /* we are reading rlbytes into ritem; */
903 1037
             if (c->rlbytes == 0) {
904 1038
                 complete_nread(c);
905 1039
                 break;
@@ -907,21 +1041,19 @@ void drive_machine(conn *c) {
907 1041
             /* first check if we have leftovers in the conn_read buffer */
908 1042
             if (c->rbytes > 0) {
909 1043
                 int tocopy = c->rbytes > c->rlbytes ? c->rlbytes : c->rbytes;
910  
-                memcpy(c->rcurr, c->rbuf, tocopy);
911  
-                c->rcurr += tocopy;
  1044
+                memcpy(c->ritem, c->rcurr, tocopy);
  1045
+                c->ritem += tocopy;
912 1046
                 c->rlbytes -= tocopy;
913  
-                if (c->rbytes > tocopy) {
914  
-                    memmove(c->rbuf, c->rbuf+tocopy, c->rbytes - tocopy);
915  
-                }
  1047
+                c->rcurr += tocopy;
916 1048
                 c->rbytes -= tocopy;
917 1049
                 break;
918 1050
             }
919 1051
 
920 1052
             /*  now try reading from the socket */
921  
-            res = read(c->sfd, c->rcurr, c->rlbytes);
  1053
+            res = read(c->sfd, c->ritem, c->rlbytes);
922 1054
             if (res > 0) {
923 1055
                 stats.bytes_read += res;
924  
-                c->rcurr += res;
  1056
+                c->ritem += res;
925 1057
                 c->rlbytes -= res;
926 1058
                 break;
927 1059
             }
@@ -956,9 +1088,7 @@ void drive_machine(conn *c) {
956 1088
             if (c->rbytes > 0) {
957 1089
                 int tocopy = c->rbytes > c->sbytes ? c->sbytes : c->rbytes;
958 1090
                 c->sbytes -= tocopy;
959  
-                if (c->rbytes > tocopy) {
960  
-                    memmove(c->rbuf, c->rbuf+tocopy, c->rbytes - tocopy);
961  
-                }
  1091
+                c->rcurr += tocopy;
962 1092
                 c->rbytes -= tocopy;
963 1093
                 break;
964 1094
             }
@@ -1248,6 +1378,7 @@ void usage(void) {
1248 1378
     printf("-vv           very verbose (also print client commands/reponses)\n");
1249 1379
     printf("-h            print this help and exit\n");
1250 1380
     printf("-i            print memcached and libevent license\n");
  1381
+    printf("-b            run a managed instanced (mnemonic: buckets)\n");
1251 1382
     printf("-P <file>     save PID in <file>, only used with -d option\n");
1252 1383
     return;
1253 1384
 }
@@ -1372,8 +1503,11 @@ int main (int argc, char **argv) {
1372 1503
     setbuf(stderr, NULL);
1373 1504
 
1374 1505
     /* process arguments */
1375  
-    while ((c = getopt(argc, argv, "p:m:Mc:khirvdl:u:P:")) != -1) {
  1506
+    while ((c = getopt(argc, argv, "bp:m:Mc:khirvdl:u:P:")) != -1) {
1376 1507
         switch (c) {
  1508
+        case 'b':
  1509
+            settings.managed = 1;
  1510
+            break;
1377 1511
         case 'p':
1378 1512
             settings.port = atoi(optarg);
1379 1513
             break;
@@ -1521,6 +1655,16 @@ int main (int argc, char **argv) {
1521 1655
     conn_init();
1522 1656
     slabs_init(settings.maxbytes);
1523 1657
 
  1658
+    /* managed instance? alloc and zero a bucket array */
  1659
+    if (settings.managed) {
  1660
+        buckets = malloc(sizeof(int)*MAX_BUCKETS);
  1661
+        if (buckets == 0) {
  1662
+            fprintf(stderr, "failed to allocate the bucket array");
  1663
+            exit(1);
  1664
+        }
  1665
+        memset(buckets, 0, sizeof(int)*MAX_BUCKETS);
  1666
+    }
  1667
+
1524 1668
     /* lock paged memory if needed */
1525 1669
     if (lock_memory) {
1526 1670
 #ifdef HAVE_MLOCKALL
18  memcached.h
@@ -29,6 +29,7 @@ struct settings {
29 29
     int port;
30 30
     struct in_addr interface;
31 31
     int verbose;
  32
+    int managed;          /* if 1, a tracker manages virtual buckets */
32 33
     time_t oldest_live;   /* ignore existing items older than this */
33 34
     int evict_to_free;
34 35
 };
@@ -83,11 +84,12 @@ typedef struct {
83 84
     int    state;
84 85
     struct event event;
85 86
     short  ev_flags;
86  
-    short  which;  /* which events were just triggered */
  87
+    short  which;   /* which events were just triggered */
87 88
 
88  
-    char   *rbuf;  
89  
-    int    rsize;  
90  
-    int    rbytes;
  89
+    char   *rbuf;   /* buffer to read commands into */
  90
+    char   *rcurr;  /* but if we parsed some already, this is where we stopped */
  91
+    int    rsize;   /* total allocated size of rbuf */
  92
+    int    rbytes;  /* how much data, starting from rcur, do we have unparsed */
91 93
 
92 94
     char   *wbuf;
93 95
     char   *wcurr;
@@ -97,7 +99,7 @@ typedef struct {
97 99
     void   *write_and_free; /* free this memory after finishing writing */
98 100
     char    is_corked;         /* boolean, connection is corked */
99 101
 
100  
-    char   *rcurr;
  102
+    char   *ritem;  /* when we read in an item's value, it goes here */
101 103
     int    rlbytes;
102 104
     
103 105
     /* data for the nread state */
@@ -123,9 +125,15 @@ typedef struct {
123 125
     char   ibuf[300]; /* for VALUE lines */
124 126
     char   *iptr;
125 127
     int    ibytes;
  128
+    int    bucket;    /* bucket number for the next command, if running as
  129
+                         a managed instance. -1 (_not_ 0) means invalid. */
  130
+    int    gen;       /* generation requested for the bucket */
126 131
                          
127 132
 } conn;
128 133
 
  134
+/* number of virtual buckets for a managed instance */
  135
+#define MAX_BUCKETS 32768
  136
+
129 137
 /* listening socket */
130 138
 extern int l_socket;
131 139
 

0 notes on commit 7a30802

Please sign in to comment.
Something went wrong with that request. Please try again.