GitHub Sale: sign up for any paid plan this week and pay nothing until January 1, 2009!  [ hide ]

public
Description: Beanstalkd is a fast, distributed, in-memory workqueue service. Its interface is generic, but was designed for use in reducing the latency of page views in high-volume web applications by running most time-consuming tasks asynchronously.
Homepage: http://xph.us/software/beanstalkd/
Clone URL: git://github.com/kr/beanstalkd.git
Click here to lend your support to: beanstalkd and make a donation at www.pledgie.com !
Added reserve timeout support.
dustin (author)
Sat May 31 14:42:09 -0700 2008
commit  209a0d61e95cd152cb05e40e8c2c64707f6ebbb1
tree    f3bc38556940f382152defebff5f568f84e330ee
parent  615c445e5fbdf45e3057f965a4d45fb56d8ee985
0
...
20
21
22
 
23
24
25
...
84
85
86
 
87
88
89
...
146
147
148
149
150
 
 
151
152
153
...
155
156
157
 
 
 
 
 
158
159
160
 
161
162
163
...
20
21
22
23
24
25
26
...
85
86
87
88
89
90
91
...
148
149
150
 
 
151
152
153
154
155
...
157
158
159
160
161
162
163
164
165
166
 
167
168
169
170
0
@@ -20,6 +20,7 @@
0
 #include <stdio.h>
0
 #include <time.h>
0
 #include <errno.h>
0
+#include <limits.h>
0
 
0
 #include "conn.h"
0
 #include "net.h"
0
@@ -84,6 +85,7 @@ make_conn(int fd, char start_state, tube use, tube watch)
0
     c->state = start_state;
0
     c->type = 0;
0
     c->cmd_read = 0;
0
+ c->pending_timeout = -1;
0
     c->in_job = c->out_job = NULL;
0
     c->in_job_read = c->out_job_sent = 0;
0
     c->prev = c->next = c; /* must be out of a linked list right now */
0
@@ -146,8 +148,8 @@ has_reserved_job(conn c)
0
 int
0
 conn_set_evq(conn c, const int events, evh handler)
0
 {
0
- int r, margin = 0;
0
- struct timeval tv = {0, 0};
0
+ int r, margin = 0, should_timeout=0;
0
+ struct timeval tv = {INT_MAX, 0};
0
 
0
     event_set(&c->evq, c->fd, events, handler, c);
0
 
0
@@ -155,9 +157,14 @@ conn_set_evq(conn c, const int events, evh handler)
0
     if (has_reserved_job(c)) {
0
         time_t t = soonest_job(c)->deadline - time(NULL) - margin;
0
         tv.tv_sec = t > 0 ? t : 0;
0
+ should_timeout = 1;
0
+ }
0
+ if (c->pending_timeout >= 0) {
0
+ tv.tv_sec = min(tv.tv_sec, c->pending_timeout);
0
+ should_timeout = 1;
0
     }
0
 
0
- r = event_add(&c->evq, has_reserved_job(c) ? &tv : NULL);
0
+ r = event_add(&c->evq, should_timeout ? &tv : NULL);
0
     if (r == -1) return twarn("event_add() err %d", errno), -1;
0
 
0
     return 0;
0
...
44
45
46
 
47
48
49
...
44
45
46
47
48
49
50
0
@@ -44,6 +44,7 @@ struct conn {
0
     char state;
0
     char type;
0
     struct event evq;
0
+ int pending_timeout;
0
 
0
     /* we cannot share this buffer with the reply line because we might read in
0
      * command line data for a subsequent command, and we need to store it
...
194
195
196
 
 
 
 
197
198
199
...
201
202
203
 
 
 
 
 
204
205
206
...
212
213
214
215
216
 
 
 
 
 
 
 
217
218
219
...
194
195
196
197
198
199
200
201
202
203
...
205
206
207
208
209
210
211
212
213
214
215
...
221
222
223
 
 
224
225
226
227
228
229
230
231
232
233
0
@@ -194,6 +194,10 @@ A process that wants to consume jobs from the queue uses "reserve", "delete",
0
 
0
 reserve\r\n
0
 
0
+Alternatively, you can specify a timeout as follows:
0
+
0
+reserve-with-timeout <seconds>\r\n
0
+
0
 This will return a newly-reserved job. If no job is available to be reserved,
0
 beanstalkd will wait to send a response until one becomes available. Once a
0
 job is reserved for the client, the client has limited time to run (TTR) the
0
@@ -201,6 +205,11 @@ job before the job times out. When the job times out, the server will put the
0
 job back into the ready queue. Both the TTR and the actual time left can be
0
 found in response to the stats-job command.
0
 
0
+A timeout value of 0 will cause the server to immediately return either a
0
+response or TIMED_OUT. A positive value of timeout will limit the amount of
0
+time the client will block on the reserve request until a job becomes
0
+available.
0
+
0
 During the TTR of a reserved job, the last second is kept by the server as a
0
 safety margin, during which the client will not be made to wait for another
0
 job. If the client issues a reserve command during the safety margin, or if
0
@@ -212,8 +221,13 @@ DEADLINE_SOON\r\n
0
 This gives the client a chance to delete or release its reserved job before
0
 the server automatically releases it.
0
 
0
-Otherwise, the only response to this command is a successful reservation in
0
-the form of a text line followed by the job body:
0
+TIMED_OUT\r\n
0
+
0
+If a non-negative timeout was specified and the timeout exceeded before a job
0
+became available, the server will respond with TIMED_OUT.
0
+
0
+Otherwise, the only other response to this command is a successful reservation
0
+in the form of a text line followed by the job body:
0
 
0
 RESERVED <id> <bytes>\r\n
0
 <data>\r\n
0
...
50
51
52
 
53
54
55
...
71
72
73
 
74
75
76
...
89
90
91
 
92
93
94
...
137
138
139
140
 
 
141
142
143
...
151
152
153
 
154
155
156
...
704
705
706
 
707
708
709
...
809
810
811
 
812
813
814
...
882
883
884
885
 
886
887
888
889
890
891
 
 
 
892
893
894
...
1068
1069
1070
1071
 
1072
1073
1074
...
1189
1190
1191
1192
 
 
 
 
 
1193
1194
 
1195
1196
1197
...
1203
1204
1205
1206
 
1207
1208
1209
...
1415
1416
1417
 
 
 
 
1418
1419
1420
...
50
51
52
53
54
55
56
...
72
73
74
75
76
77
78
...
91
92
93
94
95
96
97
...
140
141
142
 
143
144
145
146
147
...
155
156
157
158
159
160
161
...
709
710
711
712
713
714
715
...
815
816
817
818
819
820
821
...
889
890
891
 
892
893
894
895
896
897
898
899
900
901
902
903
904
...
1078
1079
1080
 
1081
1082
1083
1084
...
1199
1200
1201
 
1202
1203
1204
1205
1206
1207
 
1208
1209
1210
1211
...
1217
1218
1219
 
1220
1221
1222
1223
...
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
0
@@ -50,6 +50,7 @@ size_t job_data_size_limit = ((1 << 16) - 1);
0
 #define CMD_PEEK_DELAYED "peek-delayed"
0
 #define CMD_PEEK_BURIED "peek-buried"
0
 #define CMD_RESERVE "reserve"
0
+#define CMD_RESERVE_TIMEOUT "reserve-with-timeout "
0
 #define CMD_DELETE "delete "
0
 #define CMD_RELEASE "release "
0
 #define CMD_BURY "bury "
0
@@ -71,6 +72,7 @@ size_t job_data_size_limit = ((1 << 16) - 1);
0
 #define CMD_PEEK_BURIED_LEN CONSTSTRLEN(CMD_PEEK_BURIED)
0
 #define CMD_PEEKJOB_LEN CONSTSTRLEN(CMD_PEEKJOB)
0
 #define CMD_RESERVE_LEN CONSTSTRLEN(CMD_RESERVE)
0
+#define CMD_RESERVE_TIMEOUT_LEN CONSTSTRLEN(CMD_RESERVE_TIMEOUT)
0
 #define CMD_DELETE_LEN CONSTSTRLEN(CMD_DELETE)
0
 #define CMD_RELEASE_LEN CONSTSTRLEN(CMD_RELEASE)
0
 #define CMD_BURY_LEN CONSTSTRLEN(CMD_BURY)
0
@@ -89,6 +91,7 @@ size_t job_data_size_limit = ((1 << 16) - 1);
0
 #define MSG_NOTFOUND "NOT_FOUND\r\n"
0
 #define MSG_RESERVED "RESERVED"
0
 #define MSG_DEADLINE_SOON "DEADLINE_SOON\r\n"
0
+#define MSG_TIMED_OUT "TIMED_OUT\r\n"
0
 #define MSG_DELETED "DELETED\r\n"
0
 #define MSG_RELEASED "RELEASED\r\n"
0
 #define MSG_BURIED "BURIED\r\n"
0
@@ -137,7 +140,8 @@ size_t job_data_size_limit = ((1 << 16) - 1);
0
 #define OP_STATS_TUBE 17
0
 #define OP_PEEK_READY 18
0
 #define OP_PEEK_DELAYED 19
0
-#define TOTAL_OPS 20
0
+#define OP_RESERVE_TIMEOUT 20
0
+#define TOTAL_OPS 21
0
 
0
 #define STATS_FMT "---\n" \
0
     "current-jobs-urgent: %u\n" \
0
@@ -151,6 +155,7 @@ size_t job_data_size_limit = ((1 << 16) - 1);
0
     "cmd-peek-delayed: %llu\n" \
0
     "cmd-peek-buried: %llu\n" \
0
     "cmd-reserve: %llu\n" \
0
+ "cmd-reserve-with-timeout: %llu\n" \
0
     "cmd-delete: %llu\n" \
0
     "cmd-release: %llu\n" \
0
     "cmd-use: %llu\n" \
0
@@ -704,6 +709,7 @@ which_cmd(conn c)
0
     TEST_CMD(c->cmd, CMD_PEEK_READY, OP_PEEK_READY);
0
     TEST_CMD(c->cmd, CMD_PEEK_DELAYED, OP_PEEK_DELAYED);
0
     TEST_CMD(c->cmd, CMD_PEEK_BURIED, OP_PEEK_BURIED);
0
+ TEST_CMD(c->cmd, CMD_RESERVE_TIMEOUT, OP_RESERVE_TIMEOUT);
0
     TEST_CMD(c->cmd, CMD_RESERVE, OP_RESERVE);
0
     TEST_CMD(c->cmd, CMD_DELETE, OP_DELETE);
0
     TEST_CMD(c->cmd, CMD_RELEASE, OP_RELEASE);
0
@@ -809,6 +815,7 @@ fmt_stats(char *buf, size_t size, void *x)
0
             op_ct[OP_PEEK_DELAYED],
0
             op_ct[OP_PEEK_BURIED],
0
             op_ct[OP_RESERVE],
0
+ op_ct[OP_RESERVE_TIMEOUT],
0
             op_ct[OP_DELETE],
0
             op_ct[OP_RELEASE],
0
             op_ct[OP_USE],
0
@@ -882,13 +889,16 @@ read_ttr(unsigned int *ttr, const char *buf, char **end)
0
 }
0
 
0
 static void
0
-wait_for_job(conn c)
0
+wait_for_job(conn c, int timeout)
0
 {
0
     int r;
0
 
0
     c->state = STATE_WAIT;
0
     enqueue_waiting_conn(c);
0
 
0
+ /* Set the pending timeout to the requested timeout amount */
0
+ c->pending_timeout = timeout;
0
+
0
     /* this conn is waiting, but we want to know if they hang up */
0
     r = conn_update_evq(c, EV_READ | EV_PERSIST);
0
     if (r == -1) return twarnx("update events failed"), conn_close(c);
0
@@ -1068,7 +1078,7 @@ find_or_make_tube(const char *name)
0
 static void
0
 dispatch_cmd(conn c)
0
 {
0
- int r, i;
0
+ int r, i, timeout = -1;
0
     unsigned int count;
0
     job j;
0
     unsigned char type;
0
@@ -1189,9 +1199,13 @@ dispatch_cmd(conn c)
0
 
0
         reply_job(c, j, MSG_FOUND);
0
         break;
0
- case OP_RESERVE:
0
+ case OP_RESERVE_TIMEOUT:
0
+ errno = 0;
0
+ timeout = strtol(c->cmd + CMD_RESERVE_TIMEOUT_LEN, &end_buf, 10);
0
+ if (errno) return reply_msg(c, MSG_BAD_FORMAT);
0
+ case OP_RESERVE: /* FALLTHROUGH */
0
         /* don't allow trailing garbage */
0
- if (c->cmd_len != CMD_RESERVE_LEN + 2) {
0
+ if (type == OP_RESERVE && c->cmd_len != CMD_RESERVE_LEN + 2) {
0
             return reply_msg(c, MSG_BAD_FORMAT);
0
         }
0
 
0
@@ -1203,7 +1217,7 @@ dispatch_cmd(conn c)
0
         }
0
 
0
         /* try to get a new job for this guy */
0
- wait_for_job(c);
0
+ wait_for_job(c, timeout);
0
         process_queue();
0
         break;
0
     case OP_DELETE:
0
@@ -1415,6 +1429,10 @@ h_conn_timeout(conn c)
0
     if (should_timeout) {
0
         dprintf("conn_waiting(%p) = %d\n", c, conn_waiting(c));
0
         return reply_msg(remove_waiting_conn(c), MSG_DEADLINE_SOON);
0
+ } else if (conn_waiting(c) && c->pending_timeout >= 0) {
0
+ dprintf("conn_waiting(%p) = %d\n", c, conn_waiting(c));
0
+ c->pending_timeout=-1;
0
+ return reply_msg(remove_waiting_conn(c), MSG_TIMED_OUT);
0
     }
0
 }
0
 

Comments

    No one has commented yet.