Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 1783 lines (1491 sloc) 59.459 kb
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1 /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
6fcace5 @trondn Move to pandora build
trondn authored
2 #include "config.h"
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
3 #include <stdio.h>
0577ccb @steveyen avoid libmemcached dependency
steveyen authored
4 #include <stdlib.h>
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
5 #include <string.h>
6 #include <errno.h>
7 #include <pthread.h>
8 #include <assert.h>
9 #include "memcached.h"
10 #include "cproxy.h"
11 #include "work.h"
f03201c @steveyen separated out agent.h
steveyen authored
12 #include "agent.h"
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
13 #include "log.h"
0ebd7f6 @steveyen added cJSON
steveyen authored
14 #include "cJSON.h"
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
15
57e17da @dustin Build against libconflate.
dustin authored
16 // Integration with libconflate.
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
17 //
88ab7db @steveyen keeping config snapshot at proxy_td level for fewer locks
steveyen authored
18 static void update_ptd_config(void *data0, void *data1);
5a56bdd @steveyen better debug fprintfs
steveyen authored
19 static bool update_str_config(char **curr, char *next, char *descrip);
88ab7db @steveyen keeping config snapshot at proxy_td level for fewer locks
steveyen authored
20
5a56bdd @steveyen better debug fprintfs
steveyen authored
21 static bool update_behaviors_config(proxy_behavior **curr,
88ab7db @steveyen keeping config snapshot at proxy_td level for fewer locks
steveyen authored
22 int *curr_num,
23 proxy_behavior *next,
24 int next_num,
25 char *descrip);
26
d907a27 @steveyen close_outdated_proxies() helper function
steveyen authored
27 void close_outdated_proxies(proxy_main *m, uint32_t new_config_ver);
28
90dcb2b @steveyen parse_kvs_servers refactored into separate func
steveyen authored
29 char *parse_kvs_servers(char *prefix,
30 char *pool_name,
31 kvpair_t *kvs,
32 char **servers,
12f9191 @steveyen using proxy_behavior_pool for parse_kvs_servers
steveyen authored
33 proxy_behavior_pool *behavior_pool);
90dcb2b @steveyen parse_kvs_servers refactored into separate func
steveyen authored
34
1b12147 @steveyen parse_kvs_behavior helper func, reused for drain server parse
steveyen authored
35 char **parse_kvs_behavior(kvpair_t *kvs,
36 char *prefix,
37 char *name,
38 proxy_behavior *behavior);
39
e3fd701 @steveyen bug 2602 - use name/saslPassword, not vBucketServerMap.user/password
steveyen authored
40 static void cproxy_parse_json_auth(char *config,
41 char *name,
42 proxy_behavior_pool *bp);
43
522ddf0 @steveyen added a NULL_BUCKET placeholder for SaaS use-case
steveyen authored
44 static void cproxy_init_null_bucket(proxy_main *m);
45
ecf81b7 @steveyen renamed cproxy_on_new_config_foobar() to cproxy_on_config_foobar()
steveyen authored
46 static void cproxy_on_config(void *data0, void *data1);
47
d8970a5 @steveyen agent_logger callback for conflate logging
steveyen authored
48 static void agent_logger(void *userdata,
49 enum conflate_log_level lvl,
50 const char *msg, ...)
51 {
e1ae2ef @alk fixes tons of warnings produced on GNU/Linux
alk authored
52 (void)userdata;
53 (void)lvl;
54 (void)msg;
73d2921 @steveyen turn off AGENT_LOGGER
steveyen authored
55 // Issues compiling vfprintf(), so turn off this unused code path for now.
56 #undef AGENT_LOGGER
57 #ifdef AGENT_LOGGER
d8970a5 @steveyen agent_logger callback for conflate logging
steveyen authored
58 char *n = NULL;
59 bool v = false;
60
61 switch(lvl) {
62 case FATAL: n = "FATAL"; v = settings.verbose > 0; break;
63 case ERROR: n = "ERROR"; v = settings.verbose > 0; break;
64 case WARN: n = "WARN"; v = settings.verbose > 1; break;
65 case INFO: n = "INFO"; v = settings.verbose > 1; break;
66 case DEBUG: n = "DEBUG"; v = settings.verbose > 2; break;
67 }
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
68 if (!v) {
d8970a5 @steveyen agent_logger callback for conflate logging
steveyen authored
69 return;
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
70 }
d8970a5 @steveyen agent_logger callback for conflate logging
steveyen authored
71
72 char fmt[strlen(msg) + 16];
73 snprintf(fmt, sizeof(fmt), "%s: %s\n", n, msg);
74
75 va_list ap;
76 va_start(ap, msg);
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
77 vfprintf(fmt, ap);
d8970a5 @steveyen agent_logger callback for conflate logging
steveyen authored
78 va_end(ap);
73d2921 @steveyen turn off AGENT_LOGGER
steveyen authored
79 #endif
d8970a5 @steveyen agent_logger callback for conflate logging
steveyen authored
80 }
81
a3bc15d @dustin Conflate functionality for increased modularity.
dustin authored
82 static void init_extensions(void)
83 {
54fe028 @steveyen assert, and better conflate_register descript string
steveyen authored
84 conflate_register_mgmt_cb("client_stats", "Retrieve stats from moxi",
a3bc15d @dustin Conflate functionality for increased modularity.
dustin authored
85 on_conflate_get_stats);
54fe028 @steveyen assert, and better conflate_register descript string
steveyen authored
86 conflate_register_mgmt_cb("reset_stats", "Reset moxi stats",
a3bc15d @dustin Conflate functionality for increased modularity.
dustin authored
87 on_conflate_reset_stats);
88 conflate_register_mgmt_cb("ping_test", "Perform a ping test",
89 on_conflate_ping_test);
90 }
91
286f08e @steveyen jid/pswd can optionally be read from a secure file
steveyen authored
92 /** The cfg_str looks like...
93 *
94 * apikey=jidname@jhostname%jpassword,config=config,host=host
95 * or...
96 * jidname@jhostname%jpassword,config=config,host=host
97 *
98 * Only the apikey is needed, so it can also look like...
99 *
100 * jidname@jhostname%jpassword
91951d5 @steveyen bug 1366 - simplified moxi cmd-line
steveyen authored
101 *
102 * Or...
103 *
104 * http://host:port/default/pools/bucketsStreamingConfig/default
105 * url=http://host:port/default/pools/bucketsStreamingConfig/default
106 * auth=,url=http://host:port/default/pools/bucketsStreamingConfig/default
107 * auth=USER%PSWD,url=http://host:port/default/pools/bucketsStreamingConfig/default
108 * auth=Administrator%password,url=http://host:port/default/pools/bucketsStreamingConfig/default
286f08e @steveyen jid/pswd can optionally be read from a secure file
steveyen authored
109 */
005a5e1 @steveyen added sasl_mech/auth to proxy behavior, which is now more inheritable
steveyen authored
110 int cproxy_init_agent(char *cfg_str,
8a7e4b6 @steveyen Moved nthreads out of proxy_behavior.
steveyen authored
111 proxy_behavior behavior,
112 int nthreads) {
a3bc15d @dustin Conflate functionality for increased modularity.
dustin authored
113 init_extensions();
114
286f08e @steveyen jid/pswd can optionally be read from a secure file
steveyen authored
115 if (cfg_str == NULL) {
10cd773 @steveyen bug 1958 - using stderr if moxi exit()'s
steveyen authored
116 moxi_log_write("ERROR: missing cfg\n");
117 if (ml->log_mode != ERRORLOG_STDERR) {
118 fprintf(stderr, "ERROR: missing cfg\n");
119 }
286f08e @steveyen jid/pswd can optionally be read from a secure file
steveyen authored
120 exit(EXIT_FAILURE);
121 }
005a5e1 @steveyen added sasl_mech/auth to proxy behavior, which is now more inheritable
steveyen authored
122
8b05c83 @steveyen cmdline tweak
steveyen authored
123 int cfg_len = strlen(cfg_str);
286f08e @steveyen jid/pswd can optionally be read from a secure file
steveyen authored
124 if (cfg_len <= 0) {
10cd773 @steveyen bug 1958 - using stderr if moxi exit()'s
steveyen authored
125 moxi_log_write("ERROR: empty cfg\n");
126 if (ml->log_mode != ERRORLOG_STDERR) {
127 fprintf(stderr, "ERROR: empty cfg\n");
128 }
286f08e @steveyen jid/pswd can optionally be read from a secure file
steveyen authored
129 exit(EXIT_FAILURE);
130 }
8b05c83 @steveyen cmdline tweak
steveyen authored
131
132 char *buff;
133
443e4f3 @steveyen aliases for conflate init parameters
steveyen authored
134 if (strncmp(cfg_str, "apikey=", 7) == 0 ||
135 strncmp(cfg_str, "auth=", 5) == 0 ||
136 strncmp(cfg_str, "url=", 4) == 0) {
947c762 @steveyen trimstr()
steveyen authored
137 buff = trimstrdup(cfg_str);
8b05c83 @steveyen cmdline tweak
steveyen authored
138 } else {
139 buff = calloc(cfg_len + 50, sizeof(char));
140 if (buff != NULL) {
91951d5 @steveyen bug 1366 - simplified moxi cmd-line
steveyen authored
141 if (strncmp(cfg_str, "http://", 7) == 0) {
142 snprintf(buff, cfg_len + 50, "url=%s", cfg_str);
0b42812 @steveyen bug 1876 - allow command separated URL's in 'convenience mode'
steveyen authored
143
144 // Allow the user to specify multiple comma-separated URL's,
145 // which we auto-translate right now to the '|' separators
146 // that the rest of the code expects.
147 //
148 for (char *x = buff; *x; x++) {
149 if (*x == ',') {
150 *x = '|';
151 }
152 }
91951d5 @steveyen bug 1366 - simplified moxi cmd-line
steveyen authored
153 } else {
656ca64 @steveyen Exit if empty -z configuration
steveyen authored
154 strcpy(buff, cfg_str);
91951d5 @steveyen bug 1366 - simplified moxi cmd-line
steveyen authored
155 }
8b05c83 @steveyen cmdline tweak
steveyen authored
156 }
947c762 @steveyen trimstr()
steveyen authored
157 buff = trimstr(buff);
8b05c83 @steveyen cmdline tweak
steveyen authored
158 }
159
160 char *next = buff;
161
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
162 int rv = 0;
163
164 while (next != NULL) {
6df1d2f @steveyen default to auth from -Z or env vars
steveyen authored
165 char *jid = behavior.usr;
166 char *jpw = behavior.pwd;
8c53072 @steveyen handle jid/pwd in optional user:password@fqdn format
steveyen authored
167 char *jpwmem = NULL;
fb1c23b @steveyen var name dbpath clarification
steveyen authored
168 char *dbpath = NULL;
40923c0 @steveyen no longer using glib hashtable iter
steveyen authored
169 char *host = NULL;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
170
947c762 @steveyen trimstr()
steveyen authored
171 char *cur = trimstr(strsep(&next, ";"));
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
172 while (cur != NULL) {
ae9534c @steveyen allow for comments and newlines in config
steveyen authored
173 char *key_val = trimstr(strsep(&cur, ",\r\n"));
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
174 if (key_val != NULL) {
947c762 @steveyen trimstr()
steveyen authored
175 char *key = trimstr(strsep(&key_val, "="));
176 char *val = trimstr(key_val);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
177
ae9534c @steveyen allow for comments and newlines in config
steveyen authored
178 bool handled = true;
179
180 if (key != NULL &&
181 val != NULL) {
443e4f3 @steveyen aliases for conflate init parameters
steveyen authored
182 if (wordeq(key, "apikey") ||
183 wordeq(key, "auth")) {
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
184 jid = strsep(&val, "%");
185 jpw = val;
fb1c23b @steveyen var name dbpath clarification
steveyen authored
186 } else if (wordeq(key, "config") ||
187 wordeq(key, "dbpath")) {
188 dbpath = val;
443e4f3 @steveyen aliases for conflate init parameters
steveyen authored
189 } else if (wordeq(key, "host") ||
190 wordeq(key, "url")) {
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
191 host = val;
ae9534c @steveyen allow for comments and newlines in config
steveyen authored
192 } else {
193 handled = false;
194 }
195 } else {
196 handled = false;
197 }
198
199 if (handled == false &&
200 key != NULL &&
201 key[0] != '#' &&
202 key[0] != '\0') {
203 if (settings.verbose > 0) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
204 moxi_log_write("unknown configuration key: %s\n", key);
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
205 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
206 }
207 }
208 }
209
fb1c23b @steveyen var name dbpath clarification
steveyen authored
210 if (jid == NULL) {
211 jid = "";
40923c0 @steveyen no longer using glib hashtable iter
steveyen authored
212 }
213
8c53072 @steveyen handle jid/pwd in optional user:password@fqdn format
steveyen authored
214 if (jpw == NULL) {
215 // Handle if jid/jpw is in user:password@fqdn format
216 // instead of user@fqdn%password format.
217 //
218 char *colon = strchr(jid, ':');
219 char *asign = strchr(jid, '@');
220 if (colon != NULL &&
221 asign != NULL &&
222 asign > colon) {
223 *asign = '\0';
224 jpw = jpwmem = strdup(colon + 1);
225 *asign = '@';
226 do {
227 *colon = *asign;
228 colon++;
229 asign++;
230 } while (*asign != '\0');
231 *colon = '\0';
232 }
233 }
234
fb1c23b @steveyen var name dbpath clarification
steveyen authored
235 if (jpw == NULL) {
236 jpw = "";
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
237 }
40923c0 @steveyen no longer using glib hashtable iter
steveyen authored
238
fb1c23b @steveyen var name dbpath clarification
steveyen authored
239 int dbpath_alloc = 0;
240 if (dbpath == NULL) {
241 dbpath_alloc = strlen(jid) + strlen(CONFLATE_DB_PATH) + 100;
242 dbpath = calloc(dbpath_alloc, 1);
243 if (dbpath != NULL) {
244 snprintf(dbpath, dbpath_alloc,
245 CONFLATE_DB_PATH "/conflate-%s.cfg",
246 (jid != NULL && strlen(jid) > 0 ? jid : "default"));
247
40923c0 @steveyen no longer using glib hashtable iter
steveyen authored
248 } else {
10cd773 @steveyen bug 1958 - using stderr if moxi exit()'s
steveyen authored
249 moxi_log_write("ERROR: conflate dbpath buf alloc\n");
40923c0 @steveyen no longer using glib hashtable iter
steveyen authored
250 exit(EXIT_FAILURE);
251 }
252 }
253
8c53072 @steveyen handle jid/pwd in optional user:password@fqdn format
steveyen authored
254 if (settings.verbose > 1) {
135282d @steveyen bug 2236 - don't printf unused dbpath anymore
steveyen authored
255 moxi_log_write("cproxy_init jid: %s host: %s\n", jid, host);
8c53072 @steveyen handle jid/pwd in optional user:password@fqdn format
steveyen authored
256 }
257
fb1c23b @steveyen var name dbpath clarification
steveyen authored
258 if (cproxy_init_agent_start(jid, jpw, dbpath, host,
40923c0 @steveyen no longer using glib hashtable iter
steveyen authored
259 behavior,
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
260 nthreads) != NULL) {
40923c0 @steveyen no longer using glib hashtable iter
steveyen authored
261 rv++;
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
262 }
40923c0 @steveyen no longer using glib hashtable iter
steveyen authored
263
fb1c23b @steveyen var name dbpath clarification
steveyen authored
264 if (dbpath_alloc > 0 &&
265 dbpath != NULL) {
266 free(dbpath);
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
267 }
8c53072 @steveyen handle jid/pwd in optional user:password@fqdn format
steveyen authored
268
269 if (jpwmem) {
270 free(jpwmem);
271 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
272 }
273
274 free(buff);
275
276 return rv;
277 }
278
3bc9c7a @steveyen cproxy_init_agent_start returns proxy_main for testability
steveyen authored
279 proxy_main *cproxy_init_agent_start(char *jid,
280 char *jpw,
fb1c23b @steveyen var name dbpath clarification
steveyen authored
281 char *dbpath,
3bc9c7a @steveyen cproxy_init_agent_start returns proxy_main for testability
steveyen authored
282 char *host,
283 proxy_behavior behavior,
284 int nthreads) {
fb1c23b @steveyen var name dbpath clarification
steveyen authored
285 assert(dbpath);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
286
8d6ae29 @CaptTofu * Changed verbosity levels a bit so that function entry is > 2
CaptTofu authored
287 if (settings.verbose > 2) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
288 moxi_log_write("cproxy_init_agent_start\n");;
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
289 }
90c377b @steveyen fixed wrong define
steveyen authored
290
f4d3531 Added new ascii commands of "stats proxy sub-command".
Joon(JunHyun Park) authored
291 proxy_main *m = cproxy_gen_proxy_main(behavior, nthreads,
292 PROXY_CONF_TYPE_DYNAMIC);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
293 if (m != NULL) {
1e0192d @steveyen bug 1998 - handle missing default bucket correctly
steveyen authored
294 // Create a NULL_BUCKET when we're not in "FIRST_BUCKET" mode.
295 //
296 // FIRST_BUCKET mode means clients start off in the first
297 // configured bucket (and this is usually the case for
298 // standalone moxi).
299 //
300 // Otherwise (when not in FIRST_BUCKET mode)...
301 // -- new clients start off in a configured, named default
302 // bucket (whose name is usually configured to be "default"),
303 // if it exists.
304 // -- if the named default bucket doesn't exist, new
305 // clients then start off in the NULL_BUCKET.
306 //
307 if (strcmp(behavior.default_bucket_name, FIRST_BUCKET) != 0) {
308 if (settings.verbose > 2) {
309 moxi_log_write("initializing null bucket, default is: %s\n",
310 behavior.default_bucket_name);
311 }
312
522ddf0 @steveyen added a NULL_BUCKET placeholder for SaaS use-case
steveyen authored
313 cproxy_init_null_bucket(m);
1e0192d @steveyen bug 1998 - handle missing default bucket correctly
steveyen authored
314 } else {
315 if (settings.verbose > 2) {
316 moxi_log_write("using first bucket\n");
317 }
522ddf0 @steveyen added a NULL_BUCKET placeholder for SaaS use-case
steveyen authored
318 }
319
57e17da @dustin Build against libconflate.
dustin authored
320 conflate_config_t config;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
321
322 memset(&config, 0, sizeof(config));
323
26bf25b @steveyen calling init_conflate()
steveyen authored
324 init_conflate(&config);
325
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
326 // Different jid's possible for production, staging, etc.
443e4f3 @steveyen aliases for conflate init parameters
steveyen authored
327 config.jid = jid; // "customer@stevenmb.local" or
328 // "Administrator"
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
329 config.pass = jpw; // "password"
443e4f3 @steveyen aliases for conflate init parameters
steveyen authored
330 config.host = host; // "localhost" or
02b6bd2 @steveyen bug 2529 - port 8080 to 8091
steveyen authored
331 // "http://x.com:8091"
332 // "http://x.com:8091/pools/default/buckets/default"
97f00f8 @steveyen conflate.h included with brackets
steveyen authored
333 config.software = PACKAGE;
440bb2a @steveyen passing VERSION to conflate config
steveyen authored
334 config.version = VERSION;
fb1c23b @steveyen var name dbpath clarification
steveyen authored
335 config.save_path = dbpath;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
336 config.userdata = m;
54fe028 @steveyen assert, and better conflate_register descript string
steveyen authored
337 config.new_config = on_conflate_new_config;
338 config.log = agent_logger;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
339
656ca64 @steveyen Exit if empty -z configuration
steveyen authored
340 if (!config.host || config.host[0] == '\0') {
341 moxi_log_write("ERROR: missing -z configuration for url/host\n");
342 if (ml->log_mode != ERRORLOG_STDERR) {
343 fprintf(stderr, "ERROR: missing -z configuration for url/host\n");
344 }
345 exit(EXIT_FAILURE);
346 }
347
57e17da @dustin Build against libconflate.
dustin authored
348 if (start_conflate(config)) {
8d6ae29 @CaptTofu * Changed verbosity levels a bit so that function entry is > 2
CaptTofu authored
349 if (settings.verbose > 2) {
522ddf0 @steveyen added a NULL_BUCKET placeholder for SaaS use-case
steveyen authored
350 moxi_log_write("cproxy_init_agent_start done\n");
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
351 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
352
3bc9c7a @steveyen cproxy_init_agent_start returns proxy_main for testability
steveyen authored
353 return m;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
354 }
3bc9c7a @steveyen cproxy_init_agent_start returns proxy_main for testability
steveyen authored
355
356 free(m);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
357 }
358
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
359 if (settings.verbose > 1) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
360 moxi_log_write("cproxy could not start conflate\n");
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
361 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
362
3bc9c7a @steveyen cproxy_init_agent_start returns proxy_main for testability
steveyen authored
363 return NULL;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
364 }
365
522ddf0 @steveyen added a NULL_BUCKET placeholder for SaaS use-case
steveyen authored
366 static void cproxy_init_null_bucket(proxy_main *m) {
367 proxy_behavior proxyb = m->behavior;
368
369 int pool_port = proxyb.port_listen;
370 int nodes_num = 0;
371
372 if (pool_port > 0) {
373 proxy_behavior_pool behavior_pool = {
374 .base = proxyb,
375 .num = nodes_num,
376 .arr = calloc(nodes_num + 1, sizeof(proxy_behavior))
377 };
378
379 if (behavior_pool.arr != NULL) {
34ffe4a @steveyen renamed cproxy_on_new_pool() to cproxy_on_config_pool()
steveyen authored
380 cproxy_on_config_pool(m, NULL_BUCKET, pool_port,
381 "", 0, &behavior_pool);
1e0192d @steveyen bug 1998 - handle missing default bucket correctly
steveyen authored
382 free(behavior_pool.arr);
522ddf0 @steveyen added a NULL_BUCKET placeholder for SaaS use-case
steveyen authored
383 }
384 }
385 }
0ebd7f6 @steveyen added cJSON
steveyen authored
386
db63df8 @steveyen MB-3575 - conflate new_config() call signature changed
steveyen authored
387 conflate_result on_conflate_new_config(void *userdata, kvpair_t *config) {
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
388 assert(config != NULL);
389
390 proxy_main *m = userdata;
391 assert(m != NULL);
392
393 LIBEVENT_THREAD *mthread = thread_by_index(0);
394 assert(mthread != NULL);
395
262e4c7 @steveyen MB-2672 - moxi logs at -v level when config is received
steveyen authored
396 if (settings.verbose > 0) {
397 moxi_log_write("configuration received\n");
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
398 }
90c377b @steveyen fixed wrong define
steveyen authored
399
db63df8 @steveyen MB-3575 - conflate new_config() call signature changed
steveyen authored
400 char **urlv = get_key_values(config, "url"); // NULL delimited array of char *.
401 char *url = urlv != NULL ? urlv[0] : NULL;
402
403 char **contentsv = get_key_values(config, "contents");
404 char *contents = contentsv != NULL ? contentsv[0] : NULL;
405
406 if (url != NULL &&
407 contents != NULL &&
408 strlen(contents) > 0) {
409 // Must be a REST/JSON config. Wastefully test parse it here,
410 // before we asynchronously invoke the real worker who can't
411 // respond nicely with an error code.
412 //
413 bool ok = false;
414
415 cJSON *c = cJSON_Parse(contents);
416 if (c != NULL) {
417 ok = true;
418 cJSON_Delete(c);
419 }
420
421 if (!ok) {
422 moxi_log_write("ERROR: parse JSON failed, from REST server: %s, %s\n",
423 url, contents);
424
425 return CONFLATE_ERROR_BAD_SOURCE;
426 }
427 }
428
51f0211 @steveyen using dup_kvpair
steveyen authored
429 kvpair_t *copy = dup_kvpair(config);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
430 if (copy != NULL) {
db63df8 @steveyen MB-3575 - conflate new_config() call signature changed
steveyen authored
431 if (work_send(mthread->work_queue, cproxy_on_config, m, copy)) {
432 return CONFLATE_SUCCESS;
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
433 }
db63df8 @steveyen MB-3575 - conflate new_config() call signature changed
steveyen authored
434
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
435 if (settings.verbose > 1) {
db63df8 @steveyen MB-3575 - conflate new_config() call signature changed
steveyen authored
436 moxi_log_write("work_send failed\n");
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
437 }
db63df8 @steveyen MB-3575 - conflate new_config() call signature changed
steveyen authored
438
439 return CONFLATE_ERROR;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
440 }
db63df8 @steveyen MB-3575 - conflate new_config() call signature changed
steveyen authored
441
442 if (settings.verbose > 1) {
443 moxi_log_write("agent_config ocnc failed dup_kvpair\n");
444 }
445
446 return CONFLATE_ERROR;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
447 }
448
2049eba @steveyen bug 2231 - MOXI_USE_LIBVBUCKET and MOXI_USE_LIBMEMCACHED
steveyen authored
449 #ifdef MOXI_USE_LIBVBUCKET
a05bf8c @steveyen calling cproxy_on_new_pool() after parsing vbucket config
steveyen authored
450
e1ae2ef @alk fixes tons of warnings produced on GNU/Linux
alk authored
451 static bool cproxy_on_config_json_one(proxy_main *m, uint32_t new_config_ver,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
452 char *config, char *name, char *src);
e1ae2ef @alk fixes tons of warnings produced on GNU/Linux
alk authored
453
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
454 static bool cproxy_on_config_json_one_vbucket(proxy_main *m,
455 uint32_t new_config_ver,
456 char *config,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
457 char *name,
458 char *src);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
459
460 static bool cproxy_on_config_json_one_ketama(proxy_main *m,
461 uint32_t new_config_ver,
462 char *config,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
463 char *name,
464 char *src);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
465
e1ae2ef @alk fixes tons of warnings produced on GNU/Linux
alk authored
466 static bool cproxy_on_config_json_buckets(proxy_main *m, uint32_t new_config_ver,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
467 cJSON *jBuckets, bool want_default,
468 char *src);
e1ae2ef @alk fixes tons of warnings produced on GNU/Linux
alk authored
469
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
470 static
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
471 bool cproxy_on_config_json(proxy_main *m, uint32_t new_config_ver, char *config,
472 char *src) {
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
473 bool rv = false;
474
0ebd7f6 @steveyen added cJSON
steveyen authored
475 cJSON *c = cJSON_Parse(config);
476 if (c != NULL) {
fcaec8e @steveyen cproxy_on_new_config_json_buckets() helper func
steveyen authored
477 cJSON *jBuckets = cJSON_GetObjectItem(c, "buckets");
478 if (jBuckets != NULL &&
479 jBuckets->type == cJSON_Array) {
480 // Make two passes through jBuckets, favoring any "default"
481 // bucket on the 1st pass, so the default bucket gets
482 // created earlier.
483 //
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
484 bool rv1 = cproxy_on_config_json_buckets(m, new_config_ver, jBuckets,
485 true, src);
486 bool rv2 = cproxy_on_config_json_buckets(m, new_config_ver, jBuckets,
487 false, src);
fcaec8e @steveyen cproxy_on_new_config_json_buckets() helper func
steveyen authored
488
489 rv = rv1 || rv2;
0ebd7f6 @steveyen added cJSON
steveyen authored
490 } else {
491 // Just a single config.
492 //
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
493 rv = cproxy_on_config_json_one(m, new_config_ver, config, "default", src);
0ebd7f6 @steveyen added cJSON
steveyen authored
494 }
495
496 cJSON_Delete(c);
2ecd867 @steveyen more error msgs when invalid JSON
steveyen authored
497 } else {
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
498 moxi_log_write("ERROR: could not parse JSON from REST server: %s, %s\n",
499 src, config);
0ebd7f6 @steveyen added cJSON
steveyen authored
500 }
501
502 return rv;
503 }
504
505 static
ecf81b7 @steveyen renamed cproxy_on_new_config_foobar() to cproxy_on_config_foobar()
steveyen authored
506 bool cproxy_on_config_json_buckets(proxy_main *m, uint32_t new_config_ver,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
507 cJSON *jBuckets, bool want_default,
508 char *src) {
fcaec8e @steveyen cproxy_on_new_config_json_buckets() helper func
steveyen authored
509 bool rv = false;
510
511 int numBuckets = cJSON_GetArraySize(jBuckets);
512 for (int i = 0; i < numBuckets; i++) {
513 cJSON *jBucket = cJSON_GetArrayItem(jBuckets, i);
514 if (jBucket != NULL &&
515 jBucket->type == cJSON_Object) {
516 char *name = "default";
517
518 cJSON *jName = cJSON_GetObjectItem(jBucket, "name");
519 if (jName != NULL &&
520 jName->type == cJSON_String &&
521 jName->valuestring != NULL) {
522 name = jName->valuestring;
523 }
524
525 bool is_default = (strcmp(name, "default") == 0);
526 if (!(is_default ^ want_default)) { // XOR.
527 char *jBucketStr = cJSON_Print(jBucket);
528 if (jBucketStr != NULL) {
ecf81b7 @steveyen renamed cproxy_on_new_config_foobar() to cproxy_on_config_foobar()
steveyen authored
529 rv = cproxy_on_config_json_one(m, new_config_ver,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
530 jBucketStr, name, src) || rv;
fcaec8e @steveyen cproxy_on_new_config_json_buckets() helper func
steveyen authored
531 free(jBucketStr);
532 }
533 }
534 }
535 }
536
537 return rv;
538 }
539
540 static
ecf81b7 @steveyen renamed cproxy_on_new_config_foobar() to cproxy_on_config_foobar()
steveyen authored
541 bool cproxy_on_config_json_one(proxy_main *m, uint32_t new_config_ver,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
542 char *config, char *name,
543 char *src) {
6b07929 @steveyen grabbing bucket name from JSON
steveyen authored
544 assert(m != NULL);
545 assert(config != NULL);
546 assert(name != NULL);
547
678ef3e @steveyen bug 2608 - use json-provided bucket name rather than "default"
steveyen authored
548 bool rv = false;
549
0ebd7f6 @steveyen added cJSON
steveyen authored
550 // Handle reconfiguration of a single proxy.
551 //
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
552 if (m != NULL &&
a16f538 @steveyen bug 1889 - don't quit moxi on empty JSON re-config messages
steveyen authored
553 config != NULL &&
554 strlen(config) > 0) {
d1ed9e3 @steveyen more printfs
steveyen authored
555 if (settings.verbose > 2) {
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
556 moxi_log_write("conjo contents config from %s: %s\n", src, config);
d1ed9e3 @steveyen more printfs
steveyen authored
557 }
558
a05bf8c @steveyen calling cproxy_on_new_pool() after parsing vbucket config
steveyen authored
559 // The config should be JSON that should look like...
8e8d5a1 @steveyen first attempt at using vbucket_config_parse_string()
steveyen authored
560 //
5889c43 @steveyen plain sasl auth for upstream clients
steveyen authored
561 // {"name":"default", // The bucket name.
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
562 // "nodeLocator":"ketama", // Optional.
678ef3e @steveyen bug 2608 - use json-provided bucket name rather than "default"
steveyen authored
563 // "saslPassword":"someSASLPwd",
8e8d5a1 @steveyen first attempt at using vbucket_config_parse_string()
steveyen authored
564 // "nodes":[{"hostname":"10.17.1.46","status":"healthy",
565 // "version":"0.3.0_114_g31859fe","os":"i386-apple-darwin9.8.0",
566 // "ports":{"proxy":11213,"direct":11212}}],
567 // "buckets":{"uri":"/pools/default/buckets"},
568 // "controllers":{"ejectNode":{"uri":"/controller/ejectNode"},
569 // "testWorkload":{"uri":"/pools/default/controller/testWorkload"}},
570 // "stats":{"uri":"/pools/default/stats"},
5889c43 @steveyen plain sasl auth for upstream clients
steveyen authored
571 // "vBucketServerMap":{
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
572 // "hashAlgorithm":"CRC",
5889c43 @steveyen plain sasl auth for upstream clients
steveyen authored
573 // "user":"optionalSASLUsr", // Optional.
678ef3e @steveyen bug 2608 - use json-provided bucket name rather than "default"
steveyen authored
574 // "password":"someSASLPwd", // Optional.
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
575 // "serverList":["10.17.1.46:11212"],
5889c43 @steveyen plain sasl auth for upstream clients
steveyen authored
576 // ...more json here...}}
8e8d5a1 @steveyen first attempt at using vbucket_config_parse_string()
steveyen authored
577 //
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
578 cJSON *jConfig = cJSON_Parse(config);
579 if (jConfig != NULL) {
678ef3e @steveyen bug 2608 - use json-provided bucket name rather than "default"
steveyen authored
580 cJSON *jName = cJSON_GetObjectItem(jConfig, "name");
581 if (jName != NULL &&
582 jName->type == cJSON_String &&
583 jName->valuestring != NULL) {
584 name = jName->valuestring;
585 }
586
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
587 cJSON *jNodeLocator = cJSON_GetObjectItem(jConfig, "nodeLocator");
588 if (jNodeLocator != NULL &&
589 jNodeLocator->type == cJSON_String &&
590 jNodeLocator->valuestring != NULL) {
591 if (strcmp(jNodeLocator->valuestring, "ketama") == 0) {
678ef3e @steveyen bug 2608 - use json-provided bucket name rather than "default"
steveyen authored
592 rv = cproxy_on_config_json_one_ketama(m, new_config_ver,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
593 config, name, src);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
594 cJSON_Delete(jConfig);
678ef3e @steveyen bug 2608 - use json-provided bucket name rather than "default"
steveyen authored
595
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
596 return rv;
597 }
d1ed9e3 @steveyen more printfs
steveyen authored
598 }
599
678ef3e @steveyen bug 2608 - use json-provided bucket name rather than "default"
steveyen authored
600 rv = cproxy_on_config_json_one_vbucket(m, new_config_ver,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
601 config, name, src);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
602 cJSON_Delete(jConfig);
603 }
604 } else {
605 if (settings.verbose > 1) {
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
606 moxi_log_write("ERROR: skipping empty config from %s\n", src);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
607 }
608 }
609
678ef3e @steveyen bug 2608 - use json-provided bucket name rather than "default"
steveyen authored
610 return rv;
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
611 }
612
613 static
614 bool cproxy_on_config_json_one_vbucket(proxy_main *m, uint32_t new_config_ver,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
615 char *config, char *name,
616 char *src) {
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
617 assert(m != NULL);
8e8d5a1 @steveyen first attempt at using vbucket_config_parse_string()
steveyen authored
618
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
619 bool rv = false;
620
621 if (settings.verbose > 2) {
622 moxi_log_write("parsing config nodeLocator:vbucket\n");
623 }
624
625 VBUCKET_CONFIG_HANDLE vch = vbucket_config_parse_string(config);
626 if (vch) {
627 if (settings.verbose > 2) {
678ef3e @steveyen bug 2608 - use json-provided bucket name rather than "default"
steveyen authored
628 moxi_log_write("conc vbucket_config_parse_string: %d for %s\n",
629 (vch != NULL), name);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
630 }
631
632 proxy_behavior proxyb = m->behavior;
3642114 @steveyen MB-2972 - return SERVER_ERROR for membase bucket GET's
steveyen authored
633 strcpy(proxyb.nodeLocator, "vbucket");
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
634
635 int pool_port = proxyb.port_listen;
636 int nodes_num = vbucket_config_get_num_servers(vch);
637
638 if (settings.verbose > 2) {
639 moxi_log_write("conc pool_port: %d nodes_num: %d\n",
640 pool_port, nodes_num);
641 }
642
643 if (pool_port > 0 &&
644 nodes_num > 0) {
645 proxy_behavior_pool behavior_pool = {
646 .base = proxyb,
647 .num = nodes_num,
648 .arr = calloc(nodes_num, sizeof(proxy_behavior))
649 };
650
651 if (behavior_pool.arr != NULL) {
e3fd701 @steveyen bug 2602 - use name/saslPassword, not vBucketServerMap.user/password
steveyen authored
652 cproxy_parse_json_auth(config, name, &behavior_pool);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
653
654 int j = 0;
655 for (; j < nodes_num; j++) {
656 // Inherit default behavior.
657 //
658 behavior_pool.arr[j] = behavior_pool.base;
659
660 const char *hostport = vbucket_config_get_server(vch, j);
661 if (hostport != NULL &&
662 strlen(hostport) > 0 &&
663 strlen(hostport) < sizeof(behavior_pool.arr[j].host) - 1) {
664 strncpy(behavior_pool.arr[j].host,
665 hostport,
666 sizeof(behavior_pool.arr[j].host) - 1);
667 behavior_pool.arr[j].host[sizeof(behavior_pool.arr[j].host) - 1] = '\0';
668
669 char *colon = strchr(behavior_pool.arr[j].host, ':');
670 if (colon != NULL) {
671 *colon = '\0';
672 behavior_pool.arr[j].port = atoi(colon + 1);
673 if (behavior_pool.arr[j].port <= 0) {
674 break;
675 }
676 } else {
677 break;
678 }
679 } else {
680 break;
681 }
682 }
683
684 if (j >= nodes_num) {
685 cproxy_on_config_pool(m, name, pool_port,
686 config, new_config_ver,
687 &behavior_pool);
688 rv = true;
689 } else {
690 if (settings.verbose > 1) {
691 moxi_log_write("ERROR: error receiving host:port"
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
692 " from %s"
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
693 " for server config %d in %s\n",
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
694 src, j, config);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
695 }
696 }
697
698 free(behavior_pool.arr);
699 }
700 }
701
702 vbucket_config_destroy(vch);
703 } else {
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
704 moxi_log_write("ERROR: bad JSON configuration from %s: %s (%s)\n",
705 src, vbucket_get_error(), config);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
706 if (ml->log_mode != ERRORLOG_STDERR) {
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
707 fprintf(stderr, "ERROR: bad JSON configuration from %s: %s (%s)\n",
708 src, vbucket_get_error(), config);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
709 }
710
711 // Bug 1961 - don't exit() as we might be in a multitenant use case.
712 //
713 // exit(EXIT_FAILURE);
714 }
715
716 return rv;
717 }
718
719 static
720 bool cproxy_on_config_json_one_ketama(proxy_main *m, uint32_t new_config_ver,
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
721 char *config, char *name,
722 char *src) {
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
723 assert(m != NULL);
724
725 bool rv = false;
726
727 #ifdef MOXI_USE_LIBMEMCACHED
728
729 if (settings.verbose > 2) {
730 moxi_log_write("parsing config nodeLocator:ketama\n");
731 }
732
733 // First, try to iterate through jConfig.vBucketServerMap.serverList
734 // if it exists, otherwise iterate through jConfig.nodes.
735 //
736 cJSON *jConfig = cJSON_Parse(config);
737 if (jConfig == NULL) {
738 return false;
739 }
740
741 cJSON *jArr = NULL;
742
743 cJSON *jVBSM = cJSON_GetObjectItem(jConfig, "vBucketServerMap");
744 if (jVBSM != NULL) {
745 jArr = cJSON_GetObjectItem(jVBSM, "serverList");
746 }
747
748 if (jArr == NULL ||
749 jArr->type != cJSON_Array) {
750 jArr = cJSON_GetObjectItem(jConfig, "nodes");
751 }
752
753 if (jArr != NULL &&
754 jArr->type == cJSON_Array) {
755 int nodes_num = cJSON_GetArraySize(jArr);
756 if (nodes_num > 0) {
757 proxy_behavior proxyb = m->behavior;
3642114 @steveyen MB-2972 - return SERVER_ERROR for membase bucket GET's
steveyen authored
758 strcpy(proxyb.nodeLocator, "ketama");
d1ed9e3 @steveyen more printfs
steveyen authored
759
760 if (settings.verbose > 2) {
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
761 moxi_log_write("conjk nodes_num: %d\n", nodes_num);
d1ed9e3 @steveyen more printfs
steveyen authored
762 }
763
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
764 proxy_behavior_pool behavior_pool = {
765 .base = proxyb,
766 .num = nodes_num,
767 .arr = calloc(nodes_num + 1, sizeof(proxy_behavior))
768 };
769
770 if (behavior_pool.arr != NULL) {
e3fd701 @steveyen bug 2602 - use name/saslPassword, not vBucketServerMap.user/password
steveyen authored
771 cproxy_parse_json_auth(config, name, &behavior_pool);
1833082 @steveyen Use the user/password provided by the JSON rest msg
steveyen authored
772
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
773 int curr = 0; // Moves slower than j so we can skip unhealthy nodes.
774
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
775 int j = 0;
776 for (; j < nodes_num; j++) {
777 // Inherit default behavior.
778 //
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
779 behavior_pool.arr[curr] = behavior_pool.base;
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
780
781 cJSON *jNode = cJSON_GetArrayItem(jArr, j);
782 if (jNode != NULL) {
783 if (jNode->type == cJSON_String &&
784 jNode->valuestring != NULL) {
785 // Should look like "host:port".
786 //
787 char *hostport = jNode->valuestring;
788
789 if (strlen(hostport) > 0 &&
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
790 strlen(hostport) < sizeof(behavior_pool.arr[curr].host) - 1) {
791 strncpy(behavior_pool.arr[curr].host,
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
792 hostport,
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
793 sizeof(behavior_pool.arr[curr].host) - 1);
794 behavior_pool.arr[curr].host[sizeof(behavior_pool.arr[curr].host) - 1] = '\0';
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
795
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
796 char *colon = strchr(behavior_pool.arr[curr].host, ':');
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
797 if (colon != NULL) {
798 *colon = '\0';
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
799 behavior_pool.arr[curr].port = atoi(colon + 1);
800 if (behavior_pool.arr[curr].port > 0) {
801 curr++;
802 } else {
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
803 break;
804 }
805 } else {
806 break;
807 }
808 } else {
809 break;
810 }
811 } else if (jNode->type == cJSON_Object) {
812 // Should look like... {
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
813 // status: "healthy",
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
814 // hostname: "host",
815 // ports: { direct: port }
816 // }
817 //
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
818 cJSON *jStatus = cJSON_GetObjectItem(jNode, "status");
819 if (jStatus != NULL &&
820 jStatus->type == cJSON_String &&
821 jStatus->valuestring != NULL &&
822 strcmp(jStatus->valuestring, "healthy") != 0) {
823 // Skip non-healthy node.
824 //
825 continue;
826 }
827
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
828 cJSON *jHostname = cJSON_GetObjectItem(jNode, "hostname");
829 if (jHostname != NULL &&
830 jHostname->type == cJSON_String &&
831 jHostname->valuestring != NULL &&
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
832 strlen(jHostname->valuestring) < sizeof(behavior_pool.arr[curr].host) - 1) {
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
833 cJSON *jPorts = cJSON_GetObjectItem(jNode, "ports");
834 if (jPorts != NULL &&
835 jPorts->type == cJSON_Object) {
836 cJSON *jDirect = cJSON_GetObjectItem(jPorts, "direct");
837 if (jDirect != NULL &&
838 jDirect->type == cJSON_Number &&
839 jDirect->valueint > 0) {
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
840 strncpy(behavior_pool.arr[curr].host,
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
841 jHostname->valuestring,
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
842 sizeof(behavior_pool.arr[curr].host) - 1);
843 behavior_pool.arr[curr].host[sizeof(behavior_pool.arr[curr].host) - 1] = '\0';
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
844
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
845 // The JSON might return a hostname that looks
846 // like "HOST:REST_PORT", so strip off the ":REST_PORT".
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
847 //
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
848 char *colon = strchr(behavior_pool.arr[curr].host, ':');
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
849 if (colon != NULL) {
850 *colon = '\0';
851 }
852
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
853 behavior_pool.arr[curr].port = jDirect->valueint;
854
855 curr++;
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
856 } else {
857 break;
858 }
859 } else {
a05bf8c @steveyen calling cproxy_on_new_pool() after parsing vbucket config
steveyen authored
860 break;
861 }
862 } else {
863 break;
864 }
865 } else {
866 break;
867 }
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
868 } else {
869 break;
a05bf8c @steveyen calling cproxy_on_new_pool() after parsing vbucket config
steveyen authored
870 }
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
871 }
a05bf8c @steveyen calling cproxy_on_new_pool() after parsing vbucket config
steveyen authored
872
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
873 if (j >= nodes_num && curr > 0) {
874 // Some unhealthy nodes might have been skipped,
875 // so curr might be <= behavior_pool.num.
876 //
877 behavior_pool.num = curr;
878
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
879 // Create a config string that libmemcached likes,
880 // such as "HOST:PORT,HOST:PORT,HOST:PORT".
881 //
882 int config_len = 200;
883 char *config_str = calloc(config_len, 1);
884
885 if (config_str != NULL) {
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
886 for (j = 0; j < behavior_pool.num; j++) {
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
887 // Grow config string for libmemcached.
888 //
889 int x = 40 + // For port and weight.
890 strlen(config_str) +
891 strlen(behavior_pool.arr[j].host);
892 if (config_len < x) {
893 config_len = 2 * (config_len + x);
894 config_str = realloc(config_str, config_len);
895 if (config_str == NULL) {
896 break;
897 }
898 }
899
900 char *config_end = config_str + strlen(config_str);
901 if (config_end != config_str) {
902 *config_end++ = ',';
903 }
904
905 if (strlen(behavior_pool.arr[j].host) > 0 &&
906 behavior_pool.arr[j].port > 0) {
907 snprintf(config_end,
908 config_len - (config_end - config_str),
909 "%s:%u",
910 behavior_pool.arr[j].host,
911 behavior_pool.arr[j].port);
912 } else {
913 if (settings.verbose > 1) {
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
914 moxi_log_write("ERROR: conjk missing host/port %d in %s from %s\n",
915 j, name, src);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
916 }
917 }
918
919 if (behavior_pool.arr[j].downstream_weight > 0) {
920 config_end = config_str + strlen(config_str);
921 snprintf(config_end,
922 config_len - (config_end - config_str),
923 ":%u",
924 behavior_pool.arr[j].downstream_weight);
925 }
926 }
927
928 if (config_str != NULL) {
579a3a4 @steveyen bug 2608 - moxi should ignore unhealthy nodes for ketama
steveyen authored
929 if (j >= behavior_pool.num) {
930 cproxy_on_config_pool(m, name, proxyb.port_listen,
931 config_str, new_config_ver,
932 &behavior_pool);
933 rv = true;
934 }
935
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
936 free(config_str);
937 }
a05bf8c @steveyen calling cproxy_on_new_pool() after parsing vbucket config
steveyen authored
938 } else {
939 if (settings.verbose > 1) {
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
940 moxi_log_write("ERROR: oom on jk re-config str\n");;
a05bf8c @steveyen calling cproxy_on_new_pool() after parsing vbucket config
steveyen authored
941 }
942 }
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
943 } else {
944 if (settings.verbose > 1) {
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
945 moxi_log_write("ERROR: conjk parse error for config %d from %s in %s\n",
946 j, src, config);
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
947 }
948 }
8e8d5a1 @steveyen first attempt at using vbucket_config_parse_string()
steveyen authored
949
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
950 free(behavior_pool.arr);
951 } else {
952 if (settings.verbose > 1) {
953 moxi_log_write("ERROR: oom on jk re-config\n");;
8e8d5a1 @steveyen first attempt at using vbucket_config_parse_string()
steveyen authored
954 }
955 }
700a94d @steveyen bug 1725 - exit if the JSON could not be parsed
steveyen authored
956 } else {
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
957 if (settings.verbose > 1) {
958 moxi_log_write("ERROR: conjk empty serverList/nodes in re-config\n");;
10cd773 @steveyen bug 1958 - using stderr if moxi exit()'s
steveyen authored
959 }
8e8d5a1 @steveyen first attempt at using vbucket_config_parse_string()
steveyen authored
960 }
a16f538 @steveyen bug 1889 - don't quit moxi on empty JSON re-config messages
steveyen authored
961 } else {
962 if (settings.verbose > 1) {
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
963 moxi_log_write("ERROR: conjk no serverList/nodes in re-config\n");;
a16f538 @steveyen bug 1889 - don't quit moxi on empty JSON re-config messages
steveyen authored
964 }
8e8d5a1 @steveyen first attempt at using vbucket_config_parse_string()
steveyen authored
965 }
479b197 @steveyen cproxy_on_new_config was returning too early
steveyen authored
966
21533e1 @steveyen bug 2541 - allow heterogeneous buckets (memcached & membase)
steveyen authored
967 cJSON_Delete(jConfig);
968
969 #else // !MOXI_USE_LIBMEMCACHED
970
971 (void) m;
972 (void) new_config_ver;
973 (void) config;
974 (void) name;
975
976 moxi_log_write("ERROR: not compiled with libmemcached support\n");
977
978 #endif // !MOXI_USE_LIBMEMCACHED
979
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
980 return rv;
981 }
982
e3fd701 @steveyen bug 2602 - use name/saslPassword, not vBucketServerMap.user/password
steveyen authored
983 static void cproxy_parse_json_auth(char *config,
984 char *name,
985 proxy_behavior_pool *bp) {
986 strncpy(bp->base.usr, name, sizeof(bp->base.usr) - 1);
987 bp->base.usr[sizeof(bp->base.usr) - 1] = '\0';
988
989 cJSON *jConfig = cJSON_Parse(config);
990 if (jConfig != NULL) {
991 cJSON *jPassword = cJSON_GetObjectItem(jConfig, "saslPassword");
992 if (jPassword != NULL &&
993 jPassword->type == cJSON_String &&
994 jPassword->valuestring != NULL) {
995 strncpy(bp->base.pwd,
996 jPassword->valuestring,
997 sizeof(bp->base.pwd) - 1);
998 bp->base.pwd[sizeof(bp->base.pwd) - 1] = '\0';
999 }
1000
1001 cJSON_Delete(jConfig);
1002 }
1003 }
1004
2049eba @steveyen bug 2231 - MOXI_USE_LIBVBUCKET and MOXI_USE_LIBMEMCACHED
steveyen authored
1005 #else // !MOXI_USE_LIBVBUCKET
8e8d5a1 @steveyen first attempt at using vbucket_config_parse_string()
steveyen authored
1006
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1007 static
ecf81b7 @steveyen renamed cproxy_on_new_config_foobar() to cproxy_on_config_foobar()
steveyen authored
1008 bool cproxy_on_config_kvs(proxy_main *m, uint32_t new_config_ver, kvpair_t *kvs) {
50187c2 @steveyen added proxy-level behavior settings
steveyen authored
1009 // The kvs key-multivalues look roughly like...
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1010 //
50187c2 @steveyen added proxy-level behavior settings
steveyen authored
1011 // pool-customer1-a
1012 // svrname3
ec7bb4b @steveyen started to move to new kvpair layout
steveyen authored
1013 // pool-customer1-b
629370a @steveyen no more host:port meaning in svr name
steveyen authored
1014 // svrname1
1015 // svrname2
1016 // svr-svrname1
1017 // host=mc1.foo.net
1018 // port=11211
1019 // weight=1
1020 // bucket=buck1
1021 // usr=test1
1022 // pwd=password
1b12147 @steveyen parse_kvs_behavior helper func, reused for drain server parse
steveyen authored
1023 // svr-svrnameX
1024 // host=mc2.foo.net
1025 // port=11211
50187c2 @steveyen added proxy-level behavior settings
steveyen authored
1026 // behavior-customer1-a
1027 // wait_queue_timeout=1000
1028 // downstream_max=10
1029 // behavior-customer1-b
1030 // wait_queue_timeout=1000
1031 // downstream_max=10
8591d16 @steveyen doing strdup before parse_behavior_key_val since parser modifies input
steveyen authored
1032 // pool_drain-customer1-b
568c436 @steveyen revert drain code to attempt more general expand/shrink code
steveyen authored
1033 // svrname1
1034 // svrname3
fa646e2 @steveyen framework for ping_test
steveyen authored
1035 // pools
ec7bb4b @steveyen started to move to new kvpair layout
steveyen authored
1036 // customer1-a
1037 // customer1-b
629370a @steveyen no more host:port meaning in svr name
steveyen authored
1038 // bindings
1039 // 11221
1040 // 11331
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1041 //
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1042 char **pools = get_key_values(kvs, "pools");
1043 char **bindings = get_key_values(kvs, "bindings");
1044
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1045 if (pools == NULL) {
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1046 return false;
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1047 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1048
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1049 int npools = 0;
1050 int nbindings = 0;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1051
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1052 while (pools && pools[npools])
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1053 npools++;
1054
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1055 while (bindings && bindings[nbindings])
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1056 nbindings++;
1057
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1058 if (nbindings > 0 &&
1059 nbindings != npools) {
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1060 if (settings.verbose > 1) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
1061 moxi_log_write("npools does not match nbindings\n");
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1062 }
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1063 return false;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1064 }
1065
42eccee @steveyen parsing default behavior during agent_config
steveyen authored
1066 char **behavior_kvs = get_key_values(kvs, "behavior");
1067 if (behavior_kvs != NULL) {
1068 // Update the default behavior.
1069 //
1070 proxy_behavior m_behavior = m->behavior;
1071
1072 for (int k = 0; behavior_kvs[k]; k++) {
47f28f7 @steveyen more trimstrdup's due to parser modifying input
steveyen authored
1073 char *bstr = trimstrdup(behavior_kvs[k]);
1074 if (bstr != NULL) {
1075 cproxy_parse_behavior_key_val_str(bstr, &m_behavior);
1076 free(bstr);
1077 }
42eccee @steveyen parsing default behavior during agent_config
steveyen authored
1078 }
1079
1080 m->behavior = m_behavior;
1081 }
1082
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1083 for (int i = 0; i < npools; i++) {
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1084 char *pool_name = skipspace(pools[i]);
1085 if (pool_name != NULL &&
1086 pool_name[0] != '\0') {
1087 char buf[200];
1b12147 @steveyen parse_kvs_behavior helper func, reused for drain server parse
steveyen authored
1088
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1089 snprintf(buf, sizeof(buf), "pool-%s", pool_name);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1090
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1091 char **servers = get_key_values(kvs, trimstr(buf));
1092 if (servers != NULL) {
50187c2 @steveyen added proxy-level behavior settings
steveyen authored
1093 // Parse proxy-level behavior.
1094 //
1095 proxy_behavior proxyb = m->behavior;
1096
1b12147 @steveyen parse_kvs_behavior helper func, reused for drain server parse
steveyen authored
1097 if (parse_kvs_behavior(kvs, "behavior", pool_name, &proxyb)) {
50187c2 @steveyen added proxy-level behavior settings
steveyen authored
1098 if (settings.verbose > 1) {
1099 cproxy_dump_behavior(&proxyb,
5a56bdd @steveyen better debug fprintfs
steveyen authored
1100 "conc proxy_behavior", 1);
50187c2 @steveyen added proxy-level behavior settings
steveyen authored
1101 }
1102 }
1103
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1104 // The legacy way to get a port is through the bindings,
1105 // but they're also available as an inheritable
1106 // proxy_behavior field of port_listen.
f6c168f @steveyen use realloc for config_str
steveyen authored
1107 //
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1108 int pool_port = proxyb.port_listen;
1109
1110 if (i < nbindings &&
1111 bindings != NULL &&
ae9534c @steveyen allow for comments and newlines in config
steveyen authored
1112 bindings[i]) {
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1113 pool_port = atoi(skipspace(bindings[i]));
ae9534c @steveyen allow for comments and newlines in config
steveyen authored
1114 }
6534a5a @steveyen Removed behaviors_str and no parser-in-a-parser.
steveyen authored
1115
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1116 if (pool_port > 0) {
1117 // Number of servers in this pool.
12f9191 @steveyen using proxy_behavior_pool for parse_kvs_servers
steveyen authored
1118 //
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1119 int s = 0;
6b07929 @steveyen grabbing bucket name from JSON
steveyen authored
1120 while (servers[s]) {
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1121 s++;
6b07929 @steveyen grabbing bucket name from JSON
steveyen authored
1122 }
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1123
1124 if (s > 0) {
1125 // Parse server-level behaviors, so we'll have an
1126 // array of behaviors, one entry for each server.
1127 //
1128 proxy_behavior_pool behavior_pool = {
1129 .base = proxyb,
1130 .num = s,
1131 .arr = calloc(s, sizeof(proxy_behavior))
1132 };
1133
1134 if (behavior_pool.arr != NULL) {
1135 char *config_str =
1136 parse_kvs_servers("svr", pool_name, kvs,
1137 servers, &behavior_pool);
1138 if (config_str != NULL &&
1139 config_str[0] != '\0') {
8d6ae29 @CaptTofu * Changed verbosity levels a bit so that function entry is > 2
CaptTofu authored
1140 if (settings.verbose > 2) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
1141 moxi_log_write("conc config: %s\n",
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1142 config_str);
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1143 }
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1144
34ffe4a @steveyen renamed cproxy_on_new_pool() to cproxy_on_config_pool()
steveyen authored
1145 cproxy_on_config_pool(m, pool_name, pool_port,
1146 config_str, new_config_ver,
1147 &behavior_pool);
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1148
1149 free(config_str);
1150 }
1151
1152 free(behavior_pool.arr);
1153 } else {
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1154 if (settings.verbose > 1) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
1155 moxi_log_write("ERROR: oom on re-config malloc\n");;
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1156 }
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1157 return false;
12f9191 @steveyen using proxy_behavior_pool for parse_kvs_servers
steveyen authored
1158 }
1159 } else {
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1160 // Note: ignore when no servers for an existing pool.
1161 // Because the config_ver won't be updated, we'll
1162 // fall into the empty_pool code path below.
12f9191 @steveyen using proxy_behavior_pool for parse_kvs_servers
steveyen authored
1163 }
1dfdc3d @steveyen more err handling during conflate reconfig
steveyen authored
1164 } else {
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1165 if (settings.verbose > 1) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
1166 moxi_log_write("ERROR: conc missing pool port\n");
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1167 }
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1168 return false;
1dfdc3d @steveyen more err handling during conflate reconfig
steveyen authored
1169 }
1170 } else {
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1171 // Note: ignore when no servers for an existing pool.
1172 // Because the config_ver won't be updated, we'll
1173 // fall into the empty_pool code path below.
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1174 }
1dfdc3d @steveyen more err handling during conflate reconfig
steveyen authored
1175 } else {
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1176 if (settings.verbose > 1) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
1177 moxi_log_write("ERROR: conc missing pool name\n");
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1178 }
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1179 return false;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1180 }
1181 }
1182
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1183 return true;
1184 }
1185
2049eba @steveyen bug 2231 - MOXI_USE_LIBVBUCKET and MOXI_USE_LIBMEMCACHED
steveyen authored
1186 #endif // !MOXI_USE_LIBVBUCKET
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1187
1188 static
ecf81b7 @steveyen renamed cproxy_on_new_config_foobar() to cproxy_on_config_foobar()
steveyen authored
1189 void cproxy_on_config(void *data0, void *data1) {
e409a28 @steveyen MB-2811 - main thread reconfig is async
steveyen authored
1190 proxy_main *m = data0;
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1191 assert(m);
1192
1193 kvpair_t *kvs = data1;
1194 assert(kvs);
1195 assert(is_listen_thread());
1196
1197 m->stat_configs++;
1198
1199 uint32_t max_config_ver = 0;
1200
887e9e7 @steveyen the proxy_main_lock now covers the linked list
steveyen authored
1201 pthread_mutex_lock(&m->proxy_main_lock);
1202
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1203 for (proxy *p = m->proxy_head; p != NULL; p = p->next) {
1204 pthread_mutex_lock(&p->proxy_lock);
1205 if (max_config_ver < p->config_ver) {
1206 max_config_ver = p->config_ver;
1207 }
1208 pthread_mutex_unlock(&p->proxy_lock);
1209 }
1210
887e9e7 @steveyen the proxy_main_lock now covers the linked list
steveyen authored
1211 pthread_mutex_unlock(&m->proxy_main_lock);
1212
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1213 uint32_t new_config_ver = max_config_ver + 1;
1214
1215 if (settings.verbose > 2) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
1216 moxi_log_write("conc new_config_ver %u\n", new_config_ver);
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1217 }
1218
2049eba @steveyen bug 2231 - MOXI_USE_LIBVBUCKET and MOXI_USE_LIBMEMCACHED
steveyen authored
1219 #ifdef MOXI_USE_LIBVBUCKET
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
1220 char **urlv = get_key_values(kvs, "url"); // NULL delimited array of char *.
1221 char *url = urlv != NULL ? (urlv[0] != NULL ? urlv[0] : "") : "";
1222
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1223 char **contents = get_key_values(kvs, "contents");
1224 if (contents != NULL &&
1225 contents[0] != NULL) {
1226 char *config = trimstrdup(contents[0]);
2ecd867 @steveyen more error msgs when invalid JSON
steveyen authored
1227 if (config != NULL &&
1228 strlen(config) > 0) {
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
1229 cproxy_on_config_json(m, new_config_ver, config, url);
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1230
1231 free(config);
1232 } else {
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
1233 moxi_log_write("ERROR: invalid, empty config from REST server %s\n",
1234 url);
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1235 goto fail;
1236 }
2ecd867 @steveyen more error msgs when invalid JSON
steveyen authored
1237 } else {
75e41c2 @steveyen MB-3575 - Pass around conflate src/url for debuggability
steveyen authored
1238 moxi_log_write("ERROR: invalid response from REST server %s\n",
1239 url);
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1240 }
2049eba @steveyen bug 2231 - MOXI_USE_LIBVBUCKET and MOXI_USE_LIBMEMCACHED
steveyen authored
1241 #else // !MOXI_USE_LIBVBUCKET
ecf81b7 @steveyen renamed cproxy_on_new_config_foobar() to cproxy_on_config_foobar()
steveyen authored
1242 if (cproxy_on_config_kvs(m, new_config_ver, kvs) == false) {
12b8c2b @steveyen cproxy_on_new_config() broken into smaller helper funcs
steveyen authored
1243 goto fail;
1244 }
2049eba @steveyen bug 2231 - MOXI_USE_LIBVBUCKET and MOXI_USE_LIBMEMCACHED
steveyen authored
1245 #endif // !MOXI_USE_LIBVBUCKET
479b197 @steveyen cproxy_on_new_config was returning too early
steveyen authored
1246
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1247 // If there were any proxies that weren't updated in the
1248 // previous loop, we need to shut them down. We mark the
1249 // proxy->config as NULL, and cproxy_check_downstream_config()
1250 // will catch it.
1251 //
d907a27 @steveyen close_outdated_proxies() helper function
steveyen authored
1252 close_outdated_proxies(m, new_config_ver);
1253
1254 free_kvpair(kvs);
1255
1256 return;
1257
1258 fail:
1259 m->stat_config_fails++;
1260 free_kvpair(kvs);
1261
1262 if (settings.verbose > 1) {
7279bd7 @steveyen more debug logging
steveyen authored
1263 moxi_log_write("ERROR: conc failed config %llu\n",
d907a27 @steveyen close_outdated_proxies() helper function
steveyen authored
1264 (long long unsigned int) m->stat_config_fails);
1265 }
1266 }
1267
1268 void close_outdated_proxies(proxy_main *m, uint32_t new_config_ver) {
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1269 // TODO: Close any listening conns for the proxy?
1270 // TODO: Close any upstream conns for the proxy?
1271 // TODO: We still need to free proxy memory, after all its
1272 // proxy_td's and downstreams are closed, and no more
1273 // upstreams are pointed at the proxy.
1274 //
16be84f @steveyen proxy_behavior_pool struct
steveyen authored
1275 proxy_behavior_pool empty_pool;
1276 memset(&empty_pool, 0, sizeof(proxy_behavior_pool));
1277
1278 empty_pool.base = m->behavior;
1279 empty_pool.num = 0;
1280 empty_pool.arr = NULL;
1281
887e9e7 @steveyen the proxy_main_lock now covers the linked list
steveyen authored
1282 pthread_mutex_lock(&m->proxy_main_lock);
1283
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1284 for (proxy *p = m->proxy_head; p != NULL; p = p->next) {
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1285 bool down = false;
1286 int port = 0;
1287 char *name = NULL;
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1288
1289 pthread_mutex_lock(&p->proxy_lock);
1290
1291 if (p->config_ver != new_config_ver) {
1292 down = true;
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1293
1294 assert(p->port > 0);
1295 assert(p->name != NULL);
1296
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1297 port = p->port;
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1298 name = strdup(p->name);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1299 }
1300
1301 pthread_mutex_unlock(&p->proxy_lock);
1302
887e9e7 @steveyen the proxy_main_lock now covers the linked list
steveyen authored
1303 pthread_mutex_unlock(&m->proxy_main_lock);
1304
1305 // Note, we don't want to own the proxy_main_lock here
1306 // because cproxy_on_config_pool() may scatter/gather
1307 // calls against the worker threads, and the worked threads
1308 // should not deadlock if they need the proxy_main_lock.
1309 //
1310 // Also, check that we're not shutting down the NULL_BUCKET.
522ddf0 @steveyen added a NULL_BUCKET placeholder for SaaS use-case
steveyen authored
1311 //
1312 // Otherwise, passing in a NULL config string signals that
1313 // a bucket's proxy struct should be shut down.
1314 //
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1315 if (name != NULL) {
7824249 @steveyen use appropriate lock before accessing proxy->name
steveyen authored
1316 if (down && (strcmp(NULL_BUCKET, name) != 0)) {
1317 cproxy_on_config_pool(m, name, port, NULL, new_config_ver,
1318 &empty_pool);
1319 }
1320
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1321 free(name);
1322 }
887e9e7 @steveyen the proxy_main_lock now covers the linked list
steveyen authored
1323
1324 pthread_mutex_lock(&m->proxy_main_lock);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1325 }
887e9e7 @steveyen the proxy_main_lock now covers the linked list
steveyen authored
1326
1327 pthread_mutex_unlock(&m->proxy_main_lock);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1328 }
1329
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1330 /**
1331 * A name and port uniquely identify a proxy.
1332 */
34ffe4a @steveyen renamed cproxy_on_new_pool() to cproxy_on_config_pool()
steveyen authored
1333 void cproxy_on_config_pool(proxy_main *m,
1334 char *name, int port,
1335 char *config,
1336 uint32_t config_ver,
1337 proxy_behavior_pool *behavior_pool) {
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1338 assert(m);
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1339 assert(name != NULL);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1340 assert(port >= 0);
1341 assert(is_listen_thread());
1342
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1343 // See if we've already got a proxy running with that name and port,
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1344 // and create one if needed.
1345 //
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1346 bool found = false;
9fef1b9 @steveyen formatting
steveyen authored
1347
887e9e7 @steveyen the proxy_main_lock now covers the linked list
steveyen authored
1348 pthread_mutex_lock(&m->proxy_main_lock);
1349
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1350 proxy *p = m->proxy_head;
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1351 while (p != NULL && !found) {
1352 pthread_mutex_lock(&p->proxy_lock);
1353
1354 assert(p->port > 0);
1355 assert(p->name != NULL);
1356
1357 found = ((p->port == port) &&
1358 (strcmp(p->name, name) == 0));
1359
1360 pthread_mutex_unlock(&p->proxy_lock);
1361
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1362 if (found) {
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1363 break;
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1364 }
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1365
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1366 p = p->next;
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1367 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1368
887e9e7 @steveyen the proxy_main_lock now covers the linked list
steveyen authored
1369 pthread_mutex_unlock(&m->proxy_main_lock);
1370
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1371 if (p == NULL) {
2bebd7a @steveyen added proxy_main ptr to the proxy struct
steveyen authored
1372 p = cproxy_create(m, name, port,
2f432a7 @steveyen made downstream timeout a full struct timeval
steveyen authored
1373 config,
1374 config_ver,
16be84f @steveyen proxy_behavior_pool struct
steveyen authored
1375 behavior_pool,
8a7e4b6 @steveyen Moved nthreads out of proxy_behavior.
steveyen authored
1376 m->nthreads);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1377 if (p != NULL) {
f4d3531 Added new ascii commands of "stats proxy sub-command".
Joon(JunHyun Park) authored
1378 pthread_mutex_lock(&m->proxy_main_lock);
7a2d764 @steveyen removed JHP_STATS conditional code
steveyen authored
1379
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1380 p->next = m->proxy_head;
1381 m->proxy_head = p;
7a2d764 @steveyen removed JHP_STATS conditional code
steveyen authored
1382
f4d3531 Added new ascii commands of "stats proxy sub-command".
Joon(JunHyun Park) authored
1383 pthread_mutex_unlock(&m->proxy_main_lock);
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1384
1385 int n = cproxy_listen(p);
1386 if (n > 0) {
8d6ae29 @CaptTofu * Changed verbosity levels a bit so that function entry is > 2
CaptTofu authored
1387 if (settings.verbose > 2) {
7279bd7 @steveyen more debug logging
steveyen authored
1388 moxi_log_write(
1389 "cproxy_listen success %u for %s to %s with %d conns\n",
1390 p->port, p->name, p->config, n);
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1391 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1392 m->stat_proxy_starts++;
1393 } else {
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1394 if (settings.verbose > 1) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
1395 moxi_log_write("ERROR: cproxy_listen failed on %u to %s\n",
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1396 p->port, p->config);
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1397 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1398 m->stat_proxy_start_fails++;
1399 }
7279bd7 @steveyen more debug logging
steveyen authored
1400 } else {
1401 if (settings.verbose > 2) {
1402 moxi_log_write("ERROR: cproxy_create failed on %s, %d, %s\n",
1403 name, port, config);
1404 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1405 }
1406 } else {
8d6ae29 @CaptTofu * Changed verbosity levels a bit so that function entry is > 2
CaptTofu authored
1407 if (settings.verbose > 2) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
1408 moxi_log_write("conp existing config change %u\n",
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1409 p->port);
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1410 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1411
f93e1b3 @steveyen port_listen and proxy name is part of identifier
steveyen authored
1412 bool changed = false;
e1ae2ef @alk fixes tons of warnings produced on GNU/Linux
alk authored
1413 bool shutdown_flag = false;
5a56bdd @steveyen better debug fprintfs
steveyen authored
1414
f4d3531 Added new ascii commands of "stats proxy sub-command".
Joon(JunHyun Park) authored
1415 pthread_mutex_lock(&m->proxy_main_lock);
1416
ceb2517 @steveyen front_cache_start/stop()
steveyen authored
1417 // Turn off the front_cache while we're reconfiguring.
1418 //
aa9f46b @steveyen refactored to mcache struct and api functions
steveyen authored
1419 mcache_stop(&p->front_cache);
5971996 @steveyen matcher refactor with lock
steveyen authored
1420 matcher_stop(&p->front_cache_matcher);
30e7a4a @steveyen added unmatcher for front cache and key stats
steveyen authored
1421 matcher_stop(&p->front_cache_unmatcher);
e5910ca @steveyen started key_stats
steveyen authored
1422
5971996 @steveyen matcher refactor with lock
steveyen authored
1423 matcher_stop(&p->optimize_set_matcher);
ceb2517 @steveyen front_cache_start/stop()
steveyen authored
1424
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1425 pthread_mutex_lock(&p->proxy_lock);
1426
8d6ae29 @CaptTofu * Changed verbosity levels a bit so that function entry is > 2
CaptTofu authored
1427 if (settings.verbose > 2) {
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1428 if (p->config && config &&
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1429 strcmp(p->config, config) != 0) {
6a48c21 @mtaneja Migrate to using moxi log
mtaneja authored
1430 moxi_log_write("conp config changed from %s to %s\n",
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1431 p->config, config);
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1432 }
28a8724 @steveyen renamed to agent_config/stats.c
steveyen authored
1433 }
1434
9fef1b9 @steveyen formatting
steveyen authored
1435 changed =
1436 update_str_config(&p->config, config,
f0ea4b4 @ingenthr fix ubuntu fprintf warning, issue 14, and added style braces
ingenthr authored
1437 "conp config changed") ||
9fef1b9 @steveyen formatting
steveyen authored
1438 changed;
5a56bdd @steveyen better debug fprintfs
steveyen authored
1439
b1e7b28 @steveyen dynamic reconfig bug
steveyen authored
1440 changed =
16be84f @steveyen proxy_behavior_pool struct
steveyen authored
1441 (cproxy_equal_behavior(&p->behavior_pool.base,
1442 &behavior_pool->base) == false) ||
b1e7b28 @steveyen dynamic reconfig bug
steveyen authored
1443 changed;
e5910ca @steveyen started key_stats
steveyen authored
1444
16be84f @steveyen proxy_behavior_pool struct
steveyen authored
1445 p->behavior_pool.base = behavior_pool->base;
50187c2