Merged revisions 1649-1650,1655-1692 via svnmerge from

svn+ssh://projects.linpro.no/svn/varnish/trunk/varnish-cache ........ r1657 | phk | 2007-07-05 23:08:15 +0200 (Thu, 05 Jul 2007) | 2 lines Clean up FlexeLint fluff. ........ r1658 | phk | 2007-07-06 12:07:30 +0200 (Fri, 06 Jul 2007) | 3 lines Don't rewrite pipe'ed requests to "GET". ........ r1659 | phk | 2007-07-09 22:23:41 +0200 (Mon, 09 Jul 2007) | 10 lines Make all protocol header fields writable, except obj.status and resp.status (which are numeric, they'll follow shortly) Unify the shmemlog tag used for failure to rewrite something, the Rx/Tx/Obj distinction is not helpful enough to warrant the complexity of it. ........ r1660 | phk | 2007-07-09 22:34:59 +0200 (Mon, 09 Jul 2007) | 2 lines Allow assignment to INT type variables ........ r1661 | phk | 2007-07-09 22:35:20 +0200 (Mon, 09 Jul 2007) | 2 lines Allow assignment to obj.status and resp.status ........ r1662 | phk | 2007-07-10 21:46:16 +0200 (Tue, 10 Jul 2007) | 6 lines Move string stuff to vcc_string.c, there's going to be a fair bit of it. Give vcc_StringVal() a return value to say if it did anything so we can emit better error messages when confused. ........ r1663 | phk | 2007-07-10 21:59:39 +0200 (Tue, 10 Jul 2007) | 5 lines Add conversion from IP to string format to allow things like: set bereq.http.HeyYou = client.ip " asked for " req.url; ........ r1664 | phk | 2007-07-10 22:07:07 +0200 (Tue, 10 Jul 2007) | 3 lines Properly emit the header name in VRT_SetHdr(); ........ r1665 | phk | 2007-07-10 22:08:39 +0200 (Tue, 10 Jul 2007) | 2 lines Fix VRT_SetHdr() prototype ........ r1666 | phk | 2007-07-10 22:43:24 +0200 (Tue, 10 Jul 2007) | 2 lines Add compiler side support for regsub() but only a dummy function in VRT. ........ r1667 | phk | 2007-07-10 23:30:47 +0200 (Tue, 10 Jul 2007) | 60 lines Add "regsub" support for string manipulation. Notice this facility is subject to change! "regsub" is short for regular expression substitution and it is probably easiest to explain with some examples: sub vcl_recv { set req.url = regsub(req.url, "#.*", ""); } This will replace the requests URL with the output of the regsub() function regsub() takes three arguments: the string to be examined, a regular expression and a replacement string. In this case, everything after the first '#' is removed (replaced with nothing). The replacement string recognizes the following magic sequences: & - insert everything matched by the regexp $0 - ditto. $1 - replace with the first submatch of the regexp $2 - replace with the second submatch of the regexp ... $9 - replace with the ninth submatch of the regexp (The $0..$9 syntax was chosen over the \0...\9 syntax in order to avoid a nightmare of escape characters in the VCL source code. Arguments and suggestions are welcome). A more advanced example: set bereq.http.ClientIP = regsub(client.ip, "(.*):(.*)", "$2 $1"); The client.ip variable expands to IP:port number, for instance 127.0.0.1:54662 The regular expression "(.*):(.*)" results in the the following matches: & + $0 "127.0.0.1:54662" $1 "127.0.0.1" $2 "54662" So the replacement string "$2 $1" results in "54662 127.0.0.1" And the completed header which is sent to the backend will look like: "ClientIP: 54662 127.0.0.1" An even more advanced example would be: set bereq.http.magic = "Client IP = " regsub(client.ip, ":", " port = "); Where we also exploint the string concatenation ability of the "set" statement. The result string is built in the request workspace, so you may need to increase the workspace size if you do a lot of regsub()'s. Currently there is no decent error handling for running out of workspace. ........ r1668 | phk | 2007-07-12 11:04:54 +0200 (Thu, 12 Jul 2007) | 6 lines Add TIM_mono() and TIM_real() which return double representations of timestamps on a monotonic and the UTC timescales respectively. Doubles are much more convenient than timespecs for comparisons etc. ........ r1669 | phk | 2007-07-12 11:25:07 +0200 (Thu, 12 Jul 2007) | 2 lines Replace ev_now() with TIM_mono(). ........ r1670 | phk | 2007-07-12 11:25:45 +0200 (Thu, 12 Jul 2007) | 2 lines Replace Uptime() with TIM_mono() ........ r1671 | phk | 2007-07-12 11:49:26 +0200 (Thu, 12 Jul 2007) | 6 lines Change all timekeeping to use doubles instead of time_t and struct timespec. Eliminate all direct calls to time(2) and clockgettime(2) and use TIM_real() and TIM_mono() exclusively. ........ r1672 | phk | 2007-07-12 12:00:13 +0200 (Thu, 12 Jul 2007) | 2 lines Document timescale of srcaddr->ttl ........ r1673 | phk | 2007-07-12 12:13:29 +0200 (Thu, 12 Jul 2007) | 2 lines Convert the last time(2) calls to TIM_real() ........ r1674 | cecilihf | 2007-07-12 12:20:33 +0200 (Thu, 12 Jul 2007) | 2 lines Added a new cli option, status, for checking the status of the varnish child process. This is for use in the webmin plugin. ........ r1675 | cecilihf | 2007-07-12 12:27:37 +0200 (Thu, 12 Jul 2007) | 2 lines Fixed typo ........ r1676 | des | 2007-07-12 18:00:04 +0200 (Thu, 12 Jul 2007) | 2 lines RT_LIBS dependency has moved from varnishd to libvarnish. ........ r1677 | des | 2007-07-12 18:02:47 +0200 (Thu, 12 Jul 2007) | 2 lines Add missing semicolon. ........ r1678 | des | 2007-07-12 19:37:44 +0200 (Thu, 12 Jul 2007) | 2 lines sockaddr.sa_len is not portable. ........ r1682 | phk | 2007-07-13 09:11:54 +0200 (Fri, 13 Jul 2007) | 2 lines Initialize all timestamps in the session to NAN ........ r1683 | phk | 2007-07-13 09:21:46 +0200 (Fri, 13 Jul 2007) | 4 lines Unify the recycle functionality of the acceptors, all three used the same method. ........ r1684 | phk | 2007-07-13 09:27:50 +0200 (Fri, 13 Jul 2007) | 2 lines Clean all but t_open timestamps to NAN at end of transaction. ........ r1685 | phk | 2007-07-13 09:47:45 +0200 (Fri, 13 Jul 2007) | 9 lines Rename the "idle" field of struct worker to "used", which is more precise. Don't use the "used" field to signal suicide for worker threads, use the "wrq" field which is much more natural. Set the "used" field to NAN before doing anything and assert that somebody updated during the task. ........ r1686 | phk | 2007-07-13 09:53:08 +0200 (Fri, 13 Jul 2007) | 2 lines Clarify XXX comment ........ r1687 | phk | 2007-07-13 09:58:11 +0200 (Fri, 13 Jul 2007) | 3 lines Move setting of t_resp up to before we build the response. ........ r1688 | phk | 2007-07-13 10:05:14 +0200 (Fri, 13 Jul 2007) | 2 lines Add an XXX comment ........ r1690 | des | 2007-07-13 13:42:02 +0200 (Fri, 13 Jul 2007) | 2 lines Add an entry for r1531. ........ r1691 | des | 2007-07-13 16:27:55 +0200 (Fri, 13 Jul 2007) | 2 lines Document regsub(). ........ r1692 | des | 2007-07-13 16:53:48 +0200 (Fri, 13 Jul 2007) | 2 lines Document recent changes. ........ git-svn-id: http://www.varnish-cache.org/svn/branches/1.1@1693 d4fa192b-c00b-0410-8231-f00ffab90ce4
varnish · Jul 13, 2007 · cb3214a · cb3214a
1 parent 558893b
commit cb3214a
Show file tree

Hide file tree

Showing 51 changed files with 758 additions and 448 deletions.
diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
@@ -68,4 +68,4 @@ varnishd_LDADD = \
 	$(top_builddir)/lib/libvarnish/libvarnish.la \
 	$(top_builddir)/lib/libcompat/libcompat.a \
 	$(top_builddir)/lib/libvcl/libvcl.la \
-	${DL_LIBS} ${RT_LIBS} ${PTHREAD_LIBS}
+	${DL_LIBS} ${PTHREAD_LIBS}
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
@@ -131,7 +131,7 @@ struct http {
 /*--------------------------------------------------------------------*/
 
 struct acct {
-	time_t			first;
+	double			first;
 	uint64_t		sess;
 	uint64_t		req;
 	uint64_t		pipe;
@@ -149,7 +149,7 @@ struct worker {
 	struct objhead		*nobjhead;
 	struct object		*nobj;
 
-	time_t			idle;
+	double			used;
 
 	int			pipe[2];
 
@@ -247,11 +247,11 @@ struct object {
 	unsigned		busy;
 	unsigned		len;
 
-	time_t			age;
-	time_t			entered;
-	time_t			ttl;
+	double			age;
+	double			entered;
+	double			ttl;
 
-	time_t			last_modified;
+	double			last_modified;
 
 	struct http		http;
 	TAILQ_ENTRY(object)	list;
@@ -262,7 +262,7 @@ struct object {
 
 	TAILQ_HEAD(, sess)	waitinglist;
 
-	time_t			lru_stamp;
+	double			lru_stamp;
 	TAILQ_ENTRY(object)	lru;
 };
 
@@ -300,10 +300,11 @@ struct sess {
 	const char		*doclose;
 	struct http		*http;
 
-	struct timespec		t_open;
-	struct timespec		t_req;
-	struct timespec		t_resp;
-	struct timespec		t_end;
+	/* Timestamps, all on TIM_real() timescale */
+	double			t_open;
+	double			t_req;
+	double			t_resp;
+	double			t_end;
 
 	enum step		step;
 	unsigned 		handling;
@@ -359,6 +360,7 @@ void vca_return_session(struct sess *sp);
 void vca_close_session(struct sess *sp, const char *why);
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
+extern int vca_pipes[2];
 
 /* cache_backend.c */
 void VBE_Init(void);
@@ -412,6 +414,7 @@ void http_PutStatus(struct worker *w, int fd, struct http *to, int status);
 void http_PutResponse(struct worker *w, int fd, struct http *to, const char *response);
 void http_PrintfHeader(struct worker *w, int fd, struct http *to, const char *fmt, ...);
 void http_SetHeader(struct worker *w, int fd, struct http *to, const char *hdr);
+void http_SetH(struct http *to, unsigned n, const char *fm);
 void http_Setup(struct http *ht, void *space, unsigned len);
 int http_GetHdr(struct http *hp, const char *hdr, char **ptr);
 int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr);
@@ -428,7 +431,6 @@ int http_DissectResponse(struct worker *w, struct http *sp, int fd);
 void http_DoConnection(struct sess *sp);
 void http_CopyHome(struct worker *w, int fd, struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
-void http_LogLostHeader(struct worker *w, int fd, struct http *hp, const char *hdr);
 
 
 #define HTTPH(a, b, c, d, e, f, g) extern char b[];
@@ -492,11 +494,11 @@ void VCL_Get(struct VCL_conf **vcc);
 
 /* cache_lru.c */
 // void LRU_Init(void);
-void LRU_Enter(struct object *o, time_t stamp);
+void LRU_Enter(struct object *o, double stamp);
 void LRU_Remove(struct object *o);
 int LRU_DiscardOne(void);
 int LRU_DiscardSpace(int64_t quota);
-int LRU_DiscardTime(time_t cutoff);
+int LRU_DiscardTime(double cutoff);
 
 #define VCL_RET_MAC(l,u,b,n)
 #define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *);

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
@@ -44,10 +44,6 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 
-#ifndef HAVE_CLOCK_GETTIME
-#include "compat/clock_gettime.h"
-#endif
-
 #ifndef HAVE_SRANDOMDEV
 #include "compat/srandomdev.h"
 #endif
@@ -80,6 +76,8 @@ static struct linger	linger;
 
 static unsigned char	need_sndtimeo, need_rcvtimeo, need_linger, need_test;
 
+int vca_pipes[2];
+
 static void
 sock_test(int fd)
 {
@@ -116,7 +114,7 @@ VCA_Prep(struct sess *sp)
 	TCP_name(sp->sockaddr, sp->sockaddrlen,
 	    sp->addr, sizeof sp->addr, sp->port, sizeof sp->port);
 	VSL(SLT_SessionOpen, sp->fd, "%s %s", sp->addr, sp->port);
-	sp->acct.first = sp->t_open.tv_sec;
+	sp->acct.first = sp->t_open;
 	if (need_test)
 		sock_test(sp->fd);
 	if (need_linger)
@@ -195,7 +193,7 @@ vca_acct(void *arg)
 
 			sp->fd = i;
 			sp->id = i;
-			(void)clock_gettime(CLOCK_REALTIME, &sp->t_open);
+			sp->t_open = TIM_real();
 
 			http_RecvPrep(sp->http);
 			sp->step = STP_FIRST;
@@ -259,7 +257,10 @@ vca_return_session(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->obj);
 	AZ(sp->vcl);
-	vca_act->recycle(sp);
+	if (sp->fd < 0)
+		SES_Delete(sp);
+	else
+		 assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
 }
 
 
@@ -277,6 +278,7 @@ VCA_Init(void)
 		fprintf(stderr, "No acceptor in program\n");
 		exit (2);
 	}
+	AZ(pipe(vca_pipes));
 	vca_act->init();
 	AZ(pthread_create(&vca_thread_acct, NULL, vca_acct, NULL));
 }
diff --git a/bin/varnishd/cache_acceptor.h b/bin/varnishd/cache_acceptor.h
@@ -32,12 +32,10 @@
 struct sess;
 
 typedef void acceptor_init_f(void);
-typedef void acceptor_recycle_f(struct sess *);
 
 struct acceptor {
 	const char 		*name;
 	acceptor_init_f		*init;
-	acceptor_recycle_f	*recycle;
 };
 
 #if defined(HAVE_EPOLL_CTL)

diff --git a/bin/varnishd/cache_acceptor_epoll.c b/bin/varnishd/cache_acceptor_epoll.c
@@ -41,18 +41,13 @@
 
 #include <sys/epoll.h>
 
-#ifndef HAVE_CLOCK_GETTIME
-#include "compat/clock_gettime.h"
-#endif
-
 #include "heritage.h"
 #include "shmlog.h"
 #include "cache.h"
 #include "cache_acceptor.h"
 
 static pthread_t vca_epoll_thread;
 static int epfd = -1;
-static int pipes[2];
 
 static TAILQ_HEAD(,sess) sesshead = TAILQ_HEAD_INITIALIZER(sesshead);
 
@@ -74,7 +69,7 @@ static void *
 vca_main(void *arg)
 {
 	struct epoll_event ev;
-	struct timespec ts;
+	double deadline;
 	struct sess *sp, *sp2;
 	int i;
 
@@ -83,12 +78,12 @@ vca_main(void *arg)
 	epfd = epoll_create(16);
 	assert(epfd >= 0);
 
-	vca_add(pipes[0], pipes);
+	vca_add(vca_pipes[0], vca_pipes);
 
 	while (1) {
 		if (epoll_wait(epfd, &ev, 1, 100) > 0) {
-			if (ev.data.ptr == pipes) {
-				i = read(pipes[0], &sp, sizeof sp);
+			if (ev.data.ptr == vca_pipes) {
+				i = read(vca_pipes[0], &sp, sizeof sp);
 				assert(i == sizeof sp);
 				CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 				TAILQ_INSERT_TAIL(&sesshead, sp, list);
@@ -108,14 +103,10 @@ vca_main(void *arg)
 			}
 		}
 		/* check for timeouts */
-		clock_gettime(CLOCK_REALTIME, &ts);
-		ts.tv_sec -= params->sess_timeout;
+		deadline = TIM_real() - params->sess_timeout;
 		TAILQ_FOREACH_SAFE(sp, &sesshead, list, sp2) {
 			CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-			if (sp->t_open.tv_sec > ts.tv_sec)
-				continue;
-			if (sp->t_open.tv_sec == ts.tv_sec &&
-			    sp->t_open.tv_nsec > ts.tv_nsec)
+			if (sp->t_open > deadline)
 				continue;
 			TAILQ_REMOVE(&sesshead, sp, list);
 			vca_del(sp->fd);
@@ -127,28 +118,16 @@ vca_main(void *arg)
 
 /*--------------------------------------------------------------------*/
 
-static void
-vca_epoll_recycle(struct sess *sp)
-{
-
-	if (sp->fd < 0)
-		SES_Delete(sp);
-	else
-		assert(sizeof sp == write(pipes[1], &sp, sizeof sp));
-}
-
 static void
 vca_epoll_init(void)
 {
 
-	AZ(pipe(pipes));
 	AZ(pthread_create(&vca_epoll_thread, NULL, vca_main, NULL));
 }
 
 struct acceptor acceptor_epoll = {
 	.name =		"epoll",
 	.init =		vca_epoll_init,
-	.recycle =	vca_epoll_recycle,
 };
 
 #endif /* defined(HAVE_EPOLL_CTL) */
diff --git a/bin/varnishd/cache_acceptor_kqueue.c b/bin/varnishd/cache_acceptor_kqueue.c
@@ -43,10 +43,6 @@
 
 #include <sys/event.h>
 
-#ifndef HAVE_CLOCK_GETTIME
-#include "compat/clock_gettime.h"
-#endif
-
 #include "heritage.h"
 #include "shmlog.h"
 #include "cache.h"
@@ -56,7 +52,6 @@ static pthread_t vca_kqueue_thread;
 static int kq = -1;
 
 static TAILQ_HEAD(,sess) sesshead = TAILQ_HEAD_INITIALIZER(sesshead);
-static int pipes[2];
 
 #define NKEV	100
 
@@ -85,9 +80,9 @@ vca_kev(struct kevent *kp)
 	struct sess *ss[NKEV];
 
 	AN(kp->udata);
-	if (kp->udata == pipes) {
+	if (kp->udata == vca_pipes) {
 		j = 0;
-		i = read(pipes[0], ss, sizeof ss);
+		i = read(vca_pipes[0], ss, sizeof ss);
 		if (i == -1 && errno == EAGAIN)
 			return;
 		while (i >= sizeof ss[0]) {
@@ -129,7 +124,7 @@ vca_kqueue_main(void *arg)
 {
 	struct kevent ke[NKEV], *kp;
 	int j, n, dotimer;
-	struct timespec ts;
+	double deadline;
 	struct sess *sp;
 
 	(void)arg;
@@ -139,7 +134,7 @@ vca_kqueue_main(void *arg)
 
 	j = 0;
 	EV_SET(&ke[j++], 0, EVFILT_TIMER, EV_ADD, 0, 100, NULL);
-	EV_SET(&ke[j++], pipes[0], EVFILT_READ, EV_ADD, 0, 0, pipes);
+	EV_SET(&ke[j++], vca_pipes[0], EVFILT_READ, EV_ADD, 0, 0, vca_pipes);
 	AZ(kevent(kq, ke, j, NULL, 0, NULL));
 
 	nki = 0;
@@ -160,16 +155,12 @@ vca_kqueue_main(void *arg)
 		}
 		if (!dotimer)
 			continue;
-		clock_gettime(CLOCK_REALTIME, &ts);
-		ts.tv_sec -= params->sess_timeout;
+		deadline = TIM_real() - params->sess_timeout;
 		for (;;) {
 			sp = TAILQ_FIRST(&sesshead);
 			if (sp == NULL)
 				break;
-			if (sp->t_open.tv_sec > ts.tv_sec)
-				break;
-			if (sp->t_open.tv_sec == ts.tv_sec &&
-			    sp->t_open.tv_nsec > ts.tv_nsec)
+			if (sp->t_open > deadline)
 				break;
 			TAILQ_REMOVE(&sesshead, sp, list);
 			vca_close_session(sp, "timeout");
@@ -180,33 +171,21 @@ vca_kqueue_main(void *arg)
 
 /*--------------------------------------------------------------------*/
 
-static void
-vca_kqueue_recycle(struct sess *sp)
-{
-
-	if (sp->fd < 0)
-		SES_Delete(sp);
-	else
-		assert(write(pipes[1], &sp, sizeof sp) == sizeof sp);
-}
-
 static void
 vca_kqueue_init(void)
 {
 	int i;
 
-	AZ(pipe(pipes));
-	i = fcntl(pipes[0], F_GETFL);
+	i = fcntl(vca_pipes[0], F_GETFL);
 	i |= O_NONBLOCK;
-	i = fcntl(pipes[0], F_SETFL, i);
+	i = fcntl(vca_pipes[0], F_SETFL, i);
 
 	AZ(pthread_create(&vca_kqueue_thread, NULL, vca_kqueue_main, NULL));
 }
 
 struct acceptor acceptor_kqueue = {
 	.name =		"kqueue",
 	.init =		vca_kqueue_init,
-	.recycle =	vca_kqueue_recycle,
 };
 
 #endif /* defined(HAVE_KQUEUE) */