Add VCL_Shutdown to wait for VCL references to vanish

bin/varnishd/cache/cache_main.c

@@ -362,6 +362,20 @@ cli_quit(int sig)
 	WRONG("It's time for the big quit");
 }
 
+/*=====================================================================
+ * XXX Generalize?
+ */
+
+static void
+bit_set(volatile uint8_t *p, unsigned no)
+{
+	uint8_t b;
+
+	p += (no >> 3);
+	b = (0x80 >> (no & 7));
+	*p |= b;
+}
+
 /*=====================================================================
  * Run the child process
  */
@@ -452,12 +466,16 @@ child_main(int sigmagic, size_t altstksz)
 
 	CLI_Run();
 
+	bit_set(cache_param->debug_bits, DBG_VCLREL);
+
 	if (shutdown_delay > 0)
 		VTIM_sleep(shutdown_delay);
 
 	VCA_Shutdown();
 	BAN_Shutdown();
 	EXP_Shutdown();
+	STV_warn();
+	VCL_Shutdown();
 	STV_close();
 
 	printf("Child dies\n");

bin/varnishd/cache/cache_varnishd.h

@@ -491,6 +491,7 @@ struct vsb *VCL_Rel_CliCtx(struct vrt_ctx **);
 void VCL_Panic(struct vsb *, const char *nm, const struct vcl *);
 void VCL_Poll(void);
 void VCL_Init(void);
+void VCL_Shutdown(void);
 
 #define VCL_MET_MAC(l,u,t,b) \
     void VCL_##l##_method(struct vcl *, struct worker *, struct req *, \
@@ -569,6 +570,7 @@ void V2D_Init(void);
 
 /* stevedore.c */
 void STV_open(void);
+void STV_warn(void);
 void STV_close(void);
 const struct stevedore *STV_next(void);
 int STV_BanInfoDrop(const uint8_t *ban, unsigned len);

bin/varnishd/cache/cache_vcl.c

@@ -1027,3 +1027,25 @@ VCL_Init(void)
 	Lck_New(&vcl_mtx, lck_vcl);
 	VSL_Setup(&vsl_cli, NULL, 0);
 }
+
+void
+VCL_Shutdown(void)
+{
+	struct vcl *vcl;
+	unsigned c = 0;
+
+	while (1) {
+		Lck_Lock(&vcl_mtx);
+		VTAILQ_FOREACH(vcl, &vcl_head, list)
+			if (vcl->busy)
+				break;
+		Lck_Unlock(&vcl_mtx);
+		if (vcl == NULL)
+			return;
+		if (++c % 10 == 0) {
+			fprintf(stderr, "shutdown waiting for vcl %u\t%s\n",
+				vcl->busy, vcl->loaded_name);
+		}
+		usleep(100 * 1000);
+	}
+}

bin/varnishd/storage/stevedore.c

@@ -198,19 +198,28 @@ STV_open(void)
 	AN(stv_h2_rxbuf);
 }
 
+void
+STV_warn(void)
+{
+	struct stevedore *stv;
+
+	ASSERT_CLI();
+	/* First send close warning */
+	STV_Foreach(stv)
+		if (stv->close != NULL)
+			stv->close(stv, 0);
+}
+
 void
 STV_close(void)
 {
 	struct stevedore *stv;
-	int i;
 
 	ASSERT_CLI();
-	for (i = 1; i >= 0; i--) {
-		/* First send close warning */
-		STV_Foreach(stv)
-			if (stv->close != NULL)
-				stv->close(stv, i);
-	}
+	/* First send close warning */
+	STV_Foreach(stv)
+		if (stv->close != NULL)
+			stv->close(stv, 1);
 }
 
 /*-------------------------------------------------------------------

bin/varnishtest/tests/r01506.vtc

@@ -68,3 +68,4 @@ client c1 {
 } -run
 
 varnish v1 -expect sc_range_short == 0
+server s1 -wait

bin/varnishtest/tests/r02035.vtc

@@ -34,3 +34,4 @@ client c1 {
 
 	barrier b1 sync
 } -run
+server s1 -wait