Tarsnap 1.0.11

Changes since 1.0.10:
* Add --humanize-numbers option which "humanizes" statistics
  printed by the --print-stats option.
* Add --maxbw option to interrupt archival when a specified amount
  of upload bandwidth has been used.
* Make warnings about network glitches (e.g., "Connection lost,
  waiting X seconds before reconnecting") less noisy, but add a
  --noisy-warnings option to restore the old behaviour (probably
  useful for debugging purposes only).

Makefile.am

@@ -155,6 +155,7 @@ tarsnap_SOURCES=				\
 		util/entropy.c			\
 		util/hexify.c			\
 		util/hexlink.c			\
+		util/humansize.c		\
 		util/sigquit.c
 
 tarsnap_DEPENDENCIES = libarchive.a		\

chunks/chunks_stats_internal.c

@@ -3,7 +3,9 @@
 #include <inttypes.h>
 #include <stdio.h>
 
+#include "humansize.h"
 #include "storage.h"
+#include "tarsnap_opt.h"
 #include "warnp.h"
 
 #include "chunks_internal.h"
@@ -87,28 +89,57 @@ chunks_stats_print(FILE * stream, struct chunkstats * stats,
     const char * name, struct chunkstats * stats_extra)
 {
 	struct chunkstats s;
+	char * s_lenstr, * s_zlenstr;
 
 	/* Compute sum of stats and stats_extra. */
 	s.nchunks = stats->nchunks + stats_extra->nchunks;
 	s.s_len = stats->s_len + stats_extra->s_len;
 	s.s_zlen = stats->s_zlen + stats_extra->s_zlen;
 
+	/* Stringify values, */
+	if (tarsnap_opt_humanize_numbers) {
+		if ((s_lenstr = humansize(s.s_len)) == NULL)
+			goto err0;
+		if ((s_zlenstr = humansize(s.s_zlen +
+		    s.nchunks * STORAGE_FILE_OVERHEAD)) == NULL)
+			goto err1;
+	} else {
+		if (asprintf(&s_lenstr, "%15" PRIu64, s.s_len) == -1) {
+			warnp("asprintf");
+			goto err0;
+		}
+		if (asprintf(&s_zlenstr, "%15" PRIu64,
+		    s.s_zlen + s.nchunks * STORAGE_FILE_OVERHEAD) == -1) {
+			warnp("asprintf");
+			goto err1;
+		}
+	}
+
+	/* Print output line. */
 	if (fprintf(stream,
 #ifdef STATS_WITH_CHUNKS
-	    "%-25s  %12" PRIu64 "  %15" PRIu64 "  %15" PRIu64 "\n",
-	    name, s.nchunks, s.s_len,
+	    "%-25s  %12" PRIu64 "  %15s  %15s\n",
+	    name, s.nchunks,
 #else
-	    "%-32s  %15" PRIu64 "  %15" PRIu64 "\n",
-	    name, s.s_len,
+	    "%-32s  %15s  %15s\n",
+	    name,
 #endif
-	    s.s_zlen + s.nchunks * STORAGE_FILE_OVERHEAD) < 0) {
+	    s_lenstr, s_zlenstr) < 0) {
 		warnp("fprintf");
-		goto err0;
+		goto err2;
 	}
 
+	/* Free strings allocated by asprintf or humansize. */
+	free(s_zlenstr);
+	free(s_lenstr);
+
 	/* Success! */
 	return (0);
 
+err2:
+	free(s_zlenstr);
+err1:
+	free(s_lenstr);
 err0:
 	/* Failure! */
 	return (-1);

keygen/keygen.c

@@ -14,6 +14,7 @@
 #include "netproto.h"
 #include "network.h"
 #include "sysendian.h"
+#include "tarsnap_opt.h"
 #include "warnp.h"
 
 /* Length of buffer for reading password. */
@@ -42,6 +43,9 @@ static handlepacket_callback callback_register_challenge;
 static handlepacket_callback callback_register_response;
 static void usage(void);
 
+/* Be noisy about network errors while registering a machine. */
+int tarsnap_opt_noisy_warnings = 1;
+
 static void
 usage(void)
 {

netpacket/netpacket.h

@@ -163,6 +163,14 @@ NETPACKET_CONNECTION * netpacket_open(void);
  */
 int netpacket_op(NETPACKET_CONNECTION *, sendpacket_callback *, void *);
 
+/**
+ * netpacket_getstats(NPC, in, out, queued):
+ * Obtain the number of bytes received and sent via the connection, and the
+ * number of bytes queued to be written.
+ */
+void netpacket_getstats(NETPACKET_CONNECTION *, uint64_t *, uint64_t *,
+    uint64_t *);
+
 /**
  * netpacket_close(NPC):
  * Close a netpacket connection.

netpacket/netpacket_internal.h

@@ -7,6 +7,7 @@
 
 struct netpacket_op {
 	sendpacket_callback * writepacket;
+	int (* getbuf)(void *, uint8_t, uint8_t **, size_t);
 	handlepacket_callback * handlepacket;
 	void * cookie;
 
@@ -33,8 +34,9 @@ struct netpacket_internal {
 	/* Non-zero if a netproto_readpacket call is pending. */
 	int reading;
 
-	/* Function for getting a buffer. */
-	int (* getbuf)(void *, uint8_t, uint8_t **, size_t);
+	/* Bandwidth totals from dead connections. */
+	uint64_t bytesin;
+	uint64_t bytesout;
 
 	/* Queue of uncompleted operations. */
 	struct netpacket_op * pending_head;

netpacket/netpacket_op.c

@@ -1,12 +1,15 @@
 #include "bsdtar_platform.h"
 
+#include <sys/time.h>
+
 #include <stdlib.h>
 #include <string.h>
 
 #include "crypto.h"
 #include "netpacket_internal.h"
 #include "netproto.h"
 #include "sysendian.h"
+#include "tarsnap_opt.h"
 #include "warnp.h"
 
 #include "netpacket.h"
@@ -25,6 +28,9 @@ static int reconnect_wait[MAXRECONNECTS + 1] = {
     0, 0, 1, 2, 4, 8, 15, 30, 60, 90, 90
 };
 
+/* Time before which we shouldn't print a "connection lost" warning. */
+static struct timeval next_connlost_warning = { 0, 0};
+
 /**
  * netpacket_open(void):
  * Return a netpacket connection cookie.
@@ -52,8 +58,8 @@ netpacket_open(void)
 	/* We're not reading a packet yet. */
 	NPC->reading = 0;
 
-	/* We're using the default getbuf function. */
-	NPC->getbuf = callback_getbuf;
+	/* We haven't used any bandwidth yet. */
+	NPC->bytesin = NPC->bytesout = 0;
 
 	/* The queue is empty. */
 	NPC->pending_head = NPC->pending_tail = NPC->pending_current = NULL;
@@ -119,8 +125,9 @@ netpacket_op(NETPACKET_CONNECTION * NPC,
 	if ((op = malloc(sizeof(struct netpacket_op))) == NULL)
 		goto err0;
 
-	/* Store parameters for request. */
+	/* Store parameters for request, including default getbuf callback. */
 	op->writepacket = writepacket;
+	op->getbuf = callback_getbuf;
 	op->cookie = cookie;
 
 	/* Add operation to queue. */
@@ -167,6 +174,7 @@ static int
 callback_reconnect(void * cookie, int status)
 {
 	struct netpacket_internal * NPC = cookie;
+	uint64_t in, out, queued;
 
 	/* If we're being cancelled, return. */
 	if (status == NETWORK_STATUS_CANCEL)
@@ -178,6 +186,11 @@ callback_reconnect(void * cookie, int status)
 		goto err0;
 	}
 
+	/* Add the bandwidth used by the connection to our running totals. */
+	netproto_getstats(NPC->NC, &in, &out, &queued);
+	NPC->bytesin += in;
+	NPC->bytesout += out;
+
 	/* Close the (dead) connection. */
 	if (netproto_close(NPC->NC))
 		goto err1;
@@ -201,6 +214,7 @@ callback_reconnect(void * cookie, int status)
 static int
 reconnect(NETPACKET_CONNECTION * NPC)
 {
+	struct timeval tp;
 	int nseconds;
 
 	/* Flush any pending activity on the socket. */
@@ -224,10 +238,23 @@ reconnect(NETPACKET_CONNECTION * NPC)
 	/* Figure out how long we ought to wait before reconnecting. */
 	nseconds = reconnect_wait[NPC->ndrops];
 
-	/* Warn the user that we're waiting. */
-	if (nseconds != 0)
+	/*
+	 * Warn the user that we're waiting, if we haven't already printed a
+	 * warning message recently.
+	 */
+	if (gettimeofday(&tp, NULL)) {
+		warnp("gettimeofday");
+		goto err0;
+	}
+	if ((nseconds >= (tarsnap_opt_noisy_warnings ? 1 : 30)) &&
+	    ((tp.tv_sec > next_connlost_warning.tv_sec) ||
+	        ((tp.tv_sec == next_connlost_warning.tv_sec) &&
+		(tp.tv_usec > next_connlost_warning.tv_usec)))) {
 		warn0("Connection lost, "
 		    "waiting %d seconds before reconnecting", nseconds);
+		next_connlost_warning.tv_sec = tp.tv_sec + nseconds;
+		next_connlost_warning.tv_usec = tp.tv_usec;
+	}
 
 	/* Set a callback to reconnect. */
 	if (netproto_sleep(NPC->NC, nseconds, callback_reconnect, NPC))
@@ -260,7 +287,7 @@ netpacket_op_packetsent(void * cookie, int status)
 
 	/* We want to read a response packet if we're not already doing so. */
 	if (NPC->reading == 0) {
-		if (netproto_readpacket(NPC->NC, NPC->getbuf,
+		if (netproto_readpacket(NPC->NC, NPC->pending_head->getbuf,
 		    callback_packetreceived, NPC))
 			goto err0;
 		NPC->reading = 1;
@@ -376,7 +403,7 @@ callback_packetreceived(void * cookie, int status)
 	/* Read another packet if appropriate. */
 	if ((NPC->pending_head != NULL) &&
 	    (NPC->pending_head->handlepacket != NULL)) {
-		if (netproto_readpacket(NPC->NC, NPC->getbuf,
+		if (netproto_readpacket(NPC->NC, NPC->pending_head->getbuf,
 		    callback_packetreceived, NPC))
 			goto err0;
 	} else {
@@ -396,6 +423,27 @@ callback_packetreceived(void * cookie, int status)
 	return (-1);
 }
 
+/**
+ * netpacket_getstats(NPC, in, out, queued):
+ * Obtain the number of bytes received and sent via the connection, and the
+ * number of bytes queued to be written.
+ */
+void
+netpacket_getstats(NETPACKET_CONNECTION * NPC, uint64_t * in, uint64_t * out,
+    uint64_t * queued)
+{
+
+	/* Get statistics from the current connection if one exists. */
+	if (NPC->NC != NULL)
+		netproto_getstats(NPC->NC, in, out, queued);
+	else
+		*in = *out = *queued = 0;
+
+	/* Add statistics from past connections. */
+	*in += NPC->bytesin;
+	*out += NPC->bytesout;
+}
+
 /**
  * netpacket_close(NPC):
  * Close a netpacket connection.

netproto/netproto.c

@@ -12,11 +12,11 @@
 static network_callback callback_sleep;
 
 /**
- * netproto_printerr(status):
+ * _netproto_printerr(status):
  * Print the error message associated with the given status code.
  */
 void
-netproto_printerr(int status)
+_netproto_printerr(int status)
 {
 
 	switch (status) {

netproto/netproto.h

@@ -12,10 +12,15 @@ typedef struct netproto_connection_internal NETPROTO_CONNECTION;
     (NETWORK_STATUS_MAX + 1)		/* Protocol error. */
 
 /**
- * netproto_printerr(status):
+ * _netproto_printerr(status):
  * Print the error message associated with the given status code.
  */
-void netproto_printerr(int);
+void _netproto_printerr(int);
+
+#define netproto_printerr(x)	do {	\
+	warnline;			\
+	_netproto_printerr(x);		\
+} while (0)
 
 /**
  * netproto_connect(callback, cookie):

network/network_buf.c

@@ -8,6 +8,7 @@
 #include <string.h>
 
 #include "network_internal.h"
+#include "tarsnap_opt.h"
 #include "warnp.h"
 
 #include "network.h"
@@ -133,11 +134,12 @@ callback_buf(void * cookie, int status)
 	return (0);
 
 docallback:
-#if 0
-	/* If status is NETWORK_STATUS_ERR, print a warning from errno. */
-	if (status == NETWORK_STATUS_ERR)
+	/*
+	 * If there was a network error and we're being verbose, print the
+	 * error now in case errno gets mangled later.
+	 */
+	if ((tarsnap_opt_noisy_warnings) && (status == NETWORK_STATUS_ERR))
 		warnp("Network error");
-#endif
 
 	/* Call the user callback. */
 	rc = (C->callback)(C->cookie, status);

storage/storage.h

@@ -13,12 +13,6 @@ typedef struct storage_read_internal	STORAGE_R;
 typedef struct storage_write_internal	STORAGE_W;
 typedef struct storage_delete_internal	STORAGE_D;
 
-/*
- * Should the storage code be aggressive in its networking by using multiple
- * TCP connections for writes?
- */
-extern int storage_aggressive_networking;
-
 /**
  * storage_read_init(machinenum):
  * Prepare for read operations.  Note that since reads are non-transactional,