diff --git a/Makefile.am b/Makefile.am index cfceae3e..70bfda48 100644 --- a/Makefile.am +++ b/Makefile.am @@ -232,3 +232,6 @@ tarsnap_keygen_CFLAGS= \ tarsnap_keygen_LDADD= -lcrypto tarsnap_keygen_dist_man_MANS= keygen/tarsnap-keygen.1 + +# Tarsnap sample configuration file +sysconf_DATA = tar/tarsnap.conf.sample diff --git a/storage/storage.h b/storage/storage.h index b316914b..1d720af0 100644 --- a/storage/storage.h +++ b/storage/storage.h @@ -13,6 +13,12 @@ typedef struct storage_read_internal STORAGE_R; typedef struct storage_write_internal STORAGE_W; typedef struct storage_delete_internal STORAGE_D; +/* + * Should the storage code be aggressive in its networking by using multiple + * TCP connections for writes? + */ +extern int storage_aggressive_networking; + /** * storage_read_init(machinenum): * Prepare for read operations. Note that since reads are non-transactional, diff --git a/storage/storage_write.c b/storage/storage_write.c index 819320a4..6efd0107 100644 --- a/storage/storage_write.c +++ b/storage/storage_write.c @@ -18,12 +18,24 @@ */ #define MAXPENDING_WRITEBYTES (5 * 1024 * 1024) +/* + * Number of connections to use for writes when --aggressive-networking is + * enabled. This MUST NOT be set to more than 8. + */ +#define AGGRESSIVE_CNUM 8 + struct storage_write_internal { /* Transaction parameters. */ - NETPACKET_CONNECTION * NPC; + NETPACKET_CONNECTION * NPC[AGGRESSIVE_CNUM]; uint64_t machinenum; uint8_t nonce[32]; + /* Number of connections to use. */ + size_t numconns; + + /* Last connection through which a request was sent. */ + size_t lastcnum; + /* Number of bytes of pending writes. */ size_t nbytespending; }; @@ -62,6 +74,12 @@ static handlepacket_callback callback_fexist_response; static sendpacket_callback callback_write_file_send; static handlepacket_callback callback_write_file_response; +/* + * Should the storage code be aggressive in its networking by using multiple + * TCP connections for writes? + */ +int storage_aggressive_networking = 0; + /** * storage_write_start(machinenum, lastseq, seqnum): * Start a write transaction, presuming that ${lastseq} is the the sequence @@ -74,6 +92,7 @@ storage_write_start(uint64_t machinenum, const uint8_t lastseq[32], uint8_t seqnum[32]) { struct storage_write_internal * S; + size_t i; /* Allocate memory. */ if ((S = malloc(sizeof(struct storage_write_internal))) == NULL) @@ -82,15 +101,23 @@ storage_write_start(uint64_t machinenum, const uint8_t lastseq[32], /* Store machine number. */ S->machinenum = machinenum; + /* Figure out how many connections to use. */ + S->numconns = storage_aggressive_networking ? AGGRESSIVE_CNUM : 1; + + /* No connections used yet. */ + S->lastcnum = 0; + /* No pending writes so far. */ S->nbytespending = 0; - /* Open netpacket connection. */ - if ((S->NPC = netpacket_open()) == NULL) - goto err1; + /* Open netpacket connections. */ + for (i = 0; i < S->numconns; i++) { + if ((S->NPC[i] = netpacket_open()) == NULL) + goto err1; + } /* Start a write transaction. */ - if (storage_transaction_start_write(S->NPC, machinenum, + if (storage_transaction_start_write(S->NPC[0], machinenum, lastseq, S->nonce)) goto err2; @@ -101,8 +128,10 @@ storage_write_start(uint64_t machinenum, const uint8_t lastseq[32], return (S); err2: - netpacket_close(S->NPC); + i = S->numconns; err1: + for (i--; i < S->numconns; i--) + netpacket_close(S->NPC[i]); free(S); err0: /* Failure! */ @@ -128,7 +157,7 @@ storage_write_fexist(STORAGE_W * S, char class, const uint8_t name[32]) C.done = 0; /* Ask the netpacket layer to send a request and get a response. */ - if (netpacket_op(S->NPC, callback_fexist_send, &C)) + if (netpacket_op(S->NPC[0], callback_fexist_send, &C)) goto err0; /* Wait until the server has responded or we have failed. */ @@ -269,7 +298,8 @@ storage_write_file(STORAGE_W * S, uint8_t * buf, size_t len, } /* Ask the netpacket layer to send a request and get a response. */ - if (netpacket_op(S->NPC, callback_write_file_send, C)) + S->lastcnum = (S->lastcnum + 1) % S->numconns; + if (netpacket_op(S->NPC[S->lastcnum], callback_write_file_send, C)) goto err2; /* Success! */ @@ -396,14 +426,16 @@ storage_write_flush(STORAGE_W * S) int storage_write_end(STORAGE_W * S) { + size_t i; /* Flush any pending writes. */ if (storage_write_flush(S)) goto err2; - /* Close netpacket connection. */ - if (netpacket_close(S->NPC)) - goto err1; + /* Close netpacket connections. */ + for (i = S->numconns - 1; i < S->numconns; i--) + if (netpacket_close(S->NPC[i])) + goto err1; /* Free structure. */ free(S); @@ -412,8 +444,10 @@ storage_write_end(STORAGE_W * S) return (0); err2: - netpacket_close(S->NPC); + i = S->numconns; err1: + for (i--; i < S->numconns; i--) + netpacket_close(S->NPC[i]); free(S); /* Failure! */ @@ -428,9 +462,11 @@ storage_write_end(STORAGE_W * S) void storage_write_free(STORAGE_W * S) { + size_t i; - /* Close netpacket connection. */ - netpacket_close(S->NPC); + /* Close netpacket connections. */ + for (i = S->numconns - 1; i < S->numconns; i--) + netpacket_close(S->NPC[i]); /* Free structure. */ free(S); diff --git a/tar-version b/tar-version index ece61c60..f9cbc01a 100644 --- a/tar-version +++ b/tar-version @@ -1 +1 @@ -1.0.6 \ No newline at end of file +1.0.7 \ No newline at end of file diff --git a/tar/bsdtar.c b/tar/bsdtar.c index 9e83fd75..03bb7954 100644 --- a/tar/bsdtar.c +++ b/tar/bsdtar.c @@ -84,6 +84,7 @@ struct option { #include "bsdtar.h" #include "crypto.h" #include "network.h" +#include "storage.h" #include "sysendian.h" #if !HAVE_DECL_OPTARG @@ -135,7 +136,8 @@ static const char *tar_opts = "+@:BC:cdf:HhI:kLlmnOoPprtT:UvW:wX:x"; /* Fake short equivalents for long options that otherwise lack them. */ enum { - OPTION_CACHEDIR=1, + OPTION_AGGRESSIVE_NETWORKING=1, + OPTION_CACHEDIR, OPTION_CHECK_LINKS, OPTION_CHROOT, OPTION_EXCLUDE, @@ -170,6 +172,7 @@ enum { */ static const struct option tar_longopts[] = { { "absolute-paths", no_argument, NULL, 'P' }, + { "aggressive-networking", no_argument, NULL, OPTION_AGGRESSIVE_NETWORKING }, { "cachedir", required_argument, NULL, OPTION_CACHEDIR }, { "cd", required_argument, NULL, 'C' }, { "check-links", no_argument, NULL, OPTION_CHECK_LINKS }, @@ -332,6 +335,9 @@ main(int argc, char **argv) */ while ((opt = bsdtar_getopt(bsdtar, tar_opts, &option)) != -1) { switch (opt) { + case OPTION_AGGRESSIVE_NETWORKING: /* tarsnap */ + storage_aggressive_networking = 1; + break; case 'B': /* GNU tar */ /* libarchive doesn't need this; just ignore it. */ break; @@ -642,6 +648,8 @@ main(int argc, char **argv) only_mode(bsdtar, "--null", "cxt"); /* Check boolean options only permitted in certain modes. */ + if (storage_aggressive_networking) + only_mode(bsdtar, "--aggressive-networking", "c"); if (bsdtar->option_dont_traverse_mounts) only_mode(bsdtar, "--one-file-system", "c"); if (bsdtar->option_fast_read) @@ -1001,8 +1009,8 @@ configfile_helper(struct bsdtar *bsdtar, const char *line) char * conf_arg; size_t optlen; - /* Ignore comments. */ - if (line[0] == '#') + /* Ignore comments and blank lines. */ + if ((line[0] == '#') || (line[0] == '\0')) return (0); /* Duplicate line. */ @@ -1023,7 +1031,7 @@ configfile_helper(struct bsdtar *bsdtar, const char *line) /* * If the line is whitespace-terminated, there might not be - * an option here after all. + * an argument here after all. */ if (conf_arg[0] == '\0') conf_arg = NULL; @@ -1093,7 +1101,8 @@ configfile_helper(struct bsdtar *bsdtar, const char *line) bsdtar->option_totals++; } else { bsdtar_errc(bsdtar, 1, 0, - "Unrecognized configuration file option: %s", conf_opt); + "Unrecognized configuration file option: \"%s\"", + conf_opt); } /* Free memory allocated by strdup. */ diff --git a/tar/tarsnap.1 b/tar/tarsnap.1 index 314f028f..1474cc92 100644 --- a/tar/tarsnap.1 +++ b/tar/tarsnap.1 @@ -146,6 +146,16 @@ The specified .Nm archive is read and the entries in it will be appended to the current archive. +.It Fl -aggressive-networking +(c mode only) +Use multiple TCP connections to send data to the +.Nm +server. +If the upload rate is congestion-limited rather than +being limited by individual bottleneck(s), this may +allow tarsnap to use a significantly larger fraction +of the available bandwidth, at the expense of slowing +down any other network traffic. .It Fl C Ar directory (c and x modes only) In c mode, this changes the directory before adding diff --git a/tar/tarsnap.conf.sample b/tar/tarsnap.conf.sample new file mode 100644 index 00000000..eac36114 --- /dev/null +++ b/tar/tarsnap.conf.sample @@ -0,0 +1,11 @@ +# Tarsnap cache directory +cachedir /usr/local/tarsnap-cache + +# Tarsnap key file +keyfile /root/tarsnap.key + +# Don't archive files which have the nodump flag set +nodump + +# Print statistics when creating or deleting archievs +print-stats