Ticket3723 03 squashed rebased

changes/ticket3723

@@ -0,0 +1,3 @@
+  o Minor features (directory authority):
+    - When a bandwidth file is used to obtain the bandwidth measurements,
+      include this bandwidth file headers in the votes. Closes ticket 3723.

src/app/config/config.c

@@ -3560,7 +3560,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "(Bridge/V3)AuthoritativeDir is set.");
     /* If we have a v3bandwidthsfile and it's broken, complain on startup */
     if (options->V3BandwidthsFile && !old_options) {
-      dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL);
+      dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
     }
     /* same for guardfraction file */
     if (options->GuardfractionFile && !old_options) {

src/feature/dirauth/dirvote.c

@@ -254,6 +254,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     /* XXXX Abstraction violation: should be pulling a field out of v3_ns.*/
     char *flag_thresholds = dirserv_get_flag_thresholds_line();
     char *params;
+    char *bw_headers_line = NULL;
     authority_cert_t *cert = v3_ns->cert;
     char *methods =
       make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD,
@@ -267,8 +268,32 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
       params = smartlist_join_strings(v3_ns->net_params, " ", 0, NULL);
     else
       params = tor_strdup("");
-
     tor_assert(cert);
+
+    /* v3_ns->bw_file_headers is only set when V3BandwidthsFile is
+     * configured */
+    if (v3_ns->bw_file_headers) {
+      char *bw_file_headers = NULL;
+      /* If there are too many headers, leave the header string NULL */
+      if (! BUG(smartlist_len(v3_ns->bw_file_headers)
+                > MAX_BW_FILE_HEADER_COUNT_IN_VOTE)) {
+        bw_file_headers = smartlist_join_strings(v3_ns->bw_file_headers, " ",
+                                                 0, NULL);
+        if (BUG(strlen(bw_file_headers) > MAX_BW_FILE_HEADERS_LINE_LEN)) {
+          /* Free and set to NULL, because the line was too long */
+          tor_free(bw_file_headers);
+        }
+      }
+      if (!bw_file_headers) {
+          /* If parsing failed, add a bandwidth header line with no entries */
+          bw_file_headers = tor_strdup("");
+      }
+      /* At this point, the line will always be present */
+      bw_headers_line = format_line_if_present("bandwidth-file-headers",
+                                               bw_file_headers);
+      tor_free(bw_file_headers);
+    }
+
     smartlist_add_asprintf(chunks,
                  "network-status-version 3\n"
                  "vote-status %s\n"
@@ -286,7 +311,9 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                  "params %s\n"
                  "dir-source %s %s %s %s %d %d\n"
                  "contact %s\n"
-                 "%s", /* shared randomness information */
+                 "%s" /* shared randomness information */
+                 "%s" /* bandwidth file headers */
+                 ,
                  v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion",
                  methods,
                  published, va, fu, vu,
@@ -302,13 +329,16 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                  fmt_addr32(addr), voter->dir_port, voter->or_port,
                  voter->contact,
                  shared_random_vote_str ?
-                           shared_random_vote_str : "");
+                           shared_random_vote_str : "",
+                 bw_headers_line ?
+                           bw_headers_line : "");
 
     tor_free(params);
     tor_free(flags);
     tor_free(flag_thresholds);
     tor_free(methods);
     tor_free(shared_random_vote_str);
+    tor_free(bw_headers_line);
 
     if (!tor_digest_is_zero(voter->legacy_id_digest)) {
       char fpbuf[HEX_DIGEST_LEN+1];
@@ -4303,6 +4333,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   digestmap_t *omit_as_sybil = NULL;
   const int vote_on_reachability = running_long_enough_to_decide_unreachable();
   smartlist_t *microdescriptors = NULL;
+  smartlist_t *bw_file_headers = NULL;
 
   tor_assert(private_key);
   tor_assert(cert);
@@ -4338,7 +4369,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
    * set_routerstatus_from_routerinfo() see up-to-date bandwidth info.
    */
   if (options->V3BandwidthsFile) {
-    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL);
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
   } else {
     /*
      * No bandwidths file; clear the measured bandwidth cache in case we had
@@ -4440,8 +4471,10 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
 
   /* This pass through applies the measured bw lines to the routerstatuses */
   if (options->V3BandwidthsFile) {
+    /* Only set bw_file_headers when V3BandwidthsFile is configured */
+    bw_file_headers = smartlist_new();
     dirserv_read_measured_bandwidths(options->V3BandwidthsFile,
-                                     routerstatuses);
+                                     routerstatuses, bw_file_headers);
   } else {
     /*
      * No bandwidths file; clear the measured bandwidth cache in case we had
@@ -4537,6 +4570,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
                            options->ConsensusParams, NULL, 0, 0);
     smartlist_sort_strings(v3_out->net_params);
   }
+  v3_out->bw_file_headers = bw_file_headers;
 
   voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
   voter->nickname = tor_strdup(options->Nickname);

src/feature/dirauth/dirvote.h

@@ -89,6 +89,9 @@
 #define DGV_INCLUDE_PENDING 2
 #define DGV_INCLUDE_PREVIOUS 4
 
+/** Maximum size of a line in a vote. */
+#define MAX_BW_FILE_HEADERS_LINE_LEN 1024
+
 /*
  * Public API. Used outside of the dirauth subsystem.
  *

src/feature/dircache/dirserv.c

@@ -51,6 +51,7 @@
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/encoding/confline.h"
 
+#include "lib/encoding/keyval.h"
 /**
  * \file dirserv.c
  * \brief Directory server core implementation. Manages directory
@@ -2599,12 +2600,14 @@ measured_bw_line_apply(measured_bw_line_t *parsed_line,
 }
 
 /**
- * Read the measured bandwidth file and apply it to the list of
- * vote_routerstatus_t. Returns -1 on error, 0 otherwise.
+ * Read the measured bandwidth list file, apply it to the list of
+ * vote_routerstatus_t and store all the headers in <b>bw_file_headers</b>.
+ * Returns -1 on error, 0 otherwise.
  */
 int
 dirserv_read_measured_bandwidths(const char *from_file,
-                                 smartlist_t *routerstatuses)
+                                 smartlist_t *routerstatuses,
+                                 smartlist_t *bw_file_headers)
 {
   FILE *fp = tor_fopen_cloexec(from_file, "r");
   int applied_lines = 0;
@@ -2654,6 +2657,12 @@ dirserv_read_measured_bandwidths(const char *from_file,
     goto err;
   }
 
+  /* If timestamp was correct and bw_file_headers is not NULL,
+   * add timestamp to bw_file_headers */
+  if (bw_file_headers)
+    smartlist_add_asprintf(bw_file_headers, "timestamp=%lu",
+                           (unsigned long)file_time);
+
   if (routerstatuses)
     smartlist_sort(routerstatuses, compare_vote_routerstatus_entries);
 
@@ -2669,7 +2678,24 @@ dirserv_read_measured_bandwidths(const char *from_file,
         dirserv_cache_measured_bw(&parsed_line, file_time);
         if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0)
           applied_lines++;
-      }
+      /* if the terminator is found, it is the end of header lines, set the
+       * flag but do not store anything */
+      } else if (strcmp(line, BW_FILE_HEADERS_TERMINATOR) == 0) {
+        line_is_after_headers = 1;
+      /* if the line was not a correct relay line nor the terminator and
+       * the end of the header lines has not been detected yet
+       * and it is key_value and bw_file_headers did not reach the maximum
+       * number of headers,
+       * then assume this line is a header and add it to bw_file_headers */
+      } else if (bw_file_headers &&
+              (line_is_after_headers == 0) &&
+              string_is_key_value(LOG_DEBUG, line) &&
+              !strchr(line, ' ') &&
+              (smartlist_len(bw_file_headers)
+               < MAX_BW_FILE_HEADER_COUNT_IN_VOTE)) {
+        line[strlen(line)-1] = '\0';
+        smartlist_add_strdup(bw_file_headers, line);
+      };
     }
   }
 

src/feature/dircache/dirserv.h

@@ -49,6 +49,13 @@ typedef enum {
 /** Maximum allowable length of a version line in a networkstatus. */
 #define MAX_V_LINE_LEN 128
 
+/** Maximum allowable length of bandwidth headers in a bandwidth file */
+#define MAX_BW_FILE_HEADER_COUNT_IN_VOTE 50
+
+/** Terminatore that separates bandwidth file headers from bandwidth file
+ * relay lines */
+#define BW_FILE_HEADERS_TERMINATOR "=====\n"
+
 /** Ways to convert a spoolable_resource_t to a bunch of bytes. */
 typedef enum dir_spool_source_t {
     DIR_SPOOL_SERVER_BY_DIGEST=1, DIR_SPOOL_SERVER_BY_FP,
@@ -215,7 +222,8 @@ dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
 #endif /* defined(DIRSERV_PRIVATE) */
 
 int dirserv_read_measured_bandwidths(const char *from_file,
-                                     smartlist_t *routerstatuses);
+                                     smartlist_t *routerstatuses,
+                                     smartlist_t *bw_file_headers);
 
 int dirserv_read_guardfraction_file(const char *fname,
                                  smartlist_t *vote_routerstatuses);

src/feature/nodelist/networkstatus.c

@@ -385,6 +385,11 @@ networkstatus_vote_free_(networkstatus_t *ns)
     smartlist_free(ns->routerstatus_list);
   }
 
+  if (ns->bw_file_headers) {
+    SMARTLIST_FOREACH(ns->bw_file_headers, char *, c, tor_free(c));
+    smartlist_free(ns->bw_file_headers);
+  }
+
   digestmap_free(ns->desc_digest_map, NULL);
 
   if (ns->sr_info.commits) {

src/feature/nodelist/networkstatus_st.h

@@ -96,6 +96,9 @@ struct networkstatus_t {
 
   /** Contains the shared random protocol data from a vote or consensus. */
   networkstatus_sr_info_t sr_info;
+
+  /** List of key=value strings from the headers of the bandwidth list file */
+  smartlist_t *bw_file_headers;
 };
 
 #endif