Merge d585ebd into 674ef53

PADDING_TODO.txt

@@ -0,0 +1,64 @@
+TODO sketch for this branch, in rough priority order:
+
+- Clean up/fix XXX's and FIXMEs
+  - Test SMARTLIST_FOREACH_REVERSE_BEGIN()
+  - Tests for circuit getting marked or freed before callback.
+  - Tests for detecting padding from wrong hop
+  - Tests for machine replacement on all conditions
+  - Tests for STATE_END teardown due to infinity, bins_empty, and/or length
+    with pending timers, new machines, etc.
+  - Test histogram spacing for larger length histograms
+  - Tests for consensus and machine rate limiting
+  - Test all types of token removal
+  - Go down dist precision rabbithole and comment
+  - Tests for dists, length and iat sampling
+  - Better test coverage in general
+
+- Negotiate errors are not remembered.. If there is a consensus desync,
+  clients will keep trying machines that middles do not have..
+
+- Larger types?
+  - Larger/different mechanism for machine numbering than just uint8_t
+  - Larger circpad_delay_t?
+  - Larger circpad_hist_bin_t?
+  - Large state->range_sec for histograms?
+
+- Nick's API doc
+  - Consider how we'd use it.
+
+- Vanguards compatibility for MiddleNodes (via changes to vanguards addon)
+
+- Circuit RTT measurement will break on var_cell/EXTEND2 cells
+  - Are there any heuristics we can use here?
+
+- Rephist timer stats
+  - Is this a privacy risk? The adversary could create lots of circuits
+    to find a layer2 vanguard.. Otherwise they will be spread across middles.
+
+- Determine good hardcoded/example histograms
+
+- Specify target circuit lifetime?
+
+- Specify ports for machine??
+
+- circpad_machine_validate() function to sanity-check histograms loaded from
+  consensus/torrc (can also be used to help guide a GA).
+  - Check bin construction
+  - no type overflow (start_usec + range_sec, etc)
+  - no conflicting state transitions (or overlap with cancel events)
+  - no use of both histograms and iat_dist
+  - at least two histogram bins
+  - min_hop vs target_hop
+
+- Support torrc load+serialization of state machines
+  - ??
+
+- Support consensus load+serialization of state machines
+  - ??
+
+======== 0.3.6 ========
+
+- Come up with some good histograms for eg circuit setup fingerprinting,
+  website fingerprinting, and vanguards usage.
+- Experiment with run-length histograms and other histogram/distribution
+  encoding methods.

doc/tor.1.txt

@@ -1012,6 +1012,21 @@ The following options are useful only for clients (that is, if
     The .exit address notation, if enabled via MapAddress, overrides
     this option.
 
+[[MiddleNodes]] **MiddleNodes** __node__,__node__,__...__::
+    A list of identity fingerprints and country codes of nodes
+    to use for "middle" hops in your normal circuits.
+    Normal circuits include all circuits except for direct connections
+    to directory servers. Middle hops are all hops other than exit and entry. +
++
+    The HSLayer2Node and HSLayer3Node options override this option for onion
+    service circuits, if they are set. The vanguards addon will read this
+    option, and if set, it will set HSLayer2Nodes and HSLayer3Nodes to nodes
+    from this set.
++
+    The ExcludeNodes option overrides this option: any node listed in both
+    MiddleNodes and ExcludeNodes is treated as excluded. See
+    the **ExcludeNodes** option for more information on how to specify nodes.
+
 [[EntryNodes]] **EntryNodes** __node__,__node__,__...__::
     A list of identity fingerprints and country codes of nodes
     to use for the first hop in your normal circuits.
@@ -1028,13 +1043,14 @@ The following options are useful only for clients (that is, if
     If StrictNodes is set to 1, Tor will treat solely the ExcludeNodes option
     as a requirement to follow for all the circuits you generate, even if
     doing so will break functionality for you (StrictNodes applies to neither
-    ExcludeExitNodes nor to ExitNodes).  If StrictNodes is set to 0, Tor will
-    still try to avoid nodes in the ExcludeNodes list, but it will err on the
-    side of avoiding unexpected errors.  Specifically, StrictNodes 0 tells Tor
-    that it is okay to use an excluded node when it is *necessary* to perform
-    relay reachability self-tests, connect to a hidden service, provide a
-    hidden service to a client, fulfill a .exit request, upload directory
-    information, or download directory information.  (Default: 0)
+    ExcludeExitNodes nor to ExitNodes, nor to MiddleNodes).  If StrictNodes
+    is set to 0, Tor will still try to avoid nodes in the ExcludeNodes list,
+    but it will err on the side of avoiding unexpected errors.
+    Specifically, StrictNodes 0 tells Tor that it is okay to use an excluded
+    node when it is *necessary* to perform relay reachability self-tests,
+    connect to a hidden service, provide a hidden service to a client,
+    fulfill a .exit request, upload directory information, or download
+    directory information.  (Default: 0)
 
 [[FascistFirewall]] **FascistFirewall** **0**|**1**::
     If 1, Tor will only create outgoing connections to ORs running on ports

src/app/config/config.c

@@ -413,6 +413,10 @@ static config_var_t option_vars_[] = {
   V(ExcludeExitNodes,            ROUTERSET, NULL),
   OBSOLETE("ExcludeSingleHopRelays"),
   V(ExitNodes,                   ROUTERSET, NULL),
+  /* Researchers need a way to tell their clients to use specific
+   * middles that they also control, to allow safe live-network
+   * experimentation with new padding machines. */
+  V(MiddleNodes,                 ROUTERSET, NULL),
   V(ExitPolicy,                  LINELIST, NULL),
   V(ExitPolicyRejectPrivate,     BOOL,     "1"),
   V(ExitPolicyRejectLocalInterfaces, BOOL, "0"),
@@ -1724,6 +1728,7 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out)
   int routerset_usage =
     routerset_needs_geoip(options->EntryNodes) ||
     routerset_needs_geoip(options->ExitNodes) ||
+    routerset_needs_geoip(options->MiddleNodes) ||
     routerset_needs_geoip(options->ExcludeExitNodes) ||
     routerset_needs_geoip(options->ExcludeNodes) ||
     routerset_needs_geoip(options->HSLayer2Nodes) ||
@@ -2166,6 +2171,7 @@ options_act(const or_options_t *old_options)
                          options->HSLayer2Nodes) ||
         !routerset_equal(old_options->HSLayer3Nodes,
                          options->HSLayer3Nodes) ||
+        !routerset_equal(old_options->MiddleNodes, options->MiddleNodes) ||
         options->StrictNodes != old_options->StrictNodes) {
       log_info(LD_CIRC,
                "Changed to using entry guards or bridges, or changed "

src/app/config/or_options_st.h

@@ -72,6 +72,9 @@ struct or_options_t {
   routerset_t *ExitNodes; /**< Structure containing nicknames, digests,
                            * country codes and IP address patterns of ORs to
                            * consider as exits. */
+  routerset_t *MiddleNodes; /**< Structure containing nicknames, digests,
+                             * country codes and IP address patterns of ORs to
+                             * consider as middles. */
   routerset_t *EntryNodes;/**< Structure containing nicknames, digests,
                            * country codes and IP address patterns of ORs to
                            * consider as entry points. */

src/app/main/main.c

@@ -21,6 +21,7 @@
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/channelpadding.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/channeltls.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitmux_ewma.h"
@@ -650,9 +651,13 @@ tor_init(int argc, char *argv[])
   /* The options are now initialised */
   const or_options_t *options = get_options();
 
-  /* Initialize channelpadding parameters to defaults until we get
-   * a consensus */
+  /* Initialize channelpadding and circpad parameters to defaults
+   * until we get a consensus */
   channelpadding_new_consensus_params(NULL);
+  circpad_new_consensus_params(NULL);
+
+  /* Initialize circuit padding to defaults+torrc until we get a consensus */
+  circpad_machines_init();
 
   /* Initialize predicted ports list after loading options */
   predicted_ports_init();
@@ -771,6 +776,7 @@ tor_free_all(int postfork)
   dns_free_all();
   clear_pending_onions();
   circuit_free_all();
+  circpad_machines_free();
   entry_guards_free_all();
   pt_free_all();
   channel_tls_free_all();

src/core/include.am

@@ -30,6 +30,7 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/core/or/circuitlist.c		\
 	src/core/or/circuitmux.c		\
 	src/core/or/circuitmux_ewma.c		\
+	src/core/or/circuitpadding.c		\
 	src/core/or/circuitstats.c		\
 	src/core/or/circuituse.c		\
 	src/core/or/command.c			\
@@ -215,6 +216,7 @@ noinst_HEADERS +=					\
 	src/core/or/circuitmux.h			\
 	src/core/or/circuitmux_ewma.h			\
 	src/core/or/circuitstats.h			\
+	src/core/or/circuitpadding.h			\
 	src/core/or/circuituse.h			\
 	src/core/or/command.h				\
 	src/core/or/connection_edge.h			\

src/core/or/circuit_st.h

@@ -12,6 +12,11 @@
 #include "core/or/cell_queue_st.h"
 
 struct hs_token_t;
+typedef struct circpad_machine_t circpad_machine_t;
+typedef struct circpad_machineinfo_t circpad_machineinfo_t;
+
+/** Number of padding state machines on a circuit. */
+#define CIRCPAD_MAX_MACHINES (2)
 
 /** "magic" value for an origin_circuit_t */
 #define ORIGIN_CIRCUIT_MAGIC 0x35315243u
@@ -177,6 +182,19 @@ struct circuit_t {
   /** Hashtable node: used to look up the circuit by its HS token using the HS
       circuitmap. */
   HT_ENTRY(circuit_t) hs_circuitmap_node;
+
+  /** Adaptive Padding state machines: these are immutable. The state machines
+   *  that come from the consensus are saved to a global structure, to avoid
+   *  per-circuit allocations. This merely points to the global copy. */
+  const circpad_machine_t *padding_machine[CIRCPAD_MAX_MACHINES];
+
+  /** Adaptive Padding machine info for above machines. This is the
+   *  per-circuit mutable information, such as the current state and
+   *  histogram token counts. Some of it is optional (aka NULL).
+   *  If a machine is being shut down, these indexes can be NULL
+   *  without the corresponding padding_machine being NULL, while we
+   *  wait for the other end to respond to our shutdown request. */
+  circpad_machineinfo_t *padding_info[CIRCPAD_MAX_MACHINES];
 };
 
 #endif

src/core/or/circuitbuild.c

@@ -42,6 +42,7 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/command.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
@@ -927,12 +928,15 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
   crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath);
   circuit_build_times_handle_completed_hop(circ);
 
+  circpad_machine_event_circ_added_hop(circ);
+
   if (hop) {
     /* Case two: we're on a hop after the first. */
     return circuit_send_intermediate_onion_skin(circ, hop);
   }
 
   /* Case three: the circuit is finished. Do housekeeping tasks on it. */
+  circpad_machine_event_circ_built(circ);
   return circuit_build_no_more_hops(circ);
 }
 
@@ -2580,7 +2584,24 @@ choose_good_middle_server(uint8_t purpose,
     return choice;
   }
 
-  choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
+  if (options->MiddleNodes) {
+    smartlist_t *sl = smartlist_new();
+    routerset_get_all_nodes(sl, options->MiddleNodes,
+                            options->ExcludeNodes, 1);
+
+    smartlist_subtract(sl, excluded);
+
+    choice = node_sl_choose_by_bandwidth(sl, WEIGHT_FOR_MID);
+    smartlist_free(sl);
+    if (choice) {
+      log_fn(LOG_INFO, LD_CIRC, "Chose fixed middle node: %s",
+          hex_str(choice->identity, DIGEST_LEN));
+    } else {
+      log_fn(LOG_NOTICE, LD_CIRC, "Restricted middle not available");
+    }
+  } else {
+    choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
+  }
   smartlist_free(excluded);
   return choice;
 }

src/core/or/circuitlist.c

@@ -61,6 +61,7 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "core/or/circuitstats.h"
+#include "core/or/circuitpadding.h"
 #include "core/mainloop/connection.h"
 #include "app/config/config.h"
 #include "core/or/connection_edge.h"
@@ -1176,6 +1177,9 @@ circuit_free_(circuit_t *circ)
            CIRCUIT_IS_ORIGIN(circ) ?
               TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
 
+  /* Free any circuit padding structures */
+  circpad_circuit_machineinfo_free(circ);
+
   if (should_free) {
     memwipe(mem, 0xAA, memlen); /* poison memory */
     tor_free(mem);