Merge branch 'tcpproxying' into devel

src/github.com/getlantern/flashlight/server/server.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -242,7 +243,7 @@ func determineInternalIP() (string, error) {
 }
 
 func onBytesGiven(destAddr string, req *http.Request, bytes int64) {
-	_, port, _ := net.SplitHostPort(destAddr)
+	host, port, _ := net.SplitHostPort(destAddr)
 	if port == "" {
 		port = "0"
 	}
@@ -258,5 +259,15 @@ func onBytesGiven(destAddr string, req *http.Request, bytes int64) {
 		givenTo := statreporter.Country(clientCountry)
 		givenTo.Increment("bytesGivenTo").Add(bytes)
 		givenTo.Increment("bytesGivenToByFlashlight").Add(bytes)
+		givenTo.Member("distinctDestHosts", host)
+
+		clientIp := req.Header.Get("X-Forwarded-For")
+		if clientIp != "" {
+			// clientIp may contain multiple ips, use the first
+			ips := strings.Split(clientIp, ",")
+			clientIp := strings.TrimSpace(ips[0])
+			givenTo.Member("distinctClients", clientIp)
+		}
+
 	}
 }

src/github.com/getlantern/flashlight/statreporter/model.go

@@ -10,14 +10,10 @@ import (
 const (
 	increments = "increments"
 	gauges     = "gauges"
+	members    = "multiMembers"
 )
 
-const (
-	set = iota
-	add
-)
-
-// DimGroup represents a group of dimensions for
+// DimGroup represents a group of dimensions for categorizing stats.
 type DimGroup struct {
 	dims map[string]string
 }
@@ -33,11 +29,14 @@ type UpdateBuilder struct {
 type update struct {
 	dg       *DimGroup
 	category string
-	action   int
 	key      string
-	val      int64
+	action   interface{} // one of set, add or member
 }
 
+type set int64
+type add int64
+type member string
+
 // Dim constructs a DimGroup starting with a single dimension.
 func Dim(key string, value string) *DimGroup {
 	return &DimGroup{map[string]string{key: value}}
@@ -104,22 +103,29 @@ func (dg *DimGroup) Gauge(key string) *UpdateBuilder {
 	}
 }
 
+func (dg *DimGroup) Member(key string, val string) {
+	postUpdate(&update{
+		dg,
+		members,
+		key,
+		member(val),
+	})
+}
+
 func (b *UpdateBuilder) Add(val int64) {
 	postUpdate(&update{
 		b.dg,
 		b.category,
-		add,
 		b.key,
-		val,
+		add(val),
 	})
 }
 
 func (b *UpdateBuilder) Set(val int64) {
 	postUpdate(&update{
 		b.dg,
 		b.category,
-		set,
 		b.key,
-		val,
+		set(val),
 	})
 }

src/github.com/getlantern/flashlight/statreporter/statreporter.go

@@ -47,7 +47,7 @@ type dimGroupAccumulator struct {
 	categories map[string]stats
 }
 
-type stats map[string]int64
+type stats map[string]interface{} // either int64 or string
 
 type report map[string]interface{}
 
@@ -147,11 +147,23 @@ func (r *reporter) run() {
 					categoryStats = make(stats)
 					dgAccum.categories[update.category] = categoryStats
 				}
-				switch update.action {
+				switch a := update.action.(type) {
 				case set:
-					categoryStats[update.key] = update.val
+					categoryStats[update.key] = int64(a)
 				case add:
-					categoryStats[update.key] = categoryStats[update.key] + update.val
+					existing, found := categoryStats[update.key]
+					if !found {
+						categoryStats[update.key] = int64(a)
+					} else {
+						categoryStats[update.key] = existing.(int64) + int64(a)
+					}
+				case member:
+					existing, found := categoryStats[update.key]
+					if !found {
+						categoryStats[update.key] = map[string]bool{string(a): true}
+					} else {
+						existing.(map[string]bool)[string(a)] = true
+					}
 				}
 			case <-timer.C:
 				r.post()
@@ -196,8 +208,22 @@ func (dgAccum *dimGroupAccumulator) makeReport() report {
 		"dims": dgAccum.dg.dims,
 	}
 
-	for category, accum := range dgAccum.categories {
-		report[category] = accum
+	for category, s := range dgAccum.categories {
+		if category == members {
+			// Transform maps into arrays
+			s2 := make(stats)
+
+			for k, v := range s {
+				m := v.(map[string]bool)
+				a := make([]string, 0, len(m))
+				for member := range m {
+					a = append(a, member)
+				}
+				s2[k] = a
+			}
+			s = s2
+		}
+		report[category] = s
 	}
 
 	return report
@@ -216,11 +242,13 @@ func posterForDimGroupStats(cfg *Config) reportPoster {
 			return fmt.Errorf("Unable to post stats to statshub: %s", err)
 		}
 		defer resp.Body.Close()
+
+		jsonString := string(jsonBytes)
 		if resp.StatusCode != 200 {
-			return fmt.Errorf("Unexpected response status posting stats to statshub: %d", resp.StatusCode)
+			return fmt.Errorf("Unexpected response status posting stats %s to statshub: %d", jsonString, resp.StatusCode)
 		}
 
-		log.Debugf("Reported %s to statshub", string(jsonBytes))
+		log.Debugf("Reported %s to statshub", jsonString)
 		return nil
 	}
 }

src/github.com/getlantern/flashlight/statreporter/statreporter_test.go

@@ -42,6 +42,7 @@ func TestAll(t *testing.T) {
 	dg1.Gauge("gaugea").Add(2)
 	dg1.Gauge("gaugeb").Set(2)
 	dg1.Gauge("gaugeb").Set(48)
+	dg1.Member("membera", "I")
 
 	originalReporter := currentReporter
 
@@ -66,6 +67,8 @@ func TestAll(t *testing.T) {
 	dg2.Gauge("gaugea").Add(2)
 	dg2.Gauge("gaugeb").Set(2)
 	dg2.Gauge("gaugeb").Set(48)
+	dg2.Member("membera", "II")
+	dg2.Member("membera", "II")
 
 	updatedReporter := currentReporter
 
@@ -78,12 +81,15 @@ func TestAll(t *testing.T) {
 			"country": "us",
 		},
 		"increments": stats{
-			"incra": 2,
-			"incrb": 25,
+			"incra": int64(2),
+			"incrb": int64(25),
 		},
 		"gauges": stats{
-			"gaugea": 4,
-			"gaugeb": 48,
+			"gaugea": int64(4),
+			"gaugeb": int64(48),
+		},
+		"multiMembers": stats{
+			"membera": []string{"I"},
 		},
 	}
 	expectedReport2 := report{
@@ -93,12 +99,15 @@ func TestAll(t *testing.T) {
 			"country": "cn",
 		},
 		"increments": stats{
-			"incra": 2,
-			"incrb": 25,
+			"incra": int64(2),
+			"incrb": int64(25),
 		},
 		"gauges": stats{
-			"gaugea": 4,
-			"gaugeb": 48,
+			"gaugea": int64(4),
+			"gaugeb": int64(48),
+		},
+		"multiMembers": stats{
+			"membera": []string{"II"},
 		},
 	}
 
@@ -125,4 +134,5 @@ func compareReports(t *testing.T, expected report, actual report, index string)
 
 	assert.Equal(t, expected["increments"], actual["increments"], fmt.Sprintf("On %s, increments should match", index))
 	assert.Equal(t, expected["gauges"], actual["gauges"], fmt.Sprintf("On %s, gauges should match", index))
+	assert.Equal(t, expected["multiMembers"], actual["multiMembers"], fmt.Sprintf("On %s, members should match", index))
 }