diff --git a/src/github.com/getlantern/flashlight/server/server.go b/src/github.com/getlantern/flashlight/server/server.go index c1166811f..a5bc96d2d 100644 --- a/src/github.com/getlantern/flashlight/server/server.go +++ b/src/github.com/getlantern/flashlight/server/server.go @@ -7,6 +7,7 @@ import ( "os" "reflect" "strconv" + "strings" "sync" "time" @@ -242,7 +243,7 @@ func determineInternalIP() (string, error) { } func onBytesGiven(destAddr string, req *http.Request, bytes int64) { - _, port, _ := net.SplitHostPort(destAddr) + host, port, _ := net.SplitHostPort(destAddr) if port == "" { port = "0" } @@ -258,5 +259,15 @@ func onBytesGiven(destAddr string, req *http.Request, bytes int64) { givenTo := statreporter.Country(clientCountry) givenTo.Increment("bytesGivenTo").Add(bytes) givenTo.Increment("bytesGivenToByFlashlight").Add(bytes) + givenTo.Member("distinctDestHosts", host) + + clientIp := req.Header.Get("X-Forwarded-For") + if clientIp != "" { + // clientIp may contain multiple ips, use the first + ips := strings.Split(clientIp, ",") + clientIp := strings.TrimSpace(ips[0]) + givenTo.Member("distinctClients", clientIp) + } + } } diff --git a/src/github.com/getlantern/flashlight/statreporter/model.go b/src/github.com/getlantern/flashlight/statreporter/model.go index 02886ff6b..43bd265f7 100644 --- a/src/github.com/getlantern/flashlight/statreporter/model.go +++ b/src/github.com/getlantern/flashlight/statreporter/model.go @@ -10,14 +10,10 @@ import ( const ( increments = "increments" gauges = "gauges" + members = "multiMembers" ) -const ( - set = iota - add -) - -// DimGroup represents a group of dimensions for +// DimGroup represents a group of dimensions for categorizing stats. type DimGroup struct { dims map[string]string } @@ -33,11 +29,14 @@ type UpdateBuilder struct { type update struct { dg *DimGroup category string - action int key string - val int64 + action interface{} // one of set, add or member } +type set int64 +type add int64 +type member string + // Dim constructs a DimGroup starting with a single dimension. func Dim(key string, value string) *DimGroup { return &DimGroup{map[string]string{key: value}} @@ -104,13 +103,21 @@ func (dg *DimGroup) Gauge(key string) *UpdateBuilder { } } +func (dg *DimGroup) Member(key string, val string) { + postUpdate(&update{ + dg, + members, + key, + member(val), + }) +} + func (b *UpdateBuilder) Add(val int64) { postUpdate(&update{ b.dg, b.category, - add, b.key, - val, + add(val), }) } @@ -118,8 +125,7 @@ func (b *UpdateBuilder) Set(val int64) { postUpdate(&update{ b.dg, b.category, - set, b.key, - val, + set(val), }) } diff --git a/src/github.com/getlantern/flashlight/statreporter/statreporter.go b/src/github.com/getlantern/flashlight/statreporter/statreporter.go index a05f21e44..12d195c2a 100644 --- a/src/github.com/getlantern/flashlight/statreporter/statreporter.go +++ b/src/github.com/getlantern/flashlight/statreporter/statreporter.go @@ -47,7 +47,7 @@ type dimGroupAccumulator struct { categories map[string]stats } -type stats map[string]int64 +type stats map[string]interface{} // either int64 or string type report map[string]interface{} @@ -147,11 +147,23 @@ func (r *reporter) run() { categoryStats = make(stats) dgAccum.categories[update.category] = categoryStats } - switch update.action { + switch a := update.action.(type) { case set: - categoryStats[update.key] = update.val + categoryStats[update.key] = int64(a) case add: - categoryStats[update.key] = categoryStats[update.key] + update.val + existing, found := categoryStats[update.key] + if !found { + categoryStats[update.key] = int64(a) + } else { + categoryStats[update.key] = existing.(int64) + int64(a) + } + case member: + existing, found := categoryStats[update.key] + if !found { + categoryStats[update.key] = map[string]bool{string(a): true} + } else { + existing.(map[string]bool)[string(a)] = true + } } case <-timer.C: r.post() @@ -196,8 +208,22 @@ func (dgAccum *dimGroupAccumulator) makeReport() report { "dims": dgAccum.dg.dims, } - for category, accum := range dgAccum.categories { - report[category] = accum + for category, s := range dgAccum.categories { + if category == members { + // Transform maps into arrays + s2 := make(stats) + + for k, v := range s { + m := v.(map[string]bool) + a := make([]string, 0, len(m)) + for member := range m { + a = append(a, member) + } + s2[k] = a + } + s = s2 + } + report[category] = s } return report @@ -216,11 +242,13 @@ func posterForDimGroupStats(cfg *Config) reportPoster { return fmt.Errorf("Unable to post stats to statshub: %s", err) } defer resp.Body.Close() + + jsonString := string(jsonBytes) if resp.StatusCode != 200 { - return fmt.Errorf("Unexpected response status posting stats to statshub: %d", resp.StatusCode) + return fmt.Errorf("Unexpected response status posting stats %s to statshub: %d", jsonString, resp.StatusCode) } - log.Debugf("Reported %s to statshub", string(jsonBytes)) + log.Debugf("Reported %s to statshub", jsonString) return nil } } diff --git a/src/github.com/getlantern/flashlight/statreporter/statreporter_test.go b/src/github.com/getlantern/flashlight/statreporter/statreporter_test.go index a8cb8488a..920909e88 100644 --- a/src/github.com/getlantern/flashlight/statreporter/statreporter_test.go +++ b/src/github.com/getlantern/flashlight/statreporter/statreporter_test.go @@ -42,6 +42,7 @@ func TestAll(t *testing.T) { dg1.Gauge("gaugea").Add(2) dg1.Gauge("gaugeb").Set(2) dg1.Gauge("gaugeb").Set(48) + dg1.Member("membera", "I") originalReporter := currentReporter @@ -66,6 +67,8 @@ func TestAll(t *testing.T) { dg2.Gauge("gaugea").Add(2) dg2.Gauge("gaugeb").Set(2) dg2.Gauge("gaugeb").Set(48) + dg2.Member("membera", "II") + dg2.Member("membera", "II") updatedReporter := currentReporter @@ -78,12 +81,15 @@ func TestAll(t *testing.T) { "country": "us", }, "increments": stats{ - "incra": 2, - "incrb": 25, + "incra": int64(2), + "incrb": int64(25), }, "gauges": stats{ - "gaugea": 4, - "gaugeb": 48, + "gaugea": int64(4), + "gaugeb": int64(48), + }, + "multiMembers": stats{ + "membera": []string{"I"}, }, } expectedReport2 := report{ @@ -93,12 +99,15 @@ func TestAll(t *testing.T) { "country": "cn", }, "increments": stats{ - "incra": 2, - "incrb": 25, + "incra": int64(2), + "incrb": int64(25), }, "gauges": stats{ - "gaugea": 4, - "gaugeb": 48, + "gaugea": int64(4), + "gaugeb": int64(48), + }, + "multiMembers": stats{ + "membera": []string{"II"}, }, } @@ -125,4 +134,5 @@ func compareReports(t *testing.T, expected report, actual report, index string) assert.Equal(t, expected["increments"], actual["increments"], fmt.Sprintf("On %s, increments should match", index)) assert.Equal(t, expected["gauges"], actual["gauges"], fmt.Sprintf("On %s, gauges should match", index)) + assert.Equal(t, expected["multiMembers"], actual["multiMembers"], fmt.Sprintf("On %s, members should match", index)) }