Add more visibility to IP address resolution when using DNS

fhttp/http_client.go

@@ -46,6 +46,8 @@ type Fetcher interface {
 	// Close() cleans up connections and state - must be paired with NewClient calls.
 	// returns how many sockets have been used (Fastclient only)
 	Close() int
+	// GetIPAddress() get the ip address that DNS resolves to
+	GetIPAddress() string
 }
 
 const (
@@ -308,6 +310,7 @@ type Client struct {
 	bodyContainsUUID     bool // if body contains the "{uuid}" pattern (lowercase)
 	logErrors            bool
 	id                   int
+	socketCount          *int
 }
 
 // Close cleans up any resources used by NewStdClient.
@@ -324,7 +327,7 @@ func (c *Client) Close() int {
 	if c.transport != nil {
 		c.transport.CloseIdleConnections()
 	}
-	return 0 // TODO: find a way to track std client socket usage.
+	return *c.socketCount
 }
 
 // ChangeURL only for standard client, allows fetching a different URL.
@@ -396,6 +399,11 @@ func (c *Client) Fetch() (int, []byte, int) {
 	return code, data, 0
 }
 
+// GetIPAddress get the ip address that DNS resolves to when using stdClient.
+func (c *Client) GetIPAddress() string {
+	return c.req.RemoteAddr
+}
+
 // NewClient creates either a standard or fast client (depending on
 // the DisableFastClient flag).
 func NewClient(o *HTTPOptions) (Fetcher, error) {
@@ -415,6 +423,8 @@ func NewStdClient(o *HTTPOptions) (*Client, error) {
 	if req == nil {
 		return nil, err
 	}
+
+	var socketCount int
 	tr := http.Transport{
 		MaxIdleConns:        o.NumConnections,
 		MaxIdleConnsPerHost: o.NumConnections,
@@ -426,9 +436,14 @@ func NewStdClient(o *HTTPOptions) (*Client, error) {
 			if o.Resolve != "" {
 				addr = o.Resolve + addr[strings.LastIndex(addr, ":"):]
 			}
-			return (&net.Dialer{
+			conn, err := (&net.Dialer{
 				Timeout: o.HTTPReqTimeOut,
 			}).DialContext(ctx, network, addr)
+
+			req.RemoteAddr = conn.RemoteAddr().String()
+			socketCount++
+
+			return conn, err
 		},
 		TLSHandshakeTimeout: o.HTTPReqTimeOut,
 	}
@@ -451,9 +466,10 @@ func NewStdClient(o *HTTPOptions) (*Client, error) {
 			Timeout:   o.HTTPReqTimeOut,
 			Transport: &tr,
 		},
-		transport: &tr,
-		id:        o.ID,
-		logErrors: o.LogErrors,
+		transport:   &tr,
+		id:          o.ID,
+		logErrors:   o.LogErrors,
+		socketCount: &socketCount,
 	}
 	if !o.FollowRedirects {
 		// Lets us see the raw response instead of auto following redirects.
@@ -511,6 +527,11 @@ type FastClient struct {
 	tlsConfig    *tls.Config
 }
 
+// GetIPAddress get the ip address that DNS resolves to when using fast client.
+func (c *FastClient) GetIPAddress() string {
+	return c.dest.String()
+}
+
 // Close cleans up any resources used by FastClient.
 func (c *FastClient) Close() int {
 	log.Debugf("Closing %p %s socket count %d", c, c.url, c.socketCount)

fhttp/http_utils.go

@@ -627,3 +627,11 @@ func (r *SyncReader) Read(p []byte) (n int, err error) {
 
 	return r.reader.Read(p)
 }
+
+func getIPUsageCount(ipCountMap map[string]int) (count int) {
+	for _, v := range ipCountMap {
+		count += v
+	}
+
+	return count
+}

fhttp/httprunner.go

@@ -35,8 +35,9 @@ import (
 // Also is the internal type used per thread/goroutine.
 type HTTPRunnerResults struct {
 	periodic.RunnerResults
-	client   Fetcher
-	RetCodes map[int]int64
+	client     Fetcher
+	RetCodes   map[int]int64
+	IPCountMap map[string]int // TODO: Move it to a shared results struct where all runner should have this field
 	// internal type/data
 	sizes       *stats.Histogram
 	headerSizes *stats.Histogram
@@ -50,6 +51,12 @@ type HTTPRunnerResults struct {
 	aborter *periodic.Aborter
 }
 
+// IPCountPair stores the ip address and its corresponding usage count.
+type IPCountPair struct {
+	ip    string
+	count int
+}
+
 // Run tests http request fetching. Main call being run at the target QPS.
 // To be set as the Function in RunnerOptions.
 func (httpstate *HTTPRunnerResults) Run(t int) (bool, string) {
@@ -98,6 +105,7 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) {
 	total := HTTPRunnerResults{
 		HTTPOptions: o.HTTPOptions,
 		RetCodes:    make(map[int]int64),
+		IPCountMap:  make(map[string]int),
 		sizes:       stats.NewHistogram(0, 100),
 		headerSizes: stats.NewHistogram(0, 5),
 		AbortOn:     o.AbortOn,
@@ -182,6 +190,11 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) {
 	// unused ones. We also must cleanup all the created clients.
 	keys := []int{}
 	for i := 0; i < numThreads; i++ {
+		// Get the report on the IP address each thread use to send traffic
+		ip := httpstate[i].client.GetIPAddress()
+		log.Infof("[%d] %s resolve to IP address: %s\n", i, o.URL, ip)
+		total.IPCountMap[ip]++
+
 		total.SocketCount += httpstate[i].client.Close()
 		// Q: is there some copying each time stats[i] is used?
 		for k := range httpstate[i].RetCodes {
@@ -193,14 +206,27 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) {
 		total.sizes.Transfer(httpstate[i].sizes)
 		total.headerSizes.Transfer(httpstate[i].headerSizes)
 	}
+
+	// Sort the ip address form largest to smallest based on its usage count
+	ipCountList := make([]IPCountPair, 0, len(total.IPCountMap))
+	for k, v := range total.IPCountMap {
+		ipCountList = append(ipCountList, IPCountPair{k, v})
+	}
+
+	sort.Slice(ipCountList, func(i, j int) bool {
+		return ipCountList[i].count > ipCountList[j].count
+	})
+
 	// Cleanup state:
 	r.Options().ReleaseRunners()
 	sort.Ints(keys)
 	totalCount := float64(total.DurationHistogram.Count)
-	if !o.DisableFastClient {
-		_, _ = fmt.Fprintf(out, "Sockets used: %d (for perfect keepalive, would be %d)\n", total.SocketCount, r.Options().NumThreads)
-	}
+	_, _ = fmt.Fprintf(out, "Sockets used: %d (for perfect keepalive, would be %d)\n", total.SocketCount, r.Options().NumThreads)
 	_, _ = fmt.Fprintf(out, "Uniform: %t, Jitter: %t\n", total.Uniform, total.Jitter)
+	_, _ = fmt.Fprintf(out, "IP addresses distribution:\n")
+	for _, v := range ipCountList {
+		_, _ = fmt.Fprintf(out, "%s: %d\n", v.ip, v.count)
+	}
 	for _, k := range keys {
 		_, _ = fmt.Fprintf(out, "Code %3d : %d (%.1f %%)\n", k, total.RetCodes[k], 100.*float64(total.RetCodes[k])/totalCount)
 	}
@@ -216,7 +242,7 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) {
 	return &total, nil
 }
 
-// A errgroup is a collection of goroutines working on subtasks that are part of
+// An errgroup is a collection of goroutines working on subtasks that are part of
 // the same overall task.
 type errgroup struct {
 	wg sync.WaitGroup

fhttp/httprunner_test.go

@@ -60,6 +60,10 @@ func TestHTTPRunner(t *testing.T) {
 	if res.SocketCount != res.RunnerResults.NumThreads {
 		t.Errorf("%d socket used, expected same as thread# %d", res.SocketCount, res.RunnerResults.NumThreads)
 	}
+	count := getIPUsageCount(res.IPCountMap)
+	if count != res.RunnerResults.NumThreads {
+		t.Errorf("Total IP usage count %d, expected same as thread %d", count, res.RunnerResults.NumThreads)
+	}
 	// Test raw client, should get warning about non init timeout:
 	rawOpts := HTTPOptions{
 		URL: opts.URL,
@@ -121,11 +125,8 @@ func testHTTPNotLeaking(t *testing.T, opts *HTTPRunnerOptions) {
 	if ngAfter > ngBefore2+8 {
 		t.Errorf("Goroutines after test %d, expected it to stay near %d", ngAfter, ngBefore2)
 	}
-	if !opts.DisableFastClient {
-		// only fast client so far has a socket count
-		if res.SocketCount != res.RunnerResults.NumThreads {
-			t.Errorf("%d socket used, expected same as thread# %d", res.SocketCount, res.RunnerResults.NumThreads)
-		}
+	if res.SocketCount != res.RunnerResults.NumThreads {
+		t.Errorf("%d socket used, expected same as thread# %d", res.SocketCount, res.RunnerResults.NumThreads)
 	}
 }
 
@@ -190,17 +191,16 @@ func TestHTTPRunnerClientRace(t *testing.T) {
 	}
 }
 
-func TestClosingAndSocketCount(t *testing.T) {
+func testClosingAndSocketCount(t *testing.T, o *HTTPRunnerOptions) {
 	mux, addr := DynamicHTTPServer(false)
 	mux.HandleFunc("/echo42/", EchoHandler)
 	URL := fmt.Sprintf("http://localhost:%d/echo42/?close=true", addr.Port)
-	opts := HTTPRunnerOptions{}
-	opts.Init(URL)
-	opts.QPS = 10
+	o.Init(URL)
+	o.QPS = 10
 	numReq := int64(50) // can't do too many without running out of fds on mac
-	opts.Exactly = numReq
-	opts.NumThreads = 5
-	res, err := RunHTTPTest(&opts)
+	o.Exactly = numReq
+	o.NumThreads = 5
+	res, err := RunHTTPTest(o)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -217,6 +217,14 @@ func TestClosingAndSocketCount(t *testing.T) {
 	}
 }
 
+func TestClosingAndSocketCountFastClient(t *testing.T) {
+	testClosingAndSocketCount(t, &HTTPRunnerOptions{})
+}
+
+func TestClosingAndSocketCountStdClient(t *testing.T) {
+	testClosingAndSocketCount(t, &HTTPRunnerOptions{HTTPOptions: HTTPOptions{DisableFastClient: true}})
+}
+
 func TestHTTPRunnerBadServer(t *testing.T) {
 	// Using http to an https server (or the current 'close all' dummy https server)
 	// should fail:

stats/stats_test.go

@@ -726,7 +726,6 @@ func TestBucketLookUp(t *testing.T) {
 		input float64 // input
 		start float64 // start
 		end   float64 // end
-
 	}{
 		{input: 11, start: 10, end: 11},
 		{input: 171, start: 160, end: 180},

ui/restHandler.go

@@ -236,7 +236,8 @@ func RESTRunHandler(w http.ResponseWriter, r *http.Request) { // nolint: funlen
 
 // Run executes the run (can be called async or not, writer is nil for async mode).
 func Run(w http.ResponseWriter, r *http.Request, jd map[string]interface{},
-	runner, url string, ro periodic.RunnerOptions, httpopts *fhttp.HTTPOptions) {
+	runner, url string, ro periodic.RunnerOptions, httpopts *fhttp.HTTPOptions,
+) {
 	//	go func() {
 	var res periodic.HasRunnerResult
 	var err error