Round robin dns, option to resolve ipv4/ipv6/both, timeout and other fixes

README.md

@@ -279,6 +279,9 @@ properly). Can be in the form of host:port, ip:port, port or "disabled" to
 disable the feature. (default "8081")
   -resolve IP
         Resolve host name to this IP
+  -resolve-ip-type type
+        Resolve type: ip4 for ipv4, ip6 for ipv6 only, use ip for both (default
+ip4)
   -runid int
         Optional RunID to add to json result and auto save filename, to match
 server mode

fhttp/http_client.go

@@ -676,14 +676,15 @@ func (c *FastClient) connect() net.Conn {
 	c.socketCount++
 	var socket net.Conn
 	var err error
+	d := &net.Dialer{Timeout: c.reqTimeout}
 	if c.https {
-		socket, err = tls.Dial(c.dest.Network(), c.dest.String(), c.tlsConfig)
+		socket, err = tls.DialWithDialer(d, c.dest.Network(), c.dest.String(), c.tlsConfig)
 		if err != nil {
 			log.Errf("[%d] Unable to TLS connect to %v : %v", c.id, c.dest, err)
 			return nil
 		}
 	} else {
-		socket, err = net.Dial(c.dest.Network(), c.dest.String())
+		socket, err = d.Dial(c.dest.Network(), c.dest.String())
 		if err != nil {
 			log.Errf("[%d] Unable to connect to %v : %v", c.id, c.dest, err)
 			return nil

fhttp/httprunner.go

@@ -93,7 +93,7 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) {
 	log.Infof("Starting http test for %s with %d threads at %.1f qps and %s warmup", o.URL, o.NumThreads, o.QPS, warmupMode)
 	r := periodic.NewPeriodicRunner(&o.RunnerOptions)
 	defer r.Options().Abort()
-	numThreads := r.Options().NumThreads
+	numThreads := r.Options().NumThreads // can change during run for c > 2 n
 	o.HTTPOptions.Init(o.URL)
 	out := r.Options().Out // Important as the default value is set from nil to stdout inside NewPeriodicRunner
 	total := HTTPRunnerResults{
@@ -180,8 +180,9 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) {
 		fm.Close()
 		_, _ = fmt.Fprintf(out, "Wrote profile data to %s.{cpu|mem}\n", o.Profiler)
 	}
-	// Numthreads may have reduced but it should be ok to accumulate 0s from
-	// unused ones. We also must cleanup all the created clients.
+	// Numthreads may have reduced:
+	numThreads = total.RunnerResults.NumThreads
+	// But we also must cleanup all the created clients.
 	keys := []int{}
 	for i := 0; i < numThreads; i++ {
 		// Get the report on the IP address each thread use to send traffic
@@ -211,7 +212,7 @@ func RunHTTPTest(o *HTTPRunnerOptions) (*HTTPRunnerResults, error) {
 		return total.IPCountMap[ipList[i]] > total.IPCountMap[ipList[j]]
 	})
 
-	// Cleanup state:
+	// Cleanup state: (original num thread)
 	r.Options().ReleaseRunners()
 	sort.Ints(keys)
 	totalCount := float64(total.DurationHistogram.Count)

fnet/network.go

@@ -15,7 +15,9 @@
 package fnet // import "fortio.org/fortio/fnet"
 
 import (
+	"context"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -25,8 +27,10 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
+	"fortio.org/fortio/dflag"
 	"fortio.org/fortio/log"
 	"fortio.org/fortio/version"
 )
@@ -59,6 +63,12 @@ var (
 	MaxPayloadSize = 256 * KILOBYTE
 	// Payload that is returned during echo call.
 	Payload []byte
+	// Atomically incremented counter for dns resolution.
+	dnsRoundRobin uint32 = 0xffffffff // we want the first one, after increment to be 0
+	// IP types to resolve. With round robin you are likely to get ipv6 which may not work
+	// (in particular some test environments like the CI do have ipv6 for localhost but fail to connect).
+	FlagResolveIPType = dflag.DynString(flag.CommandLine, "resolve-ip-type", "ip4",
+		"Resolve `type`: ip4 for ipv4, ip6 for ipv6 only, use ip for both")
 )
 
 // nolint: gochecknoinits // needed here (unit change)
@@ -286,6 +296,9 @@ func Resolve(host string, port string) (*net.TCPAddr, error) {
 
 // ResolveByProto returns the address of the host,port suitable for net.Dial.
 // nil in case of errors. works for both "tcp" and "udp" proto.
+// Limit which address type is returned using `resolve-ip` ip4/ip6/ip (for both, default).
+// If the same host is requested, and it has more than 1 IP, returned value will roundrobin
+// over the ips.
 func ResolveByProto(host string, port string, proto string) (*HostPortAddr, error) {
 	log.Debugf("Resolve() called with host=%s port=%s proto=%s", host, port, proto)
 	dest := &HostPortAddr{}
@@ -294,29 +307,33 @@ func ResolveByProto(host string, port string, proto string) (*HostPortAddr, erro
 		host = host[1 : len(host)-1]
 	}
 	isAddr := net.ParseIP(host)
+	filter := FlagResolveIPType.Get()
 	var err error
 	if isAddr != nil {
 		log.Debugf("Host already an IP, will go to %s", isAddr)
 		dest.IP = isAddr
 	} else {
 		var addrs []net.IP
-		addrs, err = net.LookupIP(host)
+		addrs, err = net.DefaultResolver.LookupIP(context.Background(), filter, host)
 		if err != nil {
 			log.Errf("Unable to lookup '%s' : %v", host, err)
 			return nil, err
 		}
-		if len(addrs) > 1 && log.LogDebug() {
-			log.Debugf("Using only the first of the addresses for %s : %v", host, addrs)
+		idx := uint32(0)
+		l := uint32(len(addrs))
+		if l > 1 {
+			idx = (atomic.AddUint32(&dnsRoundRobin, 1) % l)
+			log.Debugf("Using address #%d for %s : %v", idx, host, addrs)
 		}
-		log.Debugf("%s will go to %s", proto, addrs[0])
-		dest.IP = addrs[0]
+		log.Debugf("%s will go to %s", proto, addrs[idx])
+		dest.IP = addrs[idx]
 	}
 	dest.Port, err = net.LookupPort(proto, port)
 	if err != nil {
 		log.Errf("Unable to resolve port '%s' : %v", port, err)
 		return nil, err
 	}
-	log.LogVf("Resolved %s:%s to %s addr %+v", host, port, proto, dest)
+	log.LogVf("Resolved %s:%s to %s %s addr %+v", host, port, proto, filter, dest)
 	return dest, nil
 }