add --response-workers flag to allow it to vary from request workers

config/config.go

@@ -8,6 +8,7 @@ type Config struct {
 	NoColor               bool
 	BaseDirectory         string
 	RequestWorkers        int
+	ResponseWorkers       int
 	SubdirLength          int
 	RequestMethod         string
 	ConnectTimeoutSeconds int

execcontext/execcontext.go

@@ -17,6 +17,7 @@ type Context struct {
 	BaseDirectory          string
 	SubdirLength           int
 	RequestWorkers         int
+	ResponseWorkers        int
 	ConnectTimeoutDuration time.Duration
 	Retries                int
 	Logger                 *logger.LeveledLogger
@@ -35,11 +36,22 @@ func New(conf *config.Config) (*Context, error) {
 		BaseDirectory:          conf.BaseDirectory,
 		SubdirLength:           conf.SubdirLength,
 		RequestWorkers:         conf.RequestWorkers,
+		ResponseWorkers:        conf.ResponseWorkers,
 		RequestHeaders:         conf.RequestHeaders,
 		Out:                    log.New(os.Stdout, "", 0),
 		Logger:                 createLeveledLogger(conf),
 	}
 
+	if context.RequestWorkers <= 0 {
+		context.RequestWorkers = 30
+	}
+
+	if context.ResponseWorkers <= 0 {
+		context.ResponseWorkers = context.RequestWorkers
+	}
+
+	context.Logger.Info("request workers %d, response workers %d", context.RequestWorkers, context.ResponseWorkers)
+
 	context.UrlScanner, err = createUrlScanner(conf.UrlFilename, context.Logger)
 
 	if len(conf.BaseDirectory) > 0 {

main.go

@@ -50,6 +50,11 @@ func createApp() *cli.App {
 			Value:       conf.RequestWorkers,
 			Destination: &conf.RequestWorkers,
 		},
+		cli.IntFlag{
+			Name:        "response-workers",
+			Usage:       "number of concurrent workers that will be processing responses, if not specified will be same as --workers",
+			Destination: &conf.ResponseWorkers,
+		},
 		cli.IntFlag{
 			Name:        "subdir-length, S",
 			Usage:       "length of hashed subdirectory name to put saved files when using -o; use 2 for > 5k urls, 4 for > 5M urls",

main_test.go

@@ -214,10 +214,11 @@ func TestAddHeadersToRequestCreatesCanonicalKeys(t *testing.T) {
 
 func newTestContext(scaffold *Scaffold, expectedURLPaths []string) *execcontext.Context {
 	return &execcontext.Context{
-		RequestWorkers: 1,
-		UrlScanner:     urlsScanner(expectedURLPaths),
-		Out:            scaffold.StandardOutMock,
-		Logger:         logger.NewPlainLeveledLogger(scaffold.LoggerMock),
+		RequestWorkers:  1,
+		ResponseWorkers: 1,
+		UrlScanner:      urlsScanner(expectedURLPaths),
+		Out:             scaffold.StandardOutMock,
+		Logger:          logger.NewPlainLeveledLogger(scaffold.LoggerMock),
 	}
 }
 

responses/responses.go

@@ -14,9 +14,9 @@ import (
 
 func StartResponseWorkers(responses <-chan *http.Response, context *execcontext.Context) *sync.WaitGroup {
 	var responseWaitGroup sync.WaitGroup
-	responseWaitGroup.Add(context.RequestWorkers)
+	responseWaitGroup.Add(context.ResponseWorkers)
 
-	for i := 1; i <= context.RequestWorkers; i++ {
+	for i := 1; i <= context.ResponseWorkers; i++ {
 		go func() {
 			if context.WriteFiles {
 				responseSavingWorker(responses, context)