add ability to set http basic auth for requests, update to go 1.15 an…

…d update dependencies

.travis.yml

@@ -8,7 +8,7 @@ env:
     - secure: RFRhKtSrbXyxoy3TzP9ZIwyCv2S+rNapsZyK0IgyQHuoRrUuXUN/Q5VMMja2PSoG7Ko8HXRnSOgWESIbV1lktt/6kaLHzKUiqFB5QWGnzoKi2elXDZNkMJXSj9ouqs9lC2zp1hovLJxzSdj0Igpr34YuQ3iEXodkjwHNodqzPQNMSvTy/L1Y1ZCYKc47UTDuuLAaz4lzUAgsDsbQwHiCAh+Lb/GhTrnAJaT/mVpCxvJlSy9BdJarooTJ9PekvBxuodLavqHUw/MUYfFi3JUlNottIIEVs1Q9aute6XrGxphgfQn7mEZlbWfszbzhSZkBcQN4dA3rFYj01nSW5UP5WhmytOoprcgbaxQasfA4I+V/fgJfJGKHbOlqoGw5Vklt9xu5UolSAyF0UGxdgytqccDZsmlbmL3xf6xkkZLe4g0Aa+QfGyU540Ev8vS3gnABcb1mSvtOuryu3bwYWpMn9OxdpzhXP556k1AayG2uRrXaA9IiGC9u5X0sdIHEIHZ43oIfF5eRzb6E1DAMfCsGUDWWnI2NdP2QEBC2rlDA6fWhT8+ghGVrWrRuGTGDe+iunHtkkUOwlxUWLEWFjg6XKryKsdCI2A8m5pP9rVWvp8X7uR3J1SBO0RLywf8ZdPGEBNPpj19tiuNE5GCuK9mryKlM7PiVkZgoBAalk0y5pss=
     - secure: SJRSUHAaOjbzMGgzVut1FDZPXkr4ni4ey2KA9TsSKhndwIHQr9bsM4kuPrUHsYEH3ONzoFmlzODJDHUXWa9c2mGfzXqW65Vu0Skug9dSXen6KcM+SglVHoQrc8P/MDmMsrQ3O1+Y8Kfh9ntBibSwanDKEW25KTWgCOGYz7vc7B5LLwRPCMputZLN1A8uuFQ2Gir79CFNBEDdleaw67Fu2K0u4ZXMlu8r70VX5g8X6+mrpp5cY6/puWhxOgEmvQRptr6FUaeTCbGxTTlurSjGIGe8QtZazwmb//p1ewIQNdk4WoWSg9UjNCmQZbBjPKizwbAUeVYksosoICIaB8KE3Aizes8F/EQdjZn7G4VZ0YbizhjUUYlRDhAILJywYv3fVrBUDXqFcc7BgJNQFM0/o7aJ0jnA1QgTscGZP+6Vtn8VDX2fKWwDcHRDPfiREjo2HDtSdQYG7DroeBy8oxn9uSSwc6RpqRCA3mS7ii2wGlIKZAK0NDnt+WC31CuvR/iWqrcmBIxISuI/xIP5tNoWF5OU6flVCaWxpi0QcINqsBO7fd05uIwN065X4zEmewAJLjbB2xn/sR5YNO4unh5vN8OyIRtkD8aJNdYAl+Whkd11+uNy11g0DABTDBNlhjESW8D338msdcSsReEXvKF/c25DXNJoGgTU8uKIeTNWSW0=
 go:
-  - '1.14'
+  - '1.15'
 before_script:
   - go get -u github.com/mattn/goveralls
   - go get -u github.com/golangci/golangci-lint/cmd/golangci-lint

go.mod

@@ -1,11 +1,10 @@
 module github.com/atomicptr/crab
 
-go 1.14
+go 1.15
 
 require (
 	github.com/beevik/etree v1.1.0
 	github.com/pkg/errors v0.9.1
-	github.com/spf13/cobra v0.0.7
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/testify v1.2.2
+	github.com/spf13/cobra v1.1.1
+	github.com/stretchr/testify v1.7.0
 )

pkg/cli/crawl/commons.go

@@ -33,6 +33,20 @@ func registerStandardCrawlCommandFlags(cmd *cobra.Command, flagOptions *crawlerF
 		defaultHttpTimeout,
 		"set http timeout for requests",
 	)
+	cmd.PersistentFlags().StringVarP(
+		&flagOptions.AuthUsername,
+		"auth-user",
+		"",
+		"",
+		"set HTTP basic authentication username",
+	)
+	cmd.PersistentFlags().StringVarP(
+		&flagOptions.AuthPassword,
+		"auth-pass",
+		"",
+		"",
+		"set HTTP basic authentication password",
+	)
 	cmd.PersistentFlags().StringSliceVarP(
 		&flagOptions.CookieStrings,
 		"cookie",
@@ -63,6 +77,10 @@ func registerStandardCrawlCommandFlagModifiers(modifier *crawler.RequestModifier
 		modifier.With(addPrefixUrlToRequest(flagOptions.PrefixUrl))
 	}
 
+	if len(flagOptions.AuthUsername) > 0 || len(flagOptions.AuthPassword) > 0 {
+		modifier.With(addHttpBasicAuthToRequest(flagOptions))
+	}
+
 	if len(flagOptions.CookieStrings) > 0 {
 		modifier.With(addCookiesToRequest(flagOptions))
 	}

pkg/cli/crawl/flags.go

@@ -11,6 +11,8 @@ type crawlerFlagOptions struct {
 	NumberOfWorkers   int
 	PrefixUrl         string
 	HttpTimeout       time.Duration
+	AuthUsername      string
+	AuthPassword      string
 	CookieStrings     []string
 	HeaderStrings     []string
 	FilterStatusQuery string

pkg/cli/crawl/request_modifiers.go

@@ -46,3 +46,9 @@ func addHeadersToRequest(flagOptions crawlerFlagOptions) crawler.RequestModifier
 		}
 	}
 }
+
+func addHttpBasicAuthToRequest(flagOptions crawlerFlagOptions) crawler.RequestModifierFunc {
+	return func(req *http.Request) {
+		req.SetBasicAuth(flagOptions.AuthUsername, flagOptions.AuthPassword)
+	}
+}

pkg/cli/crawl/request_modifiers_test.go

@@ -102,3 +102,20 @@ func TestAddHeadersToRequest(t *testing.T) {
 	assert.Equal(t, "c", req.Header.Get("b"))
 	assert.Equal(t, "asdf", req.Header.Get("test"))
 }
+
+func TestAddHttpBasicAuthToRequest(t *testing.T) {
+	modifier := crawler.RequestModifier{}
+	modifier.With(addHttpBasicAuthToRequest(crawlerFlagOptions{
+		AuthUsername: "username",
+		AuthPassword: "password",
+	}))
+
+	req := httptest.NewRequest("GET", "/", strings.NewReader(""))
+	modifier.Do(req)
+
+	username, password, ok := req.BasicAuth()
+
+	assert.True(t, ok)
+	assert.Equal(t, "username", username)
+	assert.Equal(t, "password", password)
+}