HTTP/2 Support, GoDocs, Response Logging, fixes (#12)

- Added godocs, and examples
- Added HTTP/2 support
- Fixed reverse proxy 404 errors
- Added proxy.log as a log handler for server error logs
- Added reponse logging
- Request, and response logging use TeeReader to allow streaming support
- Added supervisorctl support to selenium container

cmd/gomitmproxy/gomitmproxy.go

@@ -38,6 +38,7 @@ func main() {
 	DNSRegex := flag.String("dns_regex", p.DNSRegex, "domains matching this regex pattern will return the proxy address")
 	config := flag.String("config", "", "proxy config file path")
 	requestLogFile := flag.String("request_log_file", "", "file to log http requests")
+	logResponses := flag.Bool("log_responses", p.LogResponses, "enable logging upstream server responses")
 	logJSON := flag.Bool("json", false, "output json log format to standard out")
 	logDebug := flag.Bool("debug", false, "enable debug logging")
 	logLevel := flag.String("log_level", log.DefaultLogger.Level.String(), "set logging to log level")
@@ -105,6 +106,8 @@ func main() {
 			p.DNSServer = *DNSServer
 		case "dns_regex":
 			p.DNSRegex = *DNSRegex
+		case "log_responses":
+			p.LogResponses = *logResponses
 		case "log_level":
 			logConfig.Level.Parse(*logLevel)
 		}
@@ -138,6 +141,7 @@ func main() {
 	log.WithField("dns_port", p.DNSPort).Debug("")
 	log.WithField("dns_server", p.DNSServer).Debug("")
 	log.WithField("dns_regex", p.DNSRegex).Debug("")
+	log.WithField("log_responses", p.LogResponses).Debug("")
 
 	// Start the proxy
 	if err = p.Run(); err != nil {
@@ -150,12 +154,12 @@ func GenerateCA(KeyAge int, CACertFile, CAKeyFile string) {
 	c := proxy.Certs{
 		KeyAge: time.Duration(KeyAge) * time.Hour,
 	}
-	key, cert, err := c.GenerateCAPair()
+	_, _, err := c.GenerateCAPair()
 	if err != nil {
 		log.Fatal(err.Error())
 	}
 
-	err = proxy.WriteCA(CACertFile, CAKeyFile, cert, key)
+	err = c.WriteCA(CACertFile, CAKeyFile)
 	if err != nil {
 		log.Fatal(err.Error())
 	}

go.mod

@@ -1,3 +1,6 @@
 module github.com/jmizell/GoMITMProxy
 
-require github.com/benburkert/dns v0.0.0-20190225204957-d356cf78cdfc
+require (
+	github.com/benburkert/dns v0.0.0-20190225204957-d356cf78cdfc
+	golang.org/x/net v0.0.0-20190311183353-d8887717615a
+)

go.sum

@@ -1,2 +1,8 @@
 github.com/benburkert/dns v0.0.0-20190225204957-d356cf78cdfc h1:eyDlmf21vuKN61WoxV2cQLDH/PBDyyjIhUI4kT2o1yM=
 github.com/benburkert/dns v0.0.0-20190225204957-d356cf78cdfc/go.mod h1:6ul4nJKqsreAIBK5lUkibcUn2YBU6CvDzlKDH+dtZsQ=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

proxy/certs.go

@@ -19,23 +19,51 @@ import (
 	"github.com/jmizell/GoMITMProxy/proxy/log"
 )
 
+// Certificate Organization used when generating the certificate authority
 const CertOrg = "GoMITMProxy"
+
+// Key length for certificate authority, and host keys
 const KeyLength = 1024
+
+// Default max age a certificate authority will be valid for. Automatically generated host keys are set to expire
+// no later then the CA's max age.
 const DefaultKeyAge = time.Hour * 24
 
+// Error no certificate authority set in Certs
 const ERRCertNoCA = ErrorStr("no certificate authority set")
+
+// Error unable to read ca file
 const ERRCertCARead = ErrorStr("read ca file failed")
+
+// Error unable to parse ca
 const ERRCertCAParse = ErrorStr("parse ca failed")
+
+// Error certificate authority has expired
 const ERRCertCAExpired = ErrorStr("ca expired")
+
+// Error generating certificate authority
 const ERRCertGenCA = ErrorStr("generate ca failed")
+
+// Error failed to generate host key
 const ERRCertGenHostKey = ErrorStr("generate host key failed")
+
+// Error writing certificate authority files to disk
 const ERRCertWriteCA = ErrorStr("writing ca to disk failed")
+
+// Error generating key
 const ERRCertGenerateKey = ErrorStr("generate key failed")
+
+// Error creating x509 certificate
 const ERRCertx509Create = ErrorStr("create x509 cert failed")
+
+// Error parsing x509 certificate
 const ERRCertx509Parse = ErrorStr("parse x509 cert failed")
 
+// Exit code returned when an unrecoverable certificate error occurs
 const EXITCODECertFatal = 131
 
+// Certs generates and stores certificates for MITMProxy. Virtual host certificates are generated
+// on demand, and cached.
 type Certs struct {
 	certStore map[string]*tls.Certificate
 	caKey     *rsa.PrivateKey
@@ -44,6 +72,8 @@ type Certs struct {
 	lock      sync.Mutex
 }
 
+// Get attempts to retrieve a cert in the cache for the given virtual host, or generates
+// a new one if not present in the cache.
 func (c *Certs) Get(vhost string) (*tls.Certificate, error) {
 
 	if vhost == "" {
@@ -70,6 +100,7 @@ func (c *Certs) Get(vhost string) (*tls.Certificate, error) {
 	return key, nil
 }
 
+// LoadCAPair reads the certificate authority cert, and key from pem encoded files on disk.
 func (c *Certs) LoadCAPair(keyFile, certFile string) error {
 
 	keyBytes, err := ioutil.ReadFile(keyFile)
@@ -101,6 +132,8 @@ func (c *Certs) LoadCAPair(keyFile, certFile string) error {
 	return nil
 }
 
+// GenerateCAPair generates a certificate authority key and cert pair. It both stores,
+// and returns the generated pair.
 func (c *Certs) GenerateCAPair() (key *rsa.PrivateKey, cert *x509.Certificate, err error) {
 
 	if c.KeyAge == 0 {
@@ -125,9 +158,13 @@ func (c *Certs) GenerateCAPair() (key *rsa.PrivateKey, cert *x509.Certificate, e
 		return nil, nil, ERRCertGenCA.Err().WithError(err)
 	}
 
+	c.caCert = cert
+	c.caKey = key
+
 	return key, cert, nil
 }
 
+// GenerateHostKey returns a tls.Certificate for a given virtual host, signed by the CA.
 func (c *Certs) GenerateHostKey(vhost string) (*tls.Certificate, error) {
 
 	if c.caKey == nil || c.caCert == nil {
@@ -164,6 +201,32 @@ func (c *Certs) GenerateHostKey(vhost string) (*tls.Certificate, error) {
 	return &tlsCert, nil
 }
 
+// WriteCA writes the certificate authority key and cert as pem encoded files to disk.
+func (c *Certs) WriteCA(certFileName, keyFileName string) error {
+
+	if certFileName == "" || keyFileName == "" {
+		startTime := time.Now().Unix()
+		certFileName = fmt.Sprintf("gomitmproxy_ca_%d.crt", startTime)
+		keyFileName = fmt.Sprintf("gomitmproxy_ca_%d.key", startTime)
+	}
+
+	keyBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(c.caKey)})
+	err := ioutil.WriteFile(keyFileName, keyBytes, 0600)
+	if err != nil {
+		return ERRCertWriteCA.Err().WithError(err)
+	}
+	log.WithField("key_file", keyFileName).Info("wrote certificate authority key")
+
+	certBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.caCert.Raw})
+	err = ioutil.WriteFile(certFileName, certBytes, 0600)
+	if err != nil {
+		return ERRCertWriteCA.Err().WithError(err)
+	}
+	log.WithField("cert_file", certFileName).Info("wrote certificate authority certificate")
+
+	return nil
+}
+
 func genCerts(certTemplate *x509.Certificate, signingCert *x509.Certificate, signingKey *rsa.PrivateKey, KeyAge time.Duration) (
 	key *rsa.PrivateKey, cert *x509.Certificate, err error) {
 
@@ -203,28 +266,3 @@ func genSerial() *big.Int {
 
 	return serialNumber
 }
-
-func WriteCA(certFileName, keyFileName string, cert *x509.Certificate, key *rsa.PrivateKey) error {
-
-	if certFileName == "" || keyFileName == "" {
-		startTime := time.Now().Unix()
-		certFileName = fmt.Sprintf("gomitmproxy_ca_%d.crt", startTime)
-		keyFileName = fmt.Sprintf("gomitmproxy_ca_%d.key", startTime)
-	}
-
-	keyBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
-	err := ioutil.WriteFile(keyFileName, keyBytes, 0600)
-	if err != nil {
-		return ERRCertWriteCA.Err().WithError(err)
-	}
-	log.WithField("key_file", keyFileName).Info("wrote certificate authority key")
-
-	certBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
-	err = ioutil.WriteFile(certFileName, certBytes, 0600)
-	if err != nil {
-		return ERRCertWriteCA.Err().WithError(err)
-	}
-	log.WithField("cert_file", certFileName).Info("wrote certificate authority certificate")
-
-	return nil
-}

proxy/certs_test.go

@@ -228,8 +228,12 @@ func testingCAPair() (certFilename string, keyFilename string, err error) {
 	keyFilename = key.Name()
 
 	certs := &Certs{}
-	caKey, caCert, err := certs.GenerateCAPair()
-	err = WriteCA(certFilename, keyFilename, caCert, caKey)
+	_, _, err = certs.GenerateCAPair()
+	if err != nil {
+		return "", "", err
+	}
+
+	err = certs.WriteCA(certFilename, keyFilename)
 	if err != nil {
 		return "", "", err
 	}