grabber.go

/*
 * ZGrab Copyright 2015 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package zlib

import (
	"bufio"
	"bytes"
	"crypto/sha256"
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"net"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/zmap/zcrypto/tls"
	"github.com/zmap/zgrab/ztools/blacklist"
	"github.com/zmap/zgrab/ztools/ftp"
	"github.com/zmap/zgrab/ztools/http"
	"github.com/zmap/zgrab/ztools/processing"
	"github.com/zmap/zgrab/ztools/scada/dnp3"
	"github.com/zmap/zgrab/ztools/scada/fox"
	"github.com/zmap/zgrab/ztools/scada/siemens"
	"github.com/zmap/zgrab/ztools/smb"
	"github.com/zmap/zgrab/ztools/telnet"
	"github.com/zmap/zgrab/ztools/xssh"
	"github.com/zmap/zgrab/ztools/zlog"

	"github.com/zmap/zgrab/tls13measurements"
)

var ErrRedirLocalhost = errors.New("Redirecting to Localhost")

type GrabTarget struct {
	Addr         net.IP
	Domain       string
	ComsysSource string
	ComsysDate   string
	ComsysInput  string
}

type grabTargetDecoder struct {
	reader *csv.Reader
}

func (gtd *grabTargetDecoder) DecodeNext() (interface{}, error) {
	// gtd.reader.LazyQuotes = true
	record, err := gtd.reader.Read()
	if err != nil {
		return nil, err
	}
	if len(record) < 1 {
		return nil, errors.New("Invalid grab target (no fields)")
	}
	var target GrabTarget
	target.Addr = net.ParseIP(record[0])
	if target.Addr == nil {
		return nil, fmt.Errorf("Invalid IP address %s", record[0])
	}
	// Check for a domain
	if len(record) >= 2 {
		target.Domain = record[1]
	}
	// Check for comsys rabbit-mq information
	if len(record) >= 4 {
		target.ComsysSource = record[2]
		target.ComsysDate = record[3]
	}
	// Check if the full dns json info from comsys rabbit-mq was added
	if len(record) >= 5 {
		target.ComsysInput = record[4]
	}
	return target, nil
}

type grabDomainDecoder struct {
	reader *bufio.Reader
}

func (gdd *grabDomainDecoder) DecodeNext() (interface{}, error) {
again:
	record, err := gdd.reader.ReadBytes('\n')
	if err != nil {
		return nil, err
	}

	var target GrabTarget
	if record == nil {
		return nil, errors.New("No domains were found")
	}

	dom := string(record[:len(record)-1])
	dom = strings.TrimSpace(strings.SplitN(dom, "#", 2)[0])
	if len(dom) == 0 {
		goto again
	}

	target.Domain = dom
	return target, nil
}

func NewGrabTargetDecoder(reader io.Reader, domainOnly bool) processing.Decoder {

	if domainOnly {
		domainReader := bufio.NewReader(reader)
		d := grabDomainDecoder{
			reader: domainReader,
		}
		return &d
	} else {
		csvReader := csv.NewReader(reader)
		d := grabTargetDecoder{
			reader: csvReader,
		}
		return &d
	}
}

func makeDialer(c *Config) func(string) (*Conn, error) {
	proto := "tcp"
	if c.BACNet {
		proto = "udp"
	}
	timeout := c.Timeout
	return func(addr string) (*Conn, error) {
		deadline := time.Now().Add(timeout)
		d := Dialer{
			Deadline: deadline,
			Proxy:    c.Proxy,
		}
		conn, err := d.Dial(proto, addr)
		conn.maxTlsVersion = c.TLSVersion
		if err == nil {
			conn.SetDeadline(deadline)
		}
		return conn, err
	}
}

func makeNetDialer(c *Config) func(string, string) (net.Conn, error) {
	proto := "tcp"
	timeout := c.Timeout
	return func(net, addr string) (net.Conn, error) {
		deadline := time.Now().Add(timeout)
		d := Dialer{
			Deadline: deadline,
			Proxy:    c.Proxy,
		}
		conn, err := d.Dial(proto, addr)
		conn.maxTlsVersion = c.TLSVersion
		if err == nil {
			conn.SetDeadline(deadline)
		}
		return conn.getUnderlyingConn(), err
	}
}

func makeTLSConfig(config *Config, urlHost string, sni string) *tls.Config {
	tlsConfig := new(tls.Config)
	tlsConfig.InsecureSkipVerify = true
	tlsConfig.MinVersion = tls.VersionSSL30
	tlsConfig.MaxVersion = config.TLSVersion
	tlsConfig.RootCAs = config.RootCAPool
	tlsConfig.HeartbeatEnabled = true
	tlsConfig.ClientDSAEnabled = true
	if config.KeylogFile != nil {
		//tlsConfig.KeyLogWriter = config.KeylogFile
	}

	if config.DHEOnly {
		tlsConfig.CipherSuites = tls.DHECiphers
	}
	if config.ECDHEOnly {
		tlsConfig.CipherSuites = tls.ECDHECiphers
	}
	if config.ExportsOnly {
		tlsConfig.CipherSuites = tls.RSA512ExportCiphers
	}
	if config.ExportsDHOnly {
		tlsConfig.CipherSuites = tls.DHEExportCiphers
	}
	if config.ChromeOnly {
		tlsConfig.CipherSuites = tls.ChromeCiphers
	}
	if config.ChromeNoDHE {
		tlsConfig.CipherSuites = tls.ChromeNoDHECiphers
	}
	if config.FirefoxOnly {
		tlsConfig.CipherSuites = tls.FirefoxCiphers
	}
	if config.FirefoxNoDHE {
		tlsConfig.CipherSuites = tls.FirefoxNoDHECiphers
	}

	if config.SafariOnly {
		tlsConfig.CipherSuites = tls.SafariCiphers
		tlsConfig.ForceSuites = true
	}
	if config.SafariNoDHE {
		tlsConfig.CipherSuites = tls.SafariNoDHECiphers
		tlsConfig.ForceSuites = true
	}
	if config.SignedCertificateTimestampExt {
		tlsConfig.SignedCertificateTimestampExt = true
	}
	if config.GatherSessionTicket {
		tlsConfig.ForceSessionTicketExt = true
	}
	if !config.NoSNI && urlHost != "" {
		if sni != "" {
			tlsConfig.ServerName = sni
		} else {
			tlsConfig.ServerName = urlHost
		}
	}
	if config.ExternalClientHello != nil {
		tlsConfig.ExternalClientHello = config.ExternalClientHello
	}

	return tlsConfig
}

func usingDefaultPort(scheme string, port uint16) bool {
	return (scheme == "https" && port == 443) || (scheme == "http" && port == 80)
}

// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
// return true if the string includes a port, does not validate port
func containsPort(host string) bool {
	return strings.LastIndex(host, ":") > strings.LastIndex(host, "]")
}

func redirectsToLocalhost(host string) bool {
	if i := net.ParseIP(host); i != nil {
		return i.IsLoopback() || i.Equal(net.IPv4zero)
	} else {
		if host == "localhost" {
			return true
		} else {
			if addrs, err := net.LookupHost(host); err == nil {
				for _, i := range addrs {
					if ip := net.ParseIP(i); ip != nil {
						if ip.IsLoopback() || ip.Equal(net.IPv4zero) {
							return true
						}
					}
				}
			}
		}
	}
	return false
}

func makeHTTPGrabber(config *Config, grabData *GrabData, target *GrabTarget) func(string, string, string) error {
	g := func(urlHost, endpoint, httpHost string) (err error) {

		var tlsConfig *tls.Config
		if config.TLS {
			tlsConfig = makeTLSConfig(config, urlHost, target.Domain)
		}

		var testCtx *tls13measurements.TestContext
		if config.TLS13Measurements {
			testCtx, err = tls13measurements.SetupConfig(tlsConfig, target.Addr, target.Domain)
			if err != nil {
				return err
			}
		}

		transport := &http.Transport{
			Proxy:               nil, // TODO: implement proxying
			Dial:                makeNetDialer(config),
			DisableKeepAlives:   false,
			DisableCompression:  false,
			MaxIdleConnsPerHost: config.HTTP.MaxRedirects,
			TLSClientConfig:     tlsConfig,
		}

		client := http.MakeNewClient()
		client.UserAgent = config.HTTP.UserAgent
		client.CheckRedirect = func(req *http.Request, res *http.Response, via []*http.Request) error {
			if !config.HTTP.FollowLocalhostRedirects && redirectsToLocalhost(req.URL.Hostname()) {
				return ErrRedirLocalhost
			}
			grabData.HTTP.RedirectResponseChain = append(grabData.HTTP.RedirectResponseChain, res)
			b := new(bytes.Buffer)
			maxReadLen := int64(config.HTTP.MaxSize) * 1024
			readLen := maxReadLen
			if res.ContentLength >= 0 && res.ContentLength < maxReadLen {
				readLen = res.ContentLength
			}
			io.CopyN(b, res.Body, readLen)
			res.BodyText = b.String()
			if len(res.BodyText) > 0 {
				m := sha256.New()
				m.Write(b.Bytes())
				res.BodySHA256 = m.Sum(nil)
			}

			if len(via) > config.HTTP.MaxRedirects {
				return errors.New(fmt.Sprintf("stopped after %d redirects", config.HTTP.MaxRedirects))
			}

			if req.URL.Scheme == "https" && transport.TLSClientConfig == nil {
				// TODO check if sni is set correctly
				transport.TLSClientConfig = makeTLSConfig(config, req.URL.Host, req.URL.Host)
			}

			return nil
		}
		client.Jar = nil // Don't send or receive cookies (otherwise use CookieJar)
		client.Transport = transport

		var fullURL string

		if config.TLS {
			fullURL = "https://" + urlHost + endpoint
		} else {
			fullURL = "http://" + urlHost + endpoint
		}

		var resp *http.Response

		u, err := url.Parse(fullURL)
		if err != nil {
			return err
		}

		if httpHost == "" {
			httpHost = u.Host
		}

		// Remove host port if using default port
		if containsPort(httpHost) && usingDefaultPort(u.Scheme, config.Port) {
			hostWithoutPort, _, err := net.SplitHostPort(httpHost)
			if err != nil {
				return err
			}
			httpHost = hostWithoutPort
		}

		switch config.HTTP.Method {
		case "GET":
			resp, err = client.GetWithHost(fullURL, httpHost)
		case "HEAD":
			resp, err = client.HeadWithHost(fullURL, httpHost)
		default:
			zlog.Fatalf("Bad HTTP Method: %s. Valid options are: GET, HEAD.", config.HTTP.Method)
		}
		if resp != nil && resp.Body != nil {
			defer resp.Body.Close()
		}
		grabData.HTTP.Response = resp

		if err != nil {
			if urlError, ok := err.(*url.Error); ok {
				if urlError.Err == ErrRedirLocalhost {
					err = nil
				}
			}
		}

		if err != nil {
			/*
				if tls.IsTLS13notImplementedAbortError(err) {
					// TODO We intentionally aborted the TLS 1.3 handshake as it is not supported yet
					return err
				}*/
			config.ErrorLog.Errorf("Could not connect to remote host %s: %s", fullURL, err.Error())
			if config.TLS13Measurements && (resp != nil) && (resp.Request != nil) {
				testCtx.FinishTest(resp.Request.TLSHandshake, err)
			}
			return err
		}

		b := new(bytes.Buffer)
		maxReadLen := int64(config.HTTP.MaxSize) * 1024
		readLen := maxReadLen
		if resp.ContentLength >= 0 && resp.ContentLength < maxReadLen {
			readLen = resp.ContentLength
		}
		io.CopyN(b, resp.Body, readLen)
		grabData.HTTP.Response.BodyText = b.String()
		if len(grabData.HTTP.Response.BodyText) > 0 {
			m := sha256.New()
			m.Write(b.Bytes())
			grabData.HTTP.Response.BodySHA256 = m.Sum(nil)
		}

		if config.TLS13Measurements {
			testCtx.FinishTest(resp.Request.TLSHandshake, nil)
		}

		return nil
	}

	return g
}

func makeGrabber(config *Config) func(*Conn) error {
	// Do all the hard work here
	g := func(c *Conn) error {
		banner := make([]byte, 1024)
		response := make([]byte, 65536)
		c.SetCAPool(config.RootCAPool)
		if config.DHEOnly {
			c.CipherSuites = tls.DHECiphers
		}
		if config.ECDHEOnly {
			c.CipherSuites = tls.ECDHECiphers
		}
		if config.ExportsOnly {
			c.CipherSuites = tls.RSA512ExportCiphers
		}
		if config.ExportsDHOnly {
			c.CipherSuites = tls.DHEExportCiphers
		}
		if config.ChromeOnly {
			c.CipherSuites = tls.ChromeCiphers
		}
		if config.ChromeNoDHE {
			c.CipherSuites = tls.ChromeNoDHECiphers
		}
		if config.FirefoxOnly {
			c.CipherSuites = tls.FirefoxCiphers
		}
		if config.FirefoxNoDHE {
			c.CipherSuites = tls.FirefoxNoDHECiphers
		}
		if config.SafariOnly {
			c.CipherSuites = tls.SafariCiphers
			c.ForceSuites = true
		}
		if config.SafariNoDHE {
			c.CipherSuites = tls.SafariNoDHECiphers
			c.ForceSuites = true
		}
		if config.NoSNI {
			c.SetNoSNI()
		}
		if config.TLSExtendedRandom {
			c.SetExtendedRandom()
		}
		if config.GatherSessionTicket {
			c.SetGatherSessionTicket()
		}
		if config.SignedCertificateTimestampExt {
			c.SetSignedCertificateTimestampExt()
		}
		if config.ExtendedMasterSecret {
			c.SetOfferExtendedMasterSecret()
		}
		if config.ExternalClientHello != nil {
			c.SetExternalClientHello(config.ExternalClientHello)
		}
		if config.TLSVerbose {
			c.SetTLSVerbose()
		}

		if config.TLS {
			if err := c.TLSHandshake(); err != nil {
				c.erroredComponent = "tls"
				return err
			}
		}
		if config.Banners {
			if config.SMTP {
				if _, err := c.SMTPBanner(banner); err != nil {
					c.erroredComponent = "banner"
					return err
				}
			} else if config.POP3 {
				if _, err := c.POP3Banner(banner); err != nil {
					c.erroredComponent = "banner"
					return err
				}
			} else if config.IMAP {
				if _, err := c.IMAPBanner(banner); err != nil {
					c.erroredComponent = "banner"
					return err
				}
			} else {
				if _, err := c.BasicBanner(); err != nil {
					c.erroredComponent = "banner"
					return err
				}
			}
		}

		if config.FTP {
			c.grabData.FTP = new(ftp.FTPLog)

			is200Banner, err := ftp.GetFTPBanner(c.grabData.FTP, c.getUnderlyingConn())
			if err != nil {
				c.erroredComponent = "ftp"
				return err
			}

			if config.FTPAuthTLS && is200Banner {
				if err := c.GetFTPSCertificates(); err != nil {
					c.erroredComponent = "ftp-authtls"
					return err
				}
			}
		}

		if config.Fox {
			c.grabData.Fox = new(fox.FoxLog)

			if err := fox.GetFoxBanner(c.grabData.Fox, c.getUnderlyingConn()); err != nil {
				c.erroredComponent = "fox"
				return err
			}
		}

		if config.Telnet {
			c.grabData.Telnet = new(telnet.TelnetLog)

			if err := telnet.GetTelnetBanner(c.grabData.Telnet, c.getUnderlyingConn(), config.TelnetMaxSize); err != nil {
				c.erroredComponent = "telnet"
				return err
			}
		}

		if config.S7 {
			c.grabData.S7 = new(siemens.S7Log)

			if err := siemens.GetS7Banner(c.grabData.S7, c.getUnderlyingConn()); err != nil {
				c.erroredComponent = "s7"
				return err
			}
		}

		if config.DNP3 {
			c.grabData.DNP3 = new(dnp3.DNP3Log)
			dnp3.GetDNP3Banner(c.grabData.DNP3, c.getUnderlyingConn())
		}

		if config.SMB.SMB {
			c.grabData.SMB = new(smb.SMBLog)

			if err := smb.GetSMBBanner(c.grabData.SMB, c.getUnderlyingConn()); err != nil {
				c.erroredComponent = "smb"
				return err
			}
		}

		if config.SendData {
			host, _, _ := net.SplitHostPort(c.RemoteAddr().String())
			msg := bytes.Replace(config.Data, []byte("%s"), []byte(host), -1)
			msg = bytes.Replace(msg, []byte("%d"), []byte(c.domain), -1)
			if _, err := c.Write(msg); err != nil {
				c.erroredComponent = "write"
				return err
			}
			if _, err := c.Read(response); err != nil {
				c.erroredComponent = "read"
				return err
			}
		}

		if config.EHLO {
			if err := c.EHLO(config.EHLODomain); err != nil {
				c.erroredComponent = "ehlo"
				return err
			}
		}
		if config.SMTPHelp {
			if err := c.SMTPHelp(); err != nil {
				c.erroredComponent = "smtp_help"
				return err
			}
		}
		if config.StartTLS {
			if config.IMAP {
				if err := c.IMAPStartTLSHandshake(); err != nil {
					c.erroredComponent = "starttls"
					return err
				}
			} else if config.POP3 {
				if err := c.POP3StartTLSHandshake(); err != nil {
					c.erroredComponent = "starttls"
					return err
				}
			} else {
				if err := c.SMTPStartTLSHandshake(); err != nil {
					c.erroredComponent = "starttls"
					return err
				}
			}
		}

		if config.SMTP {
			if err := c.SMTPQuit(); err != nil {
				c.erroredComponent = "quit"
				return err
			}
		} else if config.POP3 {
			if err := c.POP3Quit(); err != nil {
				c.erroredComponent = "quit"
				return err
			}
		} else if config.IMAP {
			if err := c.IMAPQuit(); err != nil {
				c.erroredComponent = "quit"
				return err
			}
		}

		if config.Modbus {
			if _, err := c.SendModbusEcho(); err != nil {
				c.erroredComponent = "modbus"
				return err
			}
		}

		if config.BACNet {
			if err := c.BACNetVendorQuery(); err != nil {
				c.erroredComponent = "bacnet"
				return err
			}
		}

		if config.Heartbleed {
			buf := make([]byte, 256)
			if _, err := c.CheckHeartbleed(buf); err != nil {
				c.erroredComponent = "heartbleed"
				return err
			}
		}
		return nil
	}
	// Wrap the whole thing in a logger
	return func(c *Conn) error {
		err := g(c)
		if err != nil {
			config.ErrorLog.Errorf("Conversation error with remote host %s: %s",
				c.RemoteAddr().String(), err.Error())
		}

		c.Close()
		return err
	}
}

func makeXSSHGrabber(gblConfig *Config, grabData GrabData) func(string) error {
	return func(netAddr string) error {

		xsshConfig := xssh.MakeXSSHConfig()
		xsshConfig.Timeout = gblConfig.Timeout
		xsshConfig.ConnLog = grabData.XSSH
		_, err := xssh.Dial("tcp", netAddr, xsshConfig)
		if err != nil {
			return err
		}

		return nil
	}
}

func GrabBanner(config *Config, target *GrabTarget) *Grab {
	if blacklist.IsBlacklistedDom(target.Domain) || blacklist.IsBlacklisted(target.Addr) {
		return &Grab{
			IP:             target.Addr,
			Domain:         target.Domain,
			Time:           time.Now(),
			Error:          blacklist.BlacklistError,
			ErrorComponent: "connect",
			ComsysSource:   target.ComsysSource,
			ComsysDate:     target.ComsysDate,
			ComsysInput:    target.ComsysInput,
		}
	}

	if config.XSSH.XSSH {
		t := time.Now()

		grabData := GrabData{XSSH: new(xssh.HandshakeLog)}
		xsshGrabber := makeXSSHGrabber(config, grabData)

		port := strconv.FormatUint(uint64(config.Port), 10)
		rhost := net.JoinHostPort(target.Addr.String(), port)

		err := xsshGrabber(rhost)

		return &Grab{
			IP:    target.Addr,
			Time:  t,
			Data:  grabData,
			Error: err,
		}
	} else if len(config.HTTP.Endpoint) == 0 {
		dial := makeDialer(config)
		grabber := makeGrabber(config)
		port := strconv.FormatUint(uint64(config.Port), 10)
		addr := target.Addr.String()
		rhost := net.JoinHostPort(addr, port)
		t := time.Now()
		conn, dialErr := dial(rhost)
		if target.Domain != "" {
			conn.SetDomain(target.Domain)
		}
		if dialErr != nil {
			// Could not connect to host
			config.ErrorLog.Errorf("Could not connect to %s remote host %s: %s",
				target.Domain, addr, dialErr.Error())
			return &Grab{
				IP:             target.Addr,
				Domain:         target.Domain,
				Time:           t,
				Error:          dialErr,
				ErrorComponent: "connect",
				ComsysSource:   target.ComsysSource,
				ComsysDate:     target.ComsysDate,
				ComsysInput:    target.ComsysInput,
			}
		}
		err := grabber(conn)
		return &Grab{
			IP:             target.Addr,
			Domain:         target.Domain,
			Time:           t,
			Data:           conn.grabData,
			Error:          err,
			ErrorComponent: conn.erroredComponent,
			ComsysSource:   target.ComsysSource,
			ComsysDate:     target.ComsysDate,
			ComsysInput:    target.ComsysInput,
		}
	} else {
		grabData := GrabData{HTTP: new(HTTP)}
		httpGrabber := makeHTTPGrabber(config, &grabData, target)
		port := strconv.FormatUint(uint64(config.Port), 10)
		t := time.Now()
		var rhost string
		if config.LookupDomain {
			rhost = target.Domain
		} else {
			rhost = net.JoinHostPort(target.Addr.String(), port)
		}

		err := httpGrabber(rhost, config.HTTP.Endpoint, target.Domain)

		return &Grab{
			IP:           target.Addr,
			Domain:       target.Domain,
			Time:         t,
			Data:         grabData,
			Error:        err,
			ComsysSource: target.ComsysSource,
			ComsysDate:   target.ComsysDate,
			ComsysInput:  target.ComsysInput,
		}
	}
}