From e583191968e6ffe4d9f38df3cf77e00ec4e63368 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Sat, 12 Nov 2016 12:04:47 +0100 Subject: [PATCH] Add framework for testing protocol scanners This adds a mock SOCKS5 proxy to be able to test protocol scanners. It creates a single-threaded SOCKS5 proxy on localhost which processes connection requests and dispatches them to a test-specific interface, which can do further testing of the protocol specifics. Currently this just tests available/not available for all protocols. This is not great, but it gives surprisingly much test coverage at the moment. This will likely go down once more protocols scanners do deeper probing, however. --- protocol/bitcoin_scanner_test.go | 32 ++++++ protocol/ftp_scanner_test.go | 37 ++++++ protocol/http_scanner_test.go | 36 ++++++ protocol/irc_scanner_test.go | 37 ++++++ protocol/mongodb_scanner_test.go | 37 ++++++ protocol/protocol_scanner_test.go | 40 +++++++ protocol/ricochet_scanner_test.go | 37 ++++++ protocol/smtp_scanner_test.go | 37 ++++++ protocol/socks5server_test.go | 184 ++++++++++++++++++++++++++++++ protocol/ssh_scanner_test.go | 37 ++++++ protocol/tls_scanner_test.go | 37 ++++++ protocol/vnc_scanner_test.go | 37 ++++++ protocol/xmpp_scanner_test.go | 37 ++++++ 13 files changed, 625 insertions(+) create mode 100644 protocol/ftp_scanner_test.go create mode 100644 protocol/http_scanner_test.go create mode 100644 protocol/irc_scanner_test.go create mode 100644 protocol/mongodb_scanner_test.go create mode 100644 protocol/protocol_scanner_test.go create mode 100644 protocol/ricochet_scanner_test.go create mode 100644 protocol/smtp_scanner_test.go create mode 100644 protocol/socks5server_test.go create mode 100644 protocol/ssh_scanner_test.go create mode 100644 protocol/tls_scanner_test.go create mode 100644 protocol/vnc_scanner_test.go create mode 100644 protocol/xmpp_scanner_test.go diff --git a/protocol/bitcoin_scanner_test.go b/protocol/bitcoin_scanner_test.go index 8e90ec3..1549d19 100644 --- a/protocol/bitcoin_scanner_test.go +++ b/protocol/bitcoin_scanner_test.go @@ -3,6 +3,7 @@ package protocol import ( "bytes" "encoding/hex" + "net" "testing" ) @@ -125,3 +126,34 @@ func TestDecodeOnion(t *testing.T) { } } } + +type BitcoinIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *BitcoinIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *BitcoinIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestBitcoinScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &BitcoinIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := NewBitcoinProtocolScanner("bitcoin") + + r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if !r.BitcoinDetected { + t.Errorf("Should have detected bitcoin node") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.BitcoinDetected { + t.Errorf("Should not have detected bitcoin node") + } +} diff --git a/protocol/ftp_scanner_test.go b/protocol/ftp_scanner_test.go new file mode 100644 index 0000000..48d06c0 --- /dev/null +++ b/protocol/ftp_scanner_test.go @@ -0,0 +1,37 @@ +package protocol + +import ( + "net" + "testing" +) + +type FTPIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *FTPIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *FTPIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestFTPScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &FTPIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := new(FTPProtocolScanner) + + r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if !r.FTPDetected { + t.Errorf("Should have detected FTP") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.FTPDetected { + t.Errorf("Should not have detected FTP") + } +} diff --git a/protocol/http_scanner_test.go b/protocol/http_scanner_test.go new file mode 100644 index 0000000..d42d5a8 --- /dev/null +++ b/protocol/http_scanner_test.go @@ -0,0 +1,36 @@ +package protocol + +import ( + "net" + "testing" +) + +type HTTPIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *HTTPIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *HTTPIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestHTTPScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &HTTPIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := new(HTTPProtocolScanner) + r := MockCheckHiddenServiceWithDatabase(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if (!r.WebDetected) { + t.Errorf("Should have detected HTTP") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.WebDetected { + t.Errorf("Should not have detected HTTP") + } +} diff --git a/protocol/irc_scanner_test.go b/protocol/irc_scanner_test.go new file mode 100644 index 0000000..1f78fc0 --- /dev/null +++ b/protocol/irc_scanner_test.go @@ -0,0 +1,37 @@ +package protocol + +import ( + "net" + "testing" +) + +type IRCIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *IRCIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *IRCIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestIRCScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &IRCIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := new(IRCProtocolScanner) + + r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if !r.IRCDetected { + t.Errorf("Should have detected IRC") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.IRCDetected { + t.Errorf("Should not have detected IRC") + } +} diff --git a/protocol/mongodb_scanner_test.go b/protocol/mongodb_scanner_test.go new file mode 100644 index 0000000..dbc8ebb --- /dev/null +++ b/protocol/mongodb_scanner_test.go @@ -0,0 +1,37 @@ +package protocol + +import ( + "net" + "testing" +) + +type MongoDBIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *MongoDBIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *MongoDBIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestMongoDBScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &MongoDBIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := new(MongoDBProtocolScanner) + + r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if !r.MongoDBDetected { + t.Errorf("Should have detected MongoDB") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.MongoDBDetected { + t.Errorf("Should not have detected MongoDB") + } +} diff --git a/protocol/protocol_scanner_test.go b/protocol/protocol_scanner_test.go new file mode 100644 index 0000000..62dec7d --- /dev/null +++ b/protocol/protocol_scanner_test.go @@ -0,0 +1,40 @@ +package protocol + +import ( + "github.com/s-rah/onionscan/config" + "github.com/s-rah/onionscan/crawldb" + "github.com/s-rah/onionscan/report" + "io/ioutil" + "os" + "testing" +) + +// Quick mock hidden service check +func MockCheckHiddenService(t *testing.T, proxy *TestSOCKS5Server, ps Scanner, hiddenService string) *report.OnionScanReport { + osc := new(config.OnionScanConfig) + osc.TorProxyAddress = proxy.ListenAddress + osc.Verbose = testing.Verbose() + r := report.NewOnionScanReport(hiddenService) + ps.ScanProtocol(hiddenService, osc, r) + return r +} + +// Full setup with database, this is much slower +func MockCheckHiddenServiceWithDatabase(t *testing.T, proxy *TestSOCKS5Server, ps Scanner, hiddenService string) *report.OnionScanReport { + osc := new(config.OnionScanConfig) + osc.TorProxyAddress = proxy.ListenAddress + osc.Verbose = testing.Verbose() + dbdir, err := ioutil.TempDir("", "test-crawl") + if err != nil { + t.Errorf("Error creating temporary directory: %s", err) + return nil + } + defer os.RemoveAll(dbdir) + osc.Database = new(crawldb.CrawlDB) + osc.Database.NewDB(dbdir) + + r := report.NewOnionScanReport(hiddenService) + ps.ScanProtocol(hiddenService, osc, r) + + return r +} diff --git a/protocol/ricochet_scanner_test.go b/protocol/ricochet_scanner_test.go new file mode 100644 index 0000000..f7b6bc0 --- /dev/null +++ b/protocol/ricochet_scanner_test.go @@ -0,0 +1,37 @@ +package protocol + +import ( + "net" + "testing" +) + +type RicochetIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *RicochetIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *RicochetIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestRicochetScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &RicochetIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := new(RicochetProtocolScanner) + + r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if !r.RicochetDetected { + t.Errorf("Should have detected Ricochet") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.RicochetDetected { + t.Errorf("Should not have detected Ricochet") + } +} diff --git a/protocol/smtp_scanner_test.go b/protocol/smtp_scanner_test.go new file mode 100644 index 0000000..83ff4d7 --- /dev/null +++ b/protocol/smtp_scanner_test.go @@ -0,0 +1,37 @@ +package protocol + +import ( + "net" + "testing" +) + +type SMTPIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *SMTPIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *SMTPIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestSMTPScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &SMTPIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := new(SMTPProtocolScanner) + + r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if !r.SMTPDetected { + t.Errorf("Should have detected SMTP") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.SMTPDetected { + t.Errorf("Should not have detected SMTP") + } +} diff --git a/protocol/socks5server_test.go b/protocol/socks5server_test.go new file mode 100644 index 0000000..250e603 --- /dev/null +++ b/protocol/socks5server_test.go @@ -0,0 +1,184 @@ +package protocol + +import ( + "encoding/binary" + "fmt" + "io" + "net" + "testing" +) + +// Single-threaded SOCKS5 server for testing. +// Instead of making outgoing connections it calls an interface, that has to be provided on creation, +// to handle the connections further. +type TestSOCKS5Server struct { + ListenAddress string + t *testing.T + listener net.Listener + handler IncomingConnectionHandler + waitExit chan bool +} + +// This interface is passed to SOCKS5Server to handle incoming connections +type IncomingConnectionHandler interface { + // Check if connection succeeds. Returns false if connection + // refused, true if connection accepted. + ConnectionSucceeds(domainname string, port uint16) bool + // This is called when the connection succeeds, to handle further processing + HandleConnection(domainname string, port uint16, conn net.Conn) +} + +// SOCKS5 protocol constants +const ( + SocksAuth_NONE = 0x00 +) +const ( + SocksCommand_CONNECT = 0x01 +) +const ( + SocksAddressType_IPV4 = 0x01 + SocksAddressType_DOMAINNAME = 0x03 + SocksAddressType_IPV6 = 0x04 +) + +const ( + SocksResponse_SUCCEEDED = 0x00 + SocksResponse_CONNECTION_REFUSED = 0x05 +) + +// Handle incoming SOCKS5 connection (protocol described in https://www.ietf.org/rfc/rfc1928.txt) +func (os *TestSOCKS5Server) handleConnection(conn net.Conn) { + var err error + recvbuf := make([]byte, 4096) + defer conn.Close() + // Handle incoming connection request + // Version byte + _, err = io.ReadFull(conn, recvbuf[0:1]) + if err != nil { + os.t.Errorf("Network read error reading version byte: %s", err) + return + } + if recvbuf[0] != 0x05 { + os.t.Errorf("Invalid socks version: 0x%02x", recvbuf[0]) + return + } + // Receive authentication methods, prefixed by 1-byte length + _, err = io.ReadFull(conn, recvbuf[0:1]) + if err != nil { + os.t.Errorf("Network read error while reading authentication method count: %s", err) + return + } + nmethods := recvbuf[0] + _, err = io.ReadFull(conn, recvbuf[0:nmethods]) + if err != nil { + os.t.Errorf("Network read error while reading authentication methods: %s", err) + return + } + // We expect authentication method "none" only + if nmethods > 1 || recvbuf[0] != SocksAuth_NONE { + os.t.Errorf("Unexpected authentication methods: %v", recvbuf) + return + } + // Send response + var authresponse = []byte{0x05, SocksAuth_NONE} + var n int + n, err = conn.Write(authresponse) + if err != nil || n != len(authresponse) { + os.t.Errorf("Could not send authentication response: %s", err) + return + } + // Should handle authentication response here for authentication method != 0x00 + // As only no-authentication is supported, skip that. + // Handle connection request: reads (version, command, reserved, address_type) + _, err = io.ReadFull(conn, recvbuf[0:4]) + if err != nil { + os.t.Errorf("Network read error while reading connection request: %s", err) + return + } + if recvbuf[0] != 0x05 { + os.t.Errorf("Invalid SOCKS version: 0x%02x", recvbuf[0]) + return + } + if recvbuf[1] != SocksCommand_CONNECT { + os.t.Errorf("Unhandled SOCKS5 command: 0x%02x", recvbuf[1]) + return + } + if recvbuf[3] != SocksAddressType_DOMAINNAME { + os.t.Errorf("Unhandled SOCKS5 address type: 0x%02x", recvbuf[3]) + return + } + // When we end up here, we've received a domainname connection request + // Receive domain name length + _, err = io.ReadFull(conn, recvbuf[0:1]) + if err != nil { + os.t.Errorf("Error receiving domain name length: %s", err) + return + } + // Receive domain name and port + namelength := recvbuf[0] + _, err = io.ReadFull(conn, recvbuf[0:namelength+2]) + if err != nil { + os.t.Errorf("Error receiving domain name: %s", err) + return + } + name := string(recvbuf[0:namelength]) + port := binary.BigEndian.Uint16(recvbuf[namelength : namelength+2]) + // Send response according to test result + var resp byte + if os.handler.ConnectionSucceeds(name, port) { + resp = SocksResponse_SUCCEEDED + } else { + resp = SocksResponse_CONNECTION_REFUSED + } + var connresponse = []byte{0x05, resp, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + n, err = conn.Write(connresponse) + if err != nil || n != len(connresponse) { + os.t.Errorf("Could not send connection response: %s", err) + return + } + if resp == SocksResponse_SUCCEEDED { + os.handler.HandleConnection(name, port, conn) + } +} + +// Listen for connections. This be called in a goroutine. +func (os *TestSOCKS5Server) listen() { + for { + conn, err := os.listener.Accept() + if err != nil { + os.waitExit <- true + return + } + os.handleConnection(conn) + } +} + +// Start the SOCKS5 server +func (os *TestSOCKS5Server) Start() { + go os.listen() +} + +// Stop the SOCKS5 server +func (os *TestSOCKS5Server) Stop() { + os.listener.Close() + <-os.waitExit +} + +// Create a new SOCKS5 test server +func NewTestSOCKS5Server(t *testing.T, handler IncomingConnectionHandler) (*TestSOCKS5Server, error) { + listenPort := 12345 // Arbitrary, could be dynamic/random + listenAddress := fmt.Sprintf("127.0.0.1:%d", listenPort) + + l, err := net.Listen("tcp", listenAddress) + if err != nil { + t.Errorf("Error listening SOCKS5 server on %s: %s", listenAddress, err) + return nil, err + } + os := new(TestSOCKS5Server) + os.ListenAddress = listenAddress + os.listener = l + os.t = t + os.handler = handler + os.waitExit = make(chan bool) + return os, nil +} diff --git a/protocol/ssh_scanner_test.go b/protocol/ssh_scanner_test.go new file mode 100644 index 0000000..f3c87eb --- /dev/null +++ b/protocol/ssh_scanner_test.go @@ -0,0 +1,37 @@ +package protocol + +import ( + "net" + "testing" +) + +type SSHIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *SSHIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *SSHIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestSSHScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &SSHIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := new(SSHProtocolScanner) + + r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if !r.SSHDetected { + t.Errorf("Should have detected SSH") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.SSHDetected { + t.Errorf("Should not have detected SSH") + } +} diff --git a/protocol/tls_scanner_test.go b/protocol/tls_scanner_test.go new file mode 100644 index 0000000..7829345 --- /dev/null +++ b/protocol/tls_scanner_test.go @@ -0,0 +1,37 @@ +package protocol + +import ( + "net" + "testing" +) + +type TLSIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *TLSIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *TLSIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestTLSScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &TLSIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := new(TLSProtocolScanner) + + r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if !r.TLSDetected { + t.Errorf("Should have detected TLS") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.TLSDetected { + t.Errorf("Should not have detected TLS") + } +} diff --git a/protocol/vnc_scanner_test.go b/protocol/vnc_scanner_test.go new file mode 100644 index 0000000..da9f246 --- /dev/null +++ b/protocol/vnc_scanner_test.go @@ -0,0 +1,37 @@ +package protocol + +import ( + "net" + "testing" +) + +type VNCIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *VNCIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *VNCIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestVNCScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &VNCIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := new(VNCProtocolScanner) + + r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if !r.VNCDetected { + t.Errorf("Should have detected VNC") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.VNCDetected { + t.Errorf("Should not have detected VNC") + } +} diff --git a/protocol/xmpp_scanner_test.go b/protocol/xmpp_scanner_test.go new file mode 100644 index 0000000..142d33b --- /dev/null +++ b/protocol/xmpp_scanner_test.go @@ -0,0 +1,37 @@ +package protocol + +import ( + "net" + "testing" +) + +type XMPPIncomingConnectionHandler struct { + t *testing.T +} + +func (handler *XMPPIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool { + return domainname == "haxaxaxaxaxaxaxa.onion" +} +func (handler *XMPPIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) { + // TODO: further protocol handling +} + +func TestXMPPScanProtocol(t *testing.T) { + proxy, err := NewTestSOCKS5Server(t, &XMPPIncomingConnectionHandler{t}) + if err != nil { + return + } + proxy.Start() + defer proxy.Stop() + + bps := new(XMPPProtocolScanner) + + r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion") + if !r.XMPPDetected { + t.Errorf("Should have detected XMPP") + } + r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion") + if r.XMPPDetected { + t.Errorf("Should not have detected XMPP") + } +}