Add framework for testing protocol scanners

This adds a mock SOCKS5 proxy to be able to test protocol
scanners. It creates a single-threaded SOCKS5 proxy on localhost
which processes connection requests and dispatches them to a test-specific
interface, which can do further testing of the protocol specifics.

Currently this just tests available/not available for all protocols.
This is not great, but it gives surprisingly much test coverage at the
moment. This will likely go down once more protocols scanners do deeper
probing, however.

protocol/bitcoin_scanner_test.go

@@ -3,6 +3,7 @@ package protocol
 import (
 	"bytes"
 	"encoding/hex"
+	"net"
 	"testing"
 )
 
@@ -125,3 +126,34 @@ func TestDecodeOnion(t *testing.T) {
 		}
 	}
 }
+
+type BitcoinIncomingConnectionHandler struct {
+	t *testing.T
+}
+
+func (handler *BitcoinIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool {
+	return domainname == "haxaxaxaxaxaxaxa.onion"
+}
+func (handler *BitcoinIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) {
+	// TODO: further protocol handling
+}
+
+func TestBitcoinScanProtocol(t *testing.T) {
+	proxy, err := NewTestSOCKS5Server(t, &BitcoinIncomingConnectionHandler{t})
+	if err != nil {
+		return
+	}
+	proxy.Start()
+	defer proxy.Stop()
+
+	bps := NewBitcoinProtocolScanner("bitcoin")
+
+	r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion")
+	if !r.BitcoinDetected {
+		t.Errorf("Should have detected bitcoin node")
+	}
+	r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion")
+	if r.BitcoinDetected {
+		t.Errorf("Should not have detected bitcoin node")
+	}
+}

protocol/ftp_scanner_test.go

@@ -0,0 +1,37 @@
+package protocol
+
+import (
+	"net"
+	"testing"
+)
+
+type FTPIncomingConnectionHandler struct {
+	t *testing.T
+}
+
+func (handler *FTPIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool {
+	return domainname == "haxaxaxaxaxaxaxa.onion"
+}
+func (handler *FTPIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) {
+	// TODO: further protocol handling
+}
+
+func TestFTPScanProtocol(t *testing.T) {
+	proxy, err := NewTestSOCKS5Server(t, &FTPIncomingConnectionHandler{t})
+	if err != nil {
+		return
+	}
+	proxy.Start()
+	defer proxy.Stop()
+
+	bps := new(FTPProtocolScanner)
+
+	r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion")
+	if !r.FTPDetected {
+		t.Errorf("Should have detected FTP")
+	}
+	r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion")
+	if r.FTPDetected {
+		t.Errorf("Should not have detected FTP")
+	}
+}

protocol/http_scanner_test.go

@@ -0,0 +1,36 @@
+package protocol
+
+import (
+	"net"
+	"testing"
+)
+
+type HTTPIncomingConnectionHandler struct {
+	t *testing.T
+}
+
+func (handler *HTTPIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool {
+	return domainname == "haxaxaxaxaxaxaxa.onion"
+}
+func (handler *HTTPIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) {
+	// TODO: further protocol handling
+}
+
+func TestHTTPScanProtocol(t *testing.T) {
+	proxy, err := NewTestSOCKS5Server(t, &HTTPIncomingConnectionHandler{t})
+	if err != nil {
+		return
+	}
+	proxy.Start()
+	defer proxy.Stop()
+
+	bps := new(HTTPProtocolScanner)
+	r := MockCheckHiddenServiceWithDatabase(t, proxy, bps, "haxaxaxaxaxaxaxa.onion")
+	if (!r.WebDetected) {
+		t.Errorf("Should have detected HTTP")
+	}
+	r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion")
+	if r.WebDetected {
+		t.Errorf("Should not have detected HTTP")
+	}
+}

protocol/irc_scanner_test.go

@@ -0,0 +1,37 @@
+package protocol
+
+import (
+	"net"
+	"testing"
+)
+
+type IRCIncomingConnectionHandler struct {
+	t *testing.T
+}
+
+func (handler *IRCIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool {
+	return domainname == "haxaxaxaxaxaxaxa.onion"
+}
+func (handler *IRCIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) {
+	// TODO: further protocol handling
+}
+
+func TestIRCScanProtocol(t *testing.T) {
+	proxy, err := NewTestSOCKS5Server(t, &IRCIncomingConnectionHandler{t})
+	if err != nil {
+		return
+	}
+	proxy.Start()
+	defer proxy.Stop()
+
+	bps := new(IRCProtocolScanner)
+
+	r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion")
+	if !r.IRCDetected {
+		t.Errorf("Should have detected IRC")
+	}
+	r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion")
+	if r.IRCDetected {
+		t.Errorf("Should not have detected IRC")
+	}
+}

protocol/mongodb_scanner_test.go

@@ -0,0 +1,37 @@
+package protocol
+
+import (
+	"net"
+	"testing"
+)
+
+type MongoDBIncomingConnectionHandler struct {
+	t *testing.T
+}
+
+func (handler *MongoDBIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool {
+	return domainname == "haxaxaxaxaxaxaxa.onion"
+}
+func (handler *MongoDBIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) {
+	// TODO: further protocol handling
+}
+
+func TestMongoDBScanProtocol(t *testing.T) {
+	proxy, err := NewTestSOCKS5Server(t, &MongoDBIncomingConnectionHandler{t})
+	if err != nil {
+		return
+	}
+	proxy.Start()
+	defer proxy.Stop()
+
+	bps := new(MongoDBProtocolScanner)
+
+	r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion")
+	if !r.MongoDBDetected {
+		t.Errorf("Should have detected MongoDB")
+	}
+	r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion")
+	if r.MongoDBDetected {
+		t.Errorf("Should not have detected MongoDB")
+	}
+}

protocol/protocol_scanner_test.go

@@ -0,0 +1,40 @@
+package protocol
+
+import (
+	"github.com/s-rah/onionscan/config"
+	"github.com/s-rah/onionscan/crawldb"
+	"github.com/s-rah/onionscan/report"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+// Quick mock hidden service check
+func MockCheckHiddenService(t *testing.T, proxy *TestSOCKS5Server, ps Scanner, hiddenService string) *report.OnionScanReport {
+	osc := new(config.OnionScanConfig)
+	osc.TorProxyAddress = proxy.ListenAddress
+	osc.Verbose = testing.Verbose()
+	r := report.NewOnionScanReport(hiddenService)
+	ps.ScanProtocol(hiddenService, osc, r)
+	return r
+}
+
+// Full setup with database, this is much slower
+func MockCheckHiddenServiceWithDatabase(t *testing.T, proxy *TestSOCKS5Server, ps Scanner, hiddenService string) *report.OnionScanReport {
+	osc := new(config.OnionScanConfig)
+	osc.TorProxyAddress = proxy.ListenAddress
+	osc.Verbose = testing.Verbose()
+	dbdir, err := ioutil.TempDir("", "test-crawl")
+	if err != nil {
+		t.Errorf("Error creating temporary directory: %s", err)
+		return nil
+	}
+	defer os.RemoveAll(dbdir)
+	osc.Database = new(crawldb.CrawlDB)
+	osc.Database.NewDB(dbdir)
+
+	r := report.NewOnionScanReport(hiddenService)
+	ps.ScanProtocol(hiddenService, osc, r)
+
+	return r
+}

protocol/ricochet_scanner_test.go

@@ -0,0 +1,37 @@
+package protocol
+
+import (
+	"net"
+	"testing"
+)
+
+type RicochetIncomingConnectionHandler struct {
+	t *testing.T
+}
+
+func (handler *RicochetIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool {
+	return domainname == "haxaxaxaxaxaxaxa.onion"
+}
+func (handler *RicochetIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) {
+	// TODO: further protocol handling
+}
+
+func TestRicochetScanProtocol(t *testing.T) {
+	proxy, err := NewTestSOCKS5Server(t, &RicochetIncomingConnectionHandler{t})
+	if err != nil {
+		return
+	}
+	proxy.Start()
+	defer proxy.Stop()
+
+	bps := new(RicochetProtocolScanner)
+
+	r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion")
+	if !r.RicochetDetected {
+		t.Errorf("Should have detected Ricochet")
+	}
+	r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion")
+	if r.RicochetDetected {
+		t.Errorf("Should not have detected Ricochet")
+	}
+}

protocol/smtp_scanner_test.go

@@ -0,0 +1,37 @@
+package protocol
+
+import (
+	"net"
+	"testing"
+)
+
+type SMTPIncomingConnectionHandler struct {
+	t *testing.T
+}
+
+func (handler *SMTPIncomingConnectionHandler) ConnectionSucceeds(domainname string, port uint16) bool {
+	return domainname == "haxaxaxaxaxaxaxa.onion"
+}
+func (handler *SMTPIncomingConnectionHandler) HandleConnection(domainname string, port uint16, conn net.Conn) {
+	// TODO: further protocol handling
+}
+
+func TestSMTPScanProtocol(t *testing.T) {
+	proxy, err := NewTestSOCKS5Server(t, &SMTPIncomingConnectionHandler{t})
+	if err != nil {
+		return
+	}
+	proxy.Start()
+	defer proxy.Stop()
+
+	bps := new(SMTPProtocolScanner)
+
+	r := MockCheckHiddenService(t, proxy, bps, "haxaxaxaxaxaxaxa.onion")
+	if !r.SMTPDetected {
+		t.Errorf("Should have detected SMTP")
+	}
+	r = MockCheckHiddenService(t, proxy, bps, "nononononononono.onion")
+	if r.SMTPDetected {
+		t.Errorf("Should not have detected SMTP")
+	}
+}