breakdown for testing and add support of IDN domains

.gitignore

@@ -1,4 +1,5 @@
 cercat
 config.yaml
 dist
-dist/**
+dist/**
+lib/*.xml

go.mod

@@ -3,18 +3,14 @@ module cercat
 go 1.14
 
 require (
-	github.com/CaliDog/certstream-go v0.0.0-20180219203951-6016c5462366
-	github.com/falcosecurity/falcosidekick v0.0.0-20200128210241-efc9dea5ac3a
 	github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee // indirect
 	github.com/gobwas/pool v0.2.0 // indirect
 	github.com/gobwas/ws v1.0.3
-	github.com/google/pprof v0.0.0-20200413000643-b1a96885c1c6 // indirect
-	github.com/gorilla/websocket v1.4.2
-	github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e
-	github.com/likexian/whois-go v1.5.0
-	github.com/likexian/whois-parser-go v1.10.4
-	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
-	github.com/pkg/errors v0.9.1
+	github.com/onsi/ginkgo v1.12.2
+	github.com/onsi/gomega v1.10.1
+	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/sirupsen/logrus v1.2.0
 	github.com/spf13/viper v1.6.3
+	github.com/stretchr/testify v1.4.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 )

lib/config.go

@@ -0,0 +1,92 @@
+package lib
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
+)
+
+type Configuration struct {
+	Workers         int
+	SlackWebHookURL string
+	SlackIconURL    string
+	SlackUsername   string
+	DomainName      string
+	RegIP           string
+	Regexp          string
+	RegIDN          string
+	DisplayErrors   string
+}
+
+const RegStrIP = `^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$`
+
+func GetConfig() *Configuration {
+	c := &Configuration{
+		RegIP: RegStrIP,
+	}
+
+	configFile := kingpin.Flag("configfile", "config file").Short('c').ExistingFile()
+	kingpin.Parse()
+
+	v := viper.New()
+	v.SetDefault("SlackWebhookURL", "")
+	v.SetDefault("SlackIconURL", "")
+	v.SetDefault("SlackUsername", "Cercat")
+	v.SetDefault("DomainName", "")
+	v.SetDefault("Regexp", "")
+	v.SetDefault("Workers", 20)
+	v.SetDefault("DisplayErrors", "false")
+
+	if *configFile != "" {
+		d, f := path.Split(*configFile)
+		if d == "" {
+			d = "."
+		}
+		v.SetConfigName(f[0 : len(f)-len(filepath.Ext(f))])
+		v.AddConfigPath(d)
+		err := v.ReadInConfig()
+		if err != nil {
+			log.Fatalf("[ERROR] : Error when reading config file : %v\n", err)
+		}
+	}
+	v.AutomaticEnv()
+	v.Unmarshal(c)
+
+	if c.SlackUsername == "" {
+		c.SlackUsername = "Cercat"
+	}
+	if c.DisplayErrors == "" || c.DisplayErrors == "false" {
+		log.SetLevel(log.DebugLevel)
+	}
+	if c.Regexp == "" {
+		log.Fatal("Regexp can't be empty")
+	}
+	if c.DomainName == "" {
+		log.Fatal("Specify the domain name to monitor for IDN homographs")
+	}
+	if _, err := regexp.Compile(c.Regexp); err != nil {
+		log.Fatal("Bad regexp")
+	}
+	if c.Workers < -1 {
+		log.Fatal("Workers must be strictly a positive number")
+	}
+
+	c.RegIDN = BuildIDNRegex(c.DomainName)
+
+	return c
+}
+
+func BuildIDNRegex(name string) string {
+	if len(name) < 2 {
+		return ""
+	}
+	// Can detect up to two unicode characters in the domain name.
+	// To adjust according to false positive rate & name length
+	return fmt.Sprintf("[%s]{%d,%d}", strings.ToLower(name), len(name)-2, len(name)-1)
+}

lib/lib.go

@@ -0,0 +1,165 @@
+package lib
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"regexp"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	_ "net/http/pprof"
+
+	_ "expvar"
+
+	"github.com/gobwas/ws"
+	"github.com/gobwas/ws/wsutil"
+)
+
+type Result struct {
+	Domain    string   `json:"domain"`
+	SAN       []string `json:"SAN"`
+	Issuer    string   `json:"issuer"`
+	Addresses []string `json:"Addresses"`
+}
+
+type Certificate struct {
+	MessageType string `json:"message_type"`
+	Data        Data   `json:"data"`
+}
+
+type Data struct {
+	UpdateType string            `json:"update_type"`
+	LeafCert   LeafCert          `json:"leaf_cert"`
+	Chain      []LeafCert        `json:"chain"`
+	CertIndex  float32           `json:"cert_index"`
+	Seen       float32           `json:"seen"`
+	Source     map[string]string `json:"source"`
+}
+
+type LeafCert struct {
+	Subject      map[string]string      `json:"subject"`
+	Extensions   map[string]interface{} `json:"extensions"`
+	NotBefore    float32                `json:"not_before"`
+	NotAfter     float32                `json:"not_after"`
+	SerialNumber string                 `json:"serial_number"`
+	FingerPrint  string                 `json:"fingerprint"`
+	AsDer        string                 `json:"as_der"`
+	AllDomains   []string               `json:"all_domains"`
+}
+
+// MsgChan is the communication channel between certCheckWorkers and LoopCheckCerts
+var MsgChan chan []byte
+
+const certInput = "wss://certstream.calidog.io"
+
+// CertCheckWorker parses certificates and raises alert if matches config
+func CertCheckWorker(config *Configuration) {
+	reg, _ := regexp.Compile(config.Regexp)
+	regIP, _ := regexp.Compile(config.RegIP)
+	regIDN, _ := regexp.Compile(config.RegIDN)
+
+	for {
+		msg := <-MsgChan
+
+		detailedCert, err := ParseResultCertificate(msg, regIP)
+		if err != nil {
+			log.Warnf("Error parsing message: %s", err)
+			continue
+		}
+		if detailedCert == nil {
+			continue
+		}
+		if !IsMatchingCert(detailedCert, reg, regIDN) {
+			continue
+		}
+		notify(config, *detailedCert)
+	}
+}
+
+func ParseResultCertificate(msg []byte, regIP *regexp.Regexp) (*Result, error) {
+	var c Certificate
+	var r *Result
+
+	err := json.Unmarshal(msg, &c)
+	if err != nil || c.MessageType == "heartbeat" {
+		return nil, err
+	}
+
+	r = &Result{
+		Domain:    c.Data.LeafCert.Subject["CN"],
+		Issuer:    c.Data.Chain[0].Subject["O"],
+		SAN:       c.Data.LeafCert.AllDomains,
+		Addresses: []string{"N/A"},
+	}
+	r.Addresses = FetchIPAddresses(r.Domain, regIP)
+	return r, nil
+}
+
+func FetchIPAddresses(name string, regIP *regexp.Regexp) []string {
+	var ipsList []string
+
+	ips, err := net.LookupIP(name)
+	if err != nil || len(ips) == 0 {
+		log.Debugf("Could not fetch IP addresses of domain %s", name)
+		return ipsList
+	}
+	for _, j := range ips {
+		if regIP.MatchString(j.String()) {
+			ipsList = append(ipsList, j.String())
+		}
+	}
+	return ipsList
+}
+
+func IsMatchingCert(cert *Result, reg, regIDN *regexp.Regexp) bool {
+
+	domainList := append(cert.SAN, cert.Domain)
+	for _, domain := range domainList {
+		if isIDN(domain) && regIDN.MatchString(domain) {
+			return true
+		}
+		if reg.MatchString(domain) {
+			return true
+		}
+	}
+	return false
+}
+
+func isIDN(domain string) bool {
+	return strings.HasPrefix(domain, "xn--")
+}
+
+func notify(config *Configuration, detailedCert Result) {
+	b, _ := json.Marshal(detailedCert)
+
+	if config.SlackWebHookURL != "" {
+		go newSlackPayload(detailedCert, config).Post(config)
+	} else {
+		fmt.Printf("A certificate for '%v' has been issued : %v\n", detailedCert.Domain, string(b))
+	}
+}
+
+// LoopCheckCerts Loops on messages from source
+func LoopCheckCerts(config *Configuration) {
+	for {
+		conn, _, _, err := ws.DefaultDialer.Dial(context.Background(), certInput)
+		defer conn.Close()
+		if err != nil {
+			log.Warn("Error connecting to certstream! Sleeping a few seconds and reconnecting...")
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		for {
+			msg, _, err := wsutil.ReadServerData(conn)
+			if err != nil {
+				log.Warn("Error reading message from CertStream")
+				break
+			}
+			MsgChan <- msg
+		}
+	}
+}

lib/lib_suite_test.go

@@ -0,0 +1,14 @@
+package lib_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/reporters"
+	. "github.com/onsi/gomega"
+)
+
+func TestLib(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecsWithDefaultAndCustomReporters(t, "CerCat - Lib", []Reporter{reporters.NewJUnitReporter("test_report-lib.xml")})
+}