new method for detecting IDN + add comments for functions/types + une…

…xport some functions/types

example.yaml

@@ -2,7 +2,6 @@
 SlackWebhookURL: "" #Slack Webhook URL
 SlackIconURL: "" #Slack Icon (Avatar) URL
 SlackUsername: "" #Slack Username
-Regexp: ".*\\.fr$" #Regexp to match. Can't be empty. It uses Golang regexp format
-DomainName: test
+Regexp: ".*" #Regexp to match. Can't be empty. It uses Golang regexp format
 Workers: 20 #Number of workers for consuming stream from CertStream
-DisplayErrors: false #Enable/Disable display of errors in logs
+DisplayErrors: true #Enable/Disable display of errors in logs

go.mod

@@ -13,5 +13,6 @@ require (
 	github.com/sirupsen/logrus v1.2.0
 	github.com/spf13/viper v1.6.3
 	github.com/stretchr/testify v1.4.0 // indirect
+	golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 )

go.sum

@@ -166,6 +166,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

lib/config.go

@@ -20,14 +20,13 @@ type Configuration struct {
 	RegIP           string
 	Regexp          string
 	DisplayErrors   string
+	Homoglyph       map[string]string
 }
 
-const regStrIP = `^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$`
-
 // GetConfig provides a Configuration
 func GetConfig() *Configuration {
 	c := &Configuration{
-		RegIP: regStrIP,
+		Homoglyph: getHomoglyphMap(),
 	}
 
 	configFile := kingpin.Flag("configfile", "config file").Short('c').ExistingFile()
@@ -66,9 +65,7 @@ func GetConfig() *Configuration {
 	if c.Regexp == "" {
 		log.Fatal("Regexp can't be empty")
 	}
-	if c.DomainName == "" {
-		log.Fatal("Specify the domain name to monitor for IDN homographs")
-	}
+
 	if _, err := regexp.Compile(c.Regexp); err != nil {
 		log.Fatal("Bad regexp")
 	}

lib/homoglyph.go

@@ -4,6 +4,7 @@ import (
 	"github.com/picatz/homoglyphr"
 )
 
+// getHomoglyphMap generates a map of homoglyphs for replacement
 func getHomoglyphMap() map[string]string {
 	alphabet := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"}
 	homoglyph := map[string]string{}
@@ -14,3 +15,20 @@ func getHomoglyphMap() map[string]string {
 	}
 	return homoglyph
 }
+
+// replaceHomoglyph replaces homoglyphs in a string by close latin letters
+func replaceHomoglyph(idn string, homoglyphMap map[string]string) string {
+	var s string
+	for _, i := range idn {
+		if i > 127 {
+			if letter, present := homoglyphMap[string(i)]; present {
+				s += letter
+			} else {
+				s += string(i)
+			}
+		} else {
+			s += string(i)
+		}
+	}
+	return s
+}

lib/lib.go

@@ -3,44 +3,44 @@ package lib
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"net"
 	"regexp"
 	"strings"
 	"time"
 
-	log "github.com/sirupsen/logrus"
-
-	_ "net/http/pprof"
-
-	_ "expvar"
-
 	"github.com/gobwas/ws"
 	"github.com/gobwas/ws/wsutil"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/net/idna"
 )
 
-type Result struct {
+// result represents a catched certificate
+type result struct {
 	Domain    string   `json:"domain"`
+	IDN       string   `json:"IDN,omitempty"`
 	SAN       []string `json:"SAN"`
 	Issuer    string   `json:"issuer"`
 	Addresses []string `json:"Addresses"`
 }
 
-type Certificate struct {
+// certificate represents a certificate from CertStream
+type certificate struct {
 	MessageType string `json:"message_type"`
-	Data        Data   `json:"data"`
+	Data        data   `json:"data"`
 }
 
-type Data struct {
+// data represents data field for a certificate from CertStream
+type data struct {
 	UpdateType string            `json:"update_type"`
-	LeafCert   LeafCert          `json:"leaf_cert"`
-	Chain      []LeafCert        `json:"chain"`
+	LeafCert   leafCert          `json:"leaf_cert"`
+	Chain      []leafCert        `json:"chain"`
 	CertIndex  float32           `json:"cert_index"`
 	Seen       float32           `json:"seen"`
 	Source     map[string]string `json:"source"`
 }
 
-type LeafCert struct {
+// leafCert represents leaf_cert field from CertStream
+type leafCert struct {
 	Subject      map[string]string      `json:"subject"`
 	Extensions   map[string]interface{} `json:"extensions"`
 	NotBefore    float32                `json:"not_before"`
@@ -51,54 +51,61 @@ type LeafCert struct {
 	AllDomains   []string               `json:"all_domains"`
 }
 
-// MsgChan is the communication channel between certCheckWorkers and LoopCheckCerts
+// MsgChan is the communication channel between certCheckWorkers and LoopCertStream
 var MsgChan chan []byte
 
+// the websocket stream from calidog
 const certInput = "wss://certstream.calidog.io"
 
 // CertCheckWorker parses certificates and raises alert if matches config
 func CertCheckWorker(config *Configuration) {
 	reg, _ := regexp.Compile(config.Regexp)
-	regIP, _ := regexp.Compile(config.RegIP)
 
 	for {
 		msg := <-MsgChan
 
-		detailedCert, err := ParseResultCertificate(msg, regIP)
+		detailedCert, err := parseResultCertificate(msg)
 		if err != nil {
 			log.Warnf("Error parsing message: %s", err)
 			continue
 		}
 		if detailedCert == nil {
 			continue
 		}
-		if !IsMatchingCert(detailedCert, reg) {
+		if !isMatchingCert(config, detailedCert, reg) {
 			continue
 		}
-		notify(config, *detailedCert)
+		slackPost(config, *detailedCert)
 	}
 }
 
-func ParseResultCertificate(msg []byte, regIP *regexp.Regexp) (*Result, error) {
-	var c Certificate
-	var r *Result
+// parseResultCertificate parses certificate details
+func parseResultCertificate(msg []byte) (*result, error) {
+	var c certificate
+	var r *result
 
 	err := json.Unmarshal(msg, &c)
 	if err != nil || c.MessageType == "heartbeat" {
 		return nil, err
 	}
 
-	r = &Result{
+	r = &result{
 		Domain:    c.Data.LeafCert.Subject["CN"],
 		Issuer:    c.Data.Chain[0].Subject["O"],
 		SAN:       c.Data.LeafCert.AllDomains,
 		Addresses: []string{"N/A"},
 	}
-	r.Addresses = FetchIPAddresses(r.Domain, regIP)
+	r.Addresses = fetchIPAddresses(r.Domain)
 	return r, nil
 }
 
-func FetchIPAddresses(name string, regIP *regexp.Regexp) []string {
+// isIPv4Net checks if IP is IPv4
+func isIPv4Net(host string) bool {
+	return net.ParseIP(host) != nil
+}
+
+// fetchIPAddresses resolves domain to get IP
+func fetchIPAddresses(name string) []string {
 	var ipsList []string
 
 	ips, err := net.LookupIP(name)
@@ -107,47 +114,54 @@ func FetchIPAddresses(name string, regIP *regexp.Regexp) []string {
 		return ipsList
 	}
 	for _, j := range ips {
-		if regIP.MatchString(j.String()) {
+		if isIPv4Net(j.String()) {
 			ipsList = append(ipsList, j.String())
 		}
 	}
 	return ipsList
 }
 
-func IsMatchingCert(cert *Result, reg *regexp.Regexp) bool {
+// isIDN checks if domain is an IDN
+func isIDN(domain string) bool {
+	return strings.HasPrefix(domain, "xn--")
+}
+
+// isMatchingCert checks if certificate matches the regexp
+func isMatchingCert(config *Configuration, cert *result, reg *regexp.Regexp) bool {
 	domainList := append(cert.SAN, cert.Domain)
 	for _, domain := range domainList {
-		// if isIDN(domain) {
-		// 	return true
-		// }
+		if isIDN(domain) {
+			unicodeDomain, _ := idna.ToUnicode(domain)
+			cert.IDN = unicodeDomain
+			if reg.MatchString(replaceHomoglyph(unicodeDomain, config.Homoglyph)) {
+				return true
+			}
+		}
 		if reg.MatchString(domain) {
 			return true
 		}
 	}
 	return false
 }
 
-func isIDN(domain string) bool {
-	return strings.HasPrefix(domain, "xn--")
-}
-
-func notify(config *Configuration, detailedCert Result) {
+// slackPost posts event to Slack
+func slackPost(config *Configuration, detailedCert result) {
 	b, _ := json.Marshal(detailedCert)
 
 	if config.SlackWebHookURL != "" {
-		go newSlackPayload(detailedCert, config).Post(config)
+		go newSlackPayload(&detailedCert, config).post(config)
 	} else {
-		fmt.Printf("A certificate for '%v' has been issued : %v\n", detailedCert.Domain, string(b))
+		log.Infof("A certificate for '%v' has been issued : %v\n", detailedCert.Domain, string(b))
 	}
 }
 
-// LoopCheckCerts Loops on messages from source
-func LoopCheckCerts(config *Configuration) {
+// LoopCertStream gathers messages from CertStream
+func LoopCertStream(config *Configuration) {
 	for {
 		conn, _, _, err := ws.DefaultDialer.Dial(context.Background(), certInput)
 		defer conn.Close()
 		if err != nil {
-			log.Warn("Error connecting to certstream! Sleeping a few seconds and reconnecting...")
+			log.Warn("Error connecting to CertStream! Sleeping a few seconds and reconnecting...")
 			time.Sleep(1 * time.Second)
 			continue
 		}

lib/lib_suite_test.go

@@ -1,4 +1,4 @@
-package lib_test
+package lib
 
 import (
 	"testing"

lib/lib_test.go

@@ -1,62 +1,60 @@
-package lib_test
+package lib
 
 import (
-	"cercat/lib"
 	"io/ioutil"
-	"regexp"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 var _ = Describe("Handler", func() {
 	Describe("isMatchingCert", func() {
-		reg, _ := regexp.Compile(`.+\.com`)
-		regIDN, _ := regexp.Compile(lib.BuildIDNRegex("test"))
 		Describe("If certificate matches", func() {
-			cert := &lib.Result{Domain: "www.test.com"}
+			cert := &result{Domain: "www.test.com"}
+			reg := ".*test.*"
 			It("should return true", func() {
-				result := lib.IsMatchingCert(cert, reg, regIDN)
+				result := isMatchingCert(cert, reg)
 				Expect(result).To(BeTrue())
 			})
 		})
 		Describe("If alternative subject matches", func() {
-			cert := &lib.Result{Domain: "www.test.net", SAN: []string{"www.test.com"}}
+			cert := &Result{Domain: "www.test.net", SAN: []string{"www.test.com"}}
+			reg := ".*test.*"
 			It("should return true", func() {
-				result := lib.IsMatchingCert(cert, reg, regIDN)
+				result := isMatchingCert(cert, reg)
 				Expect(result).To(BeTrue())
 			})
 		})
 		Describe("If domain is IDN", func() {
-			cert := &lib.Result{Domain: "xn--tst-rdd.com"}
+			cert := &Result{Domain: "xn--tst-rdd.com"}
+			reg := ".*test.*"
 			It("should return true", func() {
-				result := lib.IsMatchingCert(cert, reg, regIDN)
+				result := isMatchingCert(cert, reg)
 				Expect(result).To(BeTrue())
 			})
 		})
 	})
 	Describe("parseResultCertificate", func() {
-		regIP, _ := regexp.Compile(lib.RegStrIP)
 		Describe("If cannot marshall message", func() {
 			msg := []byte("")
 			It("should return nil and error", func() {
-				result, err := lib.ParseResultCertificate(msg, regIP)
+				result, err := parseResultCertificate(msg)
 				Expect(result).To(BeNil())
 				Expect(err).To(HaveOccurred())
 			})
 		})
 		Describe("If message is heartbeat", func() {
 			msg, _ := ioutil.ReadFile("../res/heartbeat.json")
 			It("should return nil", func() {
-				result, err := lib.ParseResultCertificate(msg, regIP)
+				result, err := parseResultCertificate(msg)
 				Expect(result).To(BeNil())
 				Expect(err).ToNot(HaveOccurred())
 			})
 		})
 		Describe("If message is regular", func() {
 			msg, _ := ioutil.ReadFile("../res/cert.json")
 			It("should return valid infos", func() {
-				result, err := lib.ParseResultCertificate(msg, regIP)
+				result, err := parseResultCertificate(msg)
 				Expect(result.Domain).Should(Equal("baden-mueller.de"))
 				Expect(result.SAN).Should(Equal([]string{"baden-mueller.de", "www.baden-mueller.de"}))
 				Expect(result.Issuer).Should(Equal("Let's Encrypt"))