fix refresh

pkg/finger/banner.go

@@ -20,9 +20,11 @@ type Options struct {
 }
 
 type Rule struct {
-	Name              string              `json:"name,omitempty"`
-	MatchersCondition string              `yaml:"matchers-condition" json:"matchers_condition,omitempty"`
-	Matchers          []*matchers.Matcher `json:"matchers,omitempty"`
+	Name              string `json:"name,omitempty"`
+	MatchersCondition string `yaml:"matchers-condition" json:"matchers_condition,omitempty"`
+	// 组件太多  采用层级匹配 优化匹配速度
+	Require  []string            `json:"require,omitempty"`
+	Matchers []*matchers.Matcher `json:"matchers,omitempty"`
 }
 
 type Banner struct {
@@ -174,9 +176,21 @@ func getMatchPart(part string, banner *Banner) string {
 	return ""
 }
 
+func isRequire(requires []string, results map[string]map[string]string) bool {
+	for _, require := range requires {
+		if _, ok := results[require]; ok {
+			return true
+		}
+	}
+	return false
+}
+
 func (f *AppFinger) Match(banner *Banner) map[string]map[string]string {
 	result := make(map[string]map[string]string)
 	for _, rule := range f.Rules {
+		if len(rule.Require) > 0 && !isRequire(rule.Require, result) {
+			continue
+		}
 		ok, extract := rule.Match(banner)
 		if ok {
 			if result[rule.Name] == nil {
@@ -186,7 +200,6 @@ func (f *AppFinger) Match(banner *Banner) map[string]map[string]string {
 					result[rule.Name][k] = v
 				}
 			}
-
 		}
 	}
 	return result
@@ -220,5 +233,9 @@ func (f *AppFinger) MatchURI(uri string) (*Banner, map[string]map[string]string)
 	if _, ok := fingerprints["honeypot"]; ok {
 		return banners[len(banners)-1], map[string]map[string]string{"honeypot": make(map[string]string)}
 	}
+	if _, ok := fingerprints["Wordpress"]; ok {
+		fingerprints = mergeMaps(fingerprints, MatchWpPlugin(banners[len(banners)-1]))
+	}
+
 	return banners[len(banners)-1], fingerprints
 }

pkg/finger/http.go

@@ -10,8 +10,8 @@ import (
 	"fmt"
 	"github.com/PuerkitoBio/goquery"
 	"github.com/projectdiscovery/gologger"
-	"github.com/spaolacci/murmur3"
 	"golang.org/x/net/html"
+	"golang.org/x/net/html/charset"
 	"golang.org/x/net/proxy"
 	"golang.org/x/text/encoding/charmap"
 	"golang.org/x/text/encoding/simplifiedchinese"
@@ -49,11 +49,11 @@ func getTitle(body []byte) string {
 	}
 }
 
-func ResponseDecoding(body []byte, charset string) string {
+func ResponseDecoding(body []byte, label string) string {
 	// 根据编码 对响应结果进行解码
 	var str string
-	charset = strings.Trim(strings.Trim(strings.ToUpper(charset), "\""), ";")
-	switch charset {
+	label = strings.Trim(strings.Trim(strings.ToUpper(label), "\""), ";")
+	switch label {
 	case "UTF-8":
 		str = string(body)
 	case "UTF8":
@@ -81,12 +81,12 @@ func ResponseDecoding(body []byte, charset string) string {
 		}
 		str = string(decodedBody)
 	case "GB2312":
-		decoder := simplifiedchinese.HZGB2312.NewDecoder()
-		decodedBody, _, err := transform.Bytes(decoder, body)
+		r, err := charset.NewReaderLabel("gb2312", strings.NewReader(string(body)))
 		if err != nil {
 			return ""
 		}
-		str = string(decodedBody)
+		data, _ := io.ReadAll(r)
+		str = string(data)
 	case "US-ASCII":
 		str = string(body)
 	default:
@@ -142,26 +142,6 @@ func parseIconFile(body string) string {
 func isAbsoluteURL(url string) bool {
 	return !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://"))
 }
-func InsertInto(s string, interval int, sep rune) string {
-	var buffer bytes.Buffer
-	before := interval - 1
-	last := len(s) - 1
-	for i, char := range s {
-		buffer.WriteRune(char)
-		if i%interval == before && i != last {
-			buffer.WriteRune(sep)
-		}
-	}
-	buffer.WriteRune(sep)
-	return buffer.String()
-}
-func murmurhash(data []byte) int32 {
-	stdBase64 := base64.StdEncoding.EncodeToString(data)
-	stdBase64 = InsertInto(stdBase64, 76, '\n')
-	hasher := murmur3.New32WithSeed(0)
-	hasher.Write([]byte(stdBase64))
-	return int32(hasher.Sum32())
-}
 
 func isConnectionResetError(err error) bool {
 	var netErr net.Error
@@ -248,13 +228,21 @@ func Request(uri string, timeout time.Duration, proxyURL string, disableIcon boo
 		nextURI = resp.Request.URL.String()
 		//rawResp.
 		_ = resp.Write(&rawResp)
+		content := rawResp.Bytes()
 		var charset = "UTF-8"
 		contentType := resp.Header.Get("Content-Type")
 		if strings.Contains(contentType, "charset=") {
 			charsetIndex := strings.Index(contentType, "charset=")
 			charset = strings.Trim(contentType[charsetIndex+len("charset="):], " ")
+		} else {
+			tagCharset := extractCharset(string(content))
+			if tagCharset != "" {
+				charset = tagCharset
+			}
+
 		}
-		RawData := ResponseDecoding(rawResp.Bytes(), charset)
+		println(charset)
+		RawData := ResponseDecoding(content, charset)
 		separator := []byte("\r\n\r\n")
 		gologger.Debug().Msg("Dump HTTP Response For " + nextURI + "\r\n" + RawData)
 		index := strings.Index(RawData, "\r\n\r\n")
@@ -267,7 +255,7 @@ func Request(uri string, timeout time.Duration, proxyURL string, disableIcon boo
 		bodyBytes := RawData[index+len(separator):]
 		banner := &Banner{
 			Body:       RawData,
-			BodyHash:   murmurhash([]byte(RawData)),
+			BodyHash:   mmh3([]byte(RawData)),
 			Header:     headerBytes,
 			StatusCode: resp.StatusCode,
 			Response:   RawData,
@@ -280,8 +268,8 @@ func Request(uri string, timeout time.Duration, proxyURL string, disableIcon boo
 		if resp.TLS != nil {
 			cert := resp.TLS.PeerCertificates[0]
 			banner.Certificate = parseCertificateInfo(cert)
+			gologger.Debug().Msg("Dump Cert For " + nextURI + "\r\n" + banner.Certificate)
 		}
-		println(banner.Title)
 		banners = append(banners, banner)
 		// 解析JavaScript跳转
 		jsRedirectUri := parseJavaScript(nextURI, bodyBytes)
@@ -329,7 +317,7 @@ func Request(uri string, timeout time.Duration, proxyURL string, disableIcon boo
 				return banners, err
 			}
 		}
-		iconHash := murmurhash(body)
+		iconHash := mmh3(body)
 		for _, banner := range banners {
 			banner.IconHash = iconHash
 		}

pkg/finger/http_test.go

@@ -5,12 +5,14 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v3"
+	"golang.org/x/net/html/charset"
 	"io"
 	"log"
 	"net"
+	"net/http"
 	"os"
 	"regexp"
+	"strings"
 	"testing"
 	"time"
 )
@@ -46,23 +48,6 @@ Content-Type: text/html; charset=UTF-8`}
 
 }
 
-func TestRule(t *testing.T) {
-	content := `- name: nginx
-  matchers-condition: or
-  matchers:
-  - type: word
-    name: Jupyter
-    words:
-      - <title>Jupyter Notebook</title>
-    part: body
-`
-	a := assert.New(t)
-	var rules []*Rule
-	err := yaml.Unmarshal([]byte(content), &rules)
-	a.Nil(err)
-	a.Equal(rules[0].Name, "nginx")
-}
-
 func TestAppFinger2(t *testing.T) {
 	a := assert.New(t)
 	appFinger := AppFinger{}
@@ -229,3 +214,36 @@ func TestLocalFile(t *testing.T) {
 		return
 	}
 }
+
+func TestRegex2(t *testing.T) {
+	regex, _ := regexp.Compile(`/wp-content/plugins/([\w-]+)/(?:.*\?ver=([\d.]+))?`)
+	matched := regex.FindAllStringSubmatch(`<script type='text/javascript' src='https://care.cz/wp-content/plugins/salient-portfolio/js/third-party/imagesLoaded.min.js?ver=4.1.4' id='imagesLoaded-js'></script>
+<script type='text/javascript' src='https://care.cz/wp-content/plugins/interactive-geo-maps/assets/maps-service/assets/js/app.min.js?' id='interactive-geo-maps_map_service-js'></script>`, -1)
+	fmt.Printf("%v\n", matched)
+}
+
+func TestMurmurhash(t *testing.T) {
+	assert.Equal(t, int32(851989093), mmh3([]byte("foo")))
+}
+
+func TestCharset(t *testing.T) {
+	req, err := http.Get("http://1.180.157.154:8087/login.jsp")
+	if err != nil {
+		return
+	}
+	//println(req.TransferEncoding)
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		return
+	}
+	r, err := charset.NewReaderLabel("gb2312", strings.NewReader(string(body)))
+	if err != nil {
+		return
+	}
+
+	decodedBody, err := io.ReadAll(r)
+	if err != nil {
+		return
+	}
+	println(string(decodedBody))
+}

pkg/finger/redirect.go

@@ -16,7 +16,7 @@ func extractUri(n *html.Node) string {
 			parts := strings.Split(attr.Val, ";")
 			for _, part := range parts {
 				if strings.Contains(strings.ToLower(part), "url=") {
-					redirectURL := strings.TrimPrefix(strings.TrimSpace(strings.Split(part, "=")[1]), "/")
+					redirectURL := strings.TrimSpace(strings.SplitN(part, "=", 2)[1])
 					return redirectURL
 				}
 			}
@@ -35,7 +35,6 @@ func findRefresh(n *html.Node) string {
 					return uri
 
 				}
-				//return
 			}
 		}
 	}
@@ -125,6 +124,45 @@ func getExecRedirect(url string, jsCodes []string, onload string) string {
 	}
 	return result.String()
 }
+func findAttribute(attrs []html.Attribute, key string) string {
+	for _, attr := range attrs {
+		if attr.Key == key {
+			return attr.Val
+		}
+	}
+	return ""
+}
+func extractCharset(htmlContent string) string {
+	reader := strings.NewReader(htmlContent)
+	doc, err := html.Parse(reader)
+	if err != nil {
+		return "UTF-8"
+	}
+
+	var charset string
+	var traverse func(*html.Node)
+	traverse = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "meta" {
+			for _, attr := range n.Attr {
+				if attr.Key == "http-equiv" && strings.EqualFold(attr.Val, "Content-Type") {
+					contentAttr := findAttribute(n.Attr, "content")
+					charsetIndex := strings.Index(contentAttr, "charset=")
+					if charsetIndex != -1 {
+						charset = contentAttr[charsetIndex+len("charset="):]
+						break
+					}
+				}
+			}
+		}
+
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			traverse(c)
+		}
+	}
+	traverse(doc)
+	return strings.ToUpper(strings.TrimSpace(charset))
+}
+
 func parseJavaScript(url string, htmlContent string) string {
 	// 在这里解析JavaScript，提取跳转信息
 	//<meta http-equiv="Refresh"content="0;url=/yyoa/index.jsp">

pkg/finger/tools.go

@@ -0,0 +1,35 @@
+package finger
+
+import (
+	"bytes"
+	"encoding/base64"
+	"github.com/spaolacci/murmur3"
+)
+
+func InsertInto(s string, interval int, sep rune) string {
+	var buffer bytes.Buffer
+	before := interval - 1
+	last := len(s) - 1
+	for i, char := range s {
+		buffer.WriteRune(char)
+		if i%interval == before && i != last {
+			buffer.WriteRune(sep)
+		}
+	}
+	buffer.WriteRune(sep)
+	return buffer.String()
+}
+func mmh3(data []byte) int32 {
+	hash := murmur3.New32WithSeed(0)
+	_, err := hash.Write([]byte(base64Py(data)))
+	if err != nil {
+		return 0
+	}
+	return int32(hash.Sum32())
+}
+
+func base64Py(data []byte) string {
+	// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
+	stdBase64 := base64.StdEncoding.EncodeToString(data)
+	return InsertInto(stdBase64, 76, '\n')
+}

pkg/matchers/compile.go

@@ -9,7 +9,6 @@ import (
 // CompileMatchers performs the initial setup operation on a matcher
 func (matcher *Matcher) CompileMatchers() error {
 	var ok bool
-
 	// Support hexadecimal encoding for matchers too.
 	if matcher.Encoding == "hex" {
 		for i, word := range matcher.Words {