From cff6da74214554c605c37c0ea22585996aaa3dbe Mon Sep 17 00:00:00 2001 From: tongchengbin <58296672@qq.com> Date: Thu, 11 Jan 2024 19:13:14 +0800 Subject: [PATCH] fix refresh --- pkg/finger/banner.go | 25 ++++++++++++++++--- pkg/finger/http.go | 50 +++++++++++++++----------------------- pkg/finger/http_test.go | 54 +++++++++++++++++++++++++++-------------- pkg/finger/redirect.go | 42 ++++++++++++++++++++++++++++++-- pkg/finger/tools.go | 35 ++++++++++++++++++++++++++ pkg/matchers/compile.go | 1 - 6 files changed, 151 insertions(+), 56 deletions(-) create mode 100644 pkg/finger/tools.go diff --git a/pkg/finger/banner.go b/pkg/finger/banner.go index aac691f..770d29a 100644 --- a/pkg/finger/banner.go +++ b/pkg/finger/banner.go @@ -20,9 +20,11 @@ type Options struct { } type Rule struct { - Name string `json:"name,omitempty"` - MatchersCondition string `yaml:"matchers-condition" json:"matchers_condition,omitempty"` - Matchers []*matchers.Matcher `json:"matchers,omitempty"` + Name string `json:"name,omitempty"` + MatchersCondition string `yaml:"matchers-condition" json:"matchers_condition,omitempty"` + // 组件太多 采用层级匹配 优化匹配速度 + Require []string `json:"require,omitempty"` + Matchers []*matchers.Matcher `json:"matchers,omitempty"` } type Banner struct { @@ -174,9 +176,21 @@ func getMatchPart(part string, banner *Banner) string { return "" } +func isRequire(requires []string, results map[string]map[string]string) bool { + for _, require := range requires { + if _, ok := results[require]; ok { + return true + } + } + return false +} + func (f *AppFinger) Match(banner *Banner) map[string]map[string]string { result := make(map[string]map[string]string) for _, rule := range f.Rules { + if len(rule.Require) > 0 && !isRequire(rule.Require, result) { + continue + } ok, extract := rule.Match(banner) if ok { if result[rule.Name] == nil { @@ -186,7 +200,6 @@ func (f *AppFinger) Match(banner *Banner) map[string]map[string]string { result[rule.Name][k] = v } } - } } return result @@ -220,5 +233,9 @@ func (f *AppFinger) MatchURI(uri string) (*Banner, map[string]map[string]string) if _, ok := fingerprints["honeypot"]; ok { return banners[len(banners)-1], map[string]map[string]string{"honeypot": make(map[string]string)} } + if _, ok := fingerprints["Wordpress"]; ok { + fingerprints = mergeMaps(fingerprints, MatchWpPlugin(banners[len(banners)-1])) + } + return banners[len(banners)-1], fingerprints } diff --git a/pkg/finger/http.go b/pkg/finger/http.go index ef982fe..0307acc 100644 --- a/pkg/finger/http.go +++ b/pkg/finger/http.go @@ -10,8 +10,8 @@ import ( "fmt" "github.com/PuerkitoBio/goquery" "github.com/projectdiscovery/gologger" - "github.com/spaolacci/murmur3" "golang.org/x/net/html" + "golang.org/x/net/html/charset" "golang.org/x/net/proxy" "golang.org/x/text/encoding/charmap" "golang.org/x/text/encoding/simplifiedchinese" @@ -49,11 +49,11 @@ func getTitle(body []byte) string { } } -func ResponseDecoding(body []byte, charset string) string { +func ResponseDecoding(body []byte, label string) string { // 根据编码 对响应结果进行解码 var str string - charset = strings.Trim(strings.Trim(strings.ToUpper(charset), "\""), ";") - switch charset { + label = strings.Trim(strings.Trim(strings.ToUpper(label), "\""), ";") + switch label { case "UTF-8": str = string(body) case "UTF8": @@ -81,12 +81,12 @@ func ResponseDecoding(body []byte, charset string) string { } str = string(decodedBody) case "GB2312": - decoder := simplifiedchinese.HZGB2312.NewDecoder() - decodedBody, _, err := transform.Bytes(decoder, body) + r, err := charset.NewReaderLabel("gb2312", strings.NewReader(string(body))) if err != nil { return "" } - str = string(decodedBody) + data, _ := io.ReadAll(r) + str = string(data) case "US-ASCII": str = string(body) default: @@ -142,26 +142,6 @@ func parseIconFile(body string) string { func isAbsoluteURL(url string) bool { return !(strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://")) } -func InsertInto(s string, interval int, sep rune) string { - var buffer bytes.Buffer - before := interval - 1 - last := len(s) - 1 - for i, char := range s { - buffer.WriteRune(char) - if i%interval == before && i != last { - buffer.WriteRune(sep) - } - } - buffer.WriteRune(sep) - return buffer.String() -} -func murmurhash(data []byte) int32 { - stdBase64 := base64.StdEncoding.EncodeToString(data) - stdBase64 = InsertInto(stdBase64, 76, '\n') - hasher := murmur3.New32WithSeed(0) - hasher.Write([]byte(stdBase64)) - return int32(hasher.Sum32()) -} func isConnectionResetError(err error) bool { var netErr net.Error @@ -248,13 +228,21 @@ func Request(uri string, timeout time.Duration, proxyURL string, disableIcon boo nextURI = resp.Request.URL.String() //rawResp. _ = resp.Write(&rawResp) + content := rawResp.Bytes() var charset = "UTF-8" contentType := resp.Header.Get("Content-Type") if strings.Contains(contentType, "charset=") { charsetIndex := strings.Index(contentType, "charset=") charset = strings.Trim(contentType[charsetIndex+len("charset="):], " ") + } else { + tagCharset := extractCharset(string(content)) + if tagCharset != "" { + charset = tagCharset + } + } - RawData := ResponseDecoding(rawResp.Bytes(), charset) + println(charset) + RawData := ResponseDecoding(content, charset) separator := []byte("\r\n\r\n") gologger.Debug().Msg("Dump HTTP Response For " + nextURI + "\r\n" + RawData) index := strings.Index(RawData, "\r\n\r\n") @@ -267,7 +255,7 @@ func Request(uri string, timeout time.Duration, proxyURL string, disableIcon boo bodyBytes := RawData[index+len(separator):] banner := &Banner{ Body: RawData, - BodyHash: murmurhash([]byte(RawData)), + BodyHash: mmh3([]byte(RawData)), Header: headerBytes, StatusCode: resp.StatusCode, Response: RawData, @@ -280,8 +268,8 @@ func Request(uri string, timeout time.Duration, proxyURL string, disableIcon boo if resp.TLS != nil { cert := resp.TLS.PeerCertificates[0] banner.Certificate = parseCertificateInfo(cert) + gologger.Debug().Msg("Dump Cert For " + nextURI + "\r\n" + banner.Certificate) } - println(banner.Title) banners = append(banners, banner) // 解析JavaScript跳转 jsRedirectUri := parseJavaScript(nextURI, bodyBytes) @@ -329,7 +317,7 @@ func Request(uri string, timeout time.Duration, proxyURL string, disableIcon boo return banners, err } } - iconHash := murmurhash(body) + iconHash := mmh3(body) for _, banner := range banners { banner.IconHash = iconHash } diff --git a/pkg/finger/http_test.go b/pkg/finger/http_test.go index 3cb298d..e23635a 100644 --- a/pkg/finger/http_test.go +++ b/pkg/finger/http_test.go @@ -5,12 +5,14 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" + "golang.org/x/net/html/charset" "io" "log" "net" + "net/http" "os" "regexp" + "strings" "testing" "time" ) @@ -46,23 +48,6 @@ Content-Type: text/html; charset=UTF-8`} } -func TestRule(t *testing.T) { - content := `- name: nginx - matchers-condition: or - matchers: - - type: word - name: Jupyter - words: - - Jupyter Notebook - part: body -` - a := assert.New(t) - var rules []*Rule - err := yaml.Unmarshal([]byte(content), &rules) - a.Nil(err) - a.Equal(rules[0].Name, "nginx") -} - func TestAppFinger2(t *testing.T) { a := assert.New(t) appFinger := AppFinger{} @@ -229,3 +214,36 @@ func TestLocalFile(t *testing.T) { return } } + +func TestRegex2(t *testing.T) { + regex, _ := regexp.Compile(`/wp-content/plugins/([\w-]+)/(?:.*\?ver=([\d.]+))?`) + matched := regex.FindAllStringSubmatch(` +`, -1) + fmt.Printf("%v\n", matched) +} + +func TestMurmurhash(t *testing.T) { + assert.Equal(t, int32(851989093), mmh3([]byte("foo"))) +} + +func TestCharset(t *testing.T) { + req, err := http.Get("http://1.180.157.154:8087/login.jsp") + if err != nil { + return + } + //println(req.TransferEncoding) + body, err := io.ReadAll(req.Body) + if err != nil { + return + } + r, err := charset.NewReaderLabel("gb2312", strings.NewReader(string(body))) + if err != nil { + return + } + + decodedBody, err := io.ReadAll(r) + if err != nil { + return + } + println(string(decodedBody)) +} diff --git a/pkg/finger/redirect.go b/pkg/finger/redirect.go index a63d283..cea4d9d 100644 --- a/pkg/finger/redirect.go +++ b/pkg/finger/redirect.go @@ -16,7 +16,7 @@ func extractUri(n *html.Node) string { parts := strings.Split(attr.Val, ";") for _, part := range parts { if strings.Contains(strings.ToLower(part), "url=") { - redirectURL := strings.TrimPrefix(strings.TrimSpace(strings.Split(part, "=")[1]), "/") + redirectURL := strings.TrimSpace(strings.SplitN(part, "=", 2)[1]) return redirectURL } } @@ -35,7 +35,6 @@ func findRefresh(n *html.Node) string { return uri } - //return } } } @@ -125,6 +124,45 @@ func getExecRedirect(url string, jsCodes []string, onload string) string { } return result.String() } +func findAttribute(attrs []html.Attribute, key string) string { + for _, attr := range attrs { + if attr.Key == key { + return attr.Val + } + } + return "" +} +func extractCharset(htmlContent string) string { + reader := strings.NewReader(htmlContent) + doc, err := html.Parse(reader) + if err != nil { + return "UTF-8" + } + + var charset string + var traverse func(*html.Node) + traverse = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "meta" { + for _, attr := range n.Attr { + if attr.Key == "http-equiv" && strings.EqualFold(attr.Val, "Content-Type") { + contentAttr := findAttribute(n.Attr, "content") + charsetIndex := strings.Index(contentAttr, "charset=") + if charsetIndex != -1 { + charset = contentAttr[charsetIndex+len("charset="):] + break + } + } + } + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + traverse(c) + } + } + traverse(doc) + return strings.ToUpper(strings.TrimSpace(charset)) +} + func parseJavaScript(url string, htmlContent string) string { // 在这里解析JavaScript，提取跳转信息 // diff --git a/pkg/finger/tools.go b/pkg/finger/tools.go new file mode 100644 index 0000000..1732201 --- /dev/null +++ b/pkg/finger/tools.go @@ -0,0 +1,35 @@ +package finger + +import ( + "bytes" + "encoding/base64" + "github.com/spaolacci/murmur3" +) + +func InsertInto(s string, interval int, sep rune) string { + var buffer bytes.Buffer + before := interval - 1 + last := len(s) - 1 + for i, char := range s { + buffer.WriteRune(char) + if i%interval == before && i != last { + buffer.WriteRune(sep) + } + } + buffer.WriteRune(sep) + return buffer.String() +} +func mmh3(data []byte) int32 { + hash := murmur3.New32WithSeed(0) + _, err := hash.Write([]byte(base64Py(data))) + if err != nil { + return 0 + } + return int32(hash.Sum32()) +} + +func base64Py(data []byte) string { + // python encodes to base64 with lines of 76 bytes terminated by new line "\n" + stdBase64 := base64.StdEncoding.EncodeToString(data) + return InsertInto(stdBase64, 76, '\n') +} diff --git a/pkg/matchers/compile.go b/pkg/matchers/compile.go index 36fccd9..63be488 100644 --- a/pkg/matchers/compile.go +++ b/pkg/matchers/compile.go @@ -9,7 +9,6 @@ import ( // CompileMatchers performs the initial setup operation on a matcher func (matcher *Matcher) CompileMatchers() error { var ok bool - // Support hexadecimal encoding for matchers too. if matcher.Encoding == "hex" { for i, word := range matcher.Words {