diff --git a/cmd/go-portScan.go b/cmd/go-portScan.go index 309e7ce..aaf13da 100644 --- a/cmd/go-portScan.go +++ b/cmd/go-portScan.go @@ -32,6 +32,7 @@ var ( iL string devices bool dev string + httpx bool ) func parseFlag(c *cli.Context) { @@ -46,6 +47,7 @@ func parseFlag(c *cli.Context) { sT = c.Bool("sT") sV = c.Bool("sV") timeout = c.Int("timeout") + httpx = c.Bool("httpx") } func run(c *cli.Context) error { @@ -113,10 +115,32 @@ func run(c *cli.Context) error { single := make(chan struct{}) retChan := make(chan port.OpenIpPort, 65535) // port fingerprint + var httpxFile *os.File + var httpxFileLooker sync.Mutex + if httpx { + httpxFile, err = os.OpenFile("httpInfo.txt", os.O_APPEND|os.O_CREATE, 0644) + if err == nil { + defer httpxFile.Close() + } + } var wgPortIdentify sync.WaitGroup poolPortIdentify, _ := ants.NewPoolWithFunc(500, func(ipPort interface{}) { ret := ipPort.(port.OpenIpPort) - fmt.Printf("%s:%d %s\n", ret.Ip, ret.Port, fingerprint.PortIdentify("tcp", ret.Ip, ret.Port)) + if httpx { + _buf := fingerprint.ProbeHttpInfo(ret.Ip, ret.Port) + if _buf != nil { + buf := fmt.Sprintf("[HttpInfo]%s\n", _buf) + if httpxFile != nil { + httpxFileLooker.Lock() + httpxFile.WriteString(buf) + httpxFileLooker.Unlock() + } + fmt.Print(buf) + } + } else { + fmt.Printf("%s:%d %s\n", ret.Ip, ret.Port, fingerprint.PortIdentify("tcp", ret.Ip, ret.Port)) + } + wgPortIdentify.Done() }) defer poolPortIdentify.Release() @@ -128,11 +152,12 @@ func run(c *cli.Context) error { single <- struct{}{} return } - if sV { + if sV || httpx { // port fingerprint wgPortIdentify.Add(1) poolPortIdentify.Invoke(ret) - } else { + } + if !sV { fmt.Printf("%v:%d\n", ret.Ip, ret.Port) } default: @@ -310,6 +335,11 @@ func main() { Usage: "port service identify", Value: false, }, + &cli.BoolFlag{ + Name: "httpx", + Usage: "http server identify", + Value: false, + }, }, } diff --git a/core/port/fingerprint/encodings.go b/core/port/fingerprint/encodings.go new file mode 100644 index 0000000..579f64f --- /dev/null +++ b/core/port/fingerprint/encodings.go @@ -0,0 +1,88 @@ +package fingerprint + +import ( + "bytes" + "io/ioutil" + "net/http" + "strings" + + "github.com/projectdiscovery/stringsutil" + "golang.org/x/text/encoding/korean" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/transform" +) + +// Credits: https://gist.github.com/zhangbaohe/c691e1da5bbdc7f41ca5 + +// Decodegbk converts GBK to UTF-8 +func Decodegbk(s []byte) ([]byte, error) { + I := bytes.NewReader(s) + O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder()) + d, e := ioutil.ReadAll(O) + if e != nil { + return nil, e + } + return d, nil +} + +// Decodebig5 converts BIG5 to UTF-8 +func Decodebig5(s []byte) ([]byte, error) { + I := bytes.NewReader(s) + O := transform.NewReader(I, traditionalchinese.Big5.NewDecoder()) + d, e := ioutil.ReadAll(O) + if e != nil { + return nil, e + } + return d, nil +} + +// Encodebig5 converts UTF-8 to BIG5 +func Encodebig5(s []byte) ([]byte, error) { + I := bytes.NewReader(s) + O := transform.NewReader(I, traditionalchinese.Big5.NewEncoder()) + d, e := ioutil.ReadAll(O) + if e != nil { + return nil, e + } + return d, nil +} + +func DecodeKorean(s []byte) ([]byte, error) { + koreanDecoder := korean.EUCKR.NewDecoder() + return koreanDecoder.Bytes(s) +} + +// ExtractTitle from a response +func DecodeData(data []byte, headers http.Header) ([]byte, error) { + // Non UTF-8 + if contentTypes, ok := headers["Content-Type"]; ok { + contentType := strings.ToLower(strings.Join(contentTypes, ";")) + + switch { + case stringsutil.ContainsAny(contentType, "charset=gb2312", "charset=gbk"): + return Decodegbk([]byte(data)) + case stringsutil.ContainsAny(contentType, "euc-kr"): + return DecodeKorean(data) + } + + // Content-Type from head tag + var match = reContentType.FindSubmatch(data) + var mcontentType = "" + if len(match) != 0 { + for i, v := range match { + if string(v) != "" && i != 0 { + mcontentType = string(v) + } + } + mcontentType = strings.ToLower(mcontentType) + } + switch { + case stringsutil.ContainsAny(mcontentType, "gb2312", "gbk"): + return Decodegbk(data) + } + } + + // return as is + return data, nil +} diff --git a/core/port/fingerprint/http.go b/core/port/fingerprint/http.go new file mode 100644 index 0000000..170032e --- /dev/null +++ b/core/port/fingerprint/http.go @@ -0,0 +1,95 @@ +package fingerprint + +import ( + "compress/flate" + "compress/gzip" + "crypto/tls" + "errors" + "io" + "net" + "net/http" + "time" +) + +var ErrOverflow = errors.New("OverflowMax") + +type Options struct { +} + +func newHttpClient() *http.Client { + transport := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + DialContext: (&net.Dialer{ + Timeout: 5 * time.Second, + }).DialContext, + MaxIdleConnsPerHost: 1, + IdleConnTimeout: 100 * time.Millisecond, + TLSHandshakeTimeout: 3 * time.Second, + ExpectContinueTimeout: 3 * time.Second, + DisableKeepAlives: true, + ForceAttemptHTTP2: false, + } + + // proxy + //if options.ProxyUrl != "" { + // proxyUrl, err := url.Parse(options.ProxyUrl) + // if err != nil { + // log.Fatalln(err) + // } + // transport.Proxy = http.ProxyURL(proxyUrl) + //} + + return &http.Client{ + Timeout: 3 * time.Second, + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse /* 不进入重定向 */ + }, + } +} + +// getBody 识别响应Body的编码,读取body数据 +func getBody(resp *http.Response) (body []byte, err error) { + if resp.Body == nil || resp.Body == http.NoBody { + return + } + var reader io.Reader + switch resp.Header.Get("Content-Encoding") { + case "gzip": + reader, err = gzip.NewReader(resp.Body) + case "deflate": + reader = flate.NewReader(resp.Body) + //case "br": + // reader = brotli.NewReader(resp.Body) + default: + reader = resp.Body + } + if err == nil { + body, err = readMaxSize(reader, 300*1024) // Max Size 300kb + } + return +} + +// readMaxSize 读取io数据,限制最大读取尺寸 +func readMaxSize(r io.Reader, maxsize int) ([]byte, error) { + b := make([]byte, 0, 512) + for { + if len(b) >= maxsize { + return b, ErrOverflow + } + if len(b) == cap(b) { + // Add more capacity (let append pick how much). + b = append(b, 0)[:len(b)] + } + n, err := r.Read(b[len(b):cap(b)]) + b = b[:len(b)+n] + if err != nil { + if err == io.EOF { + err = nil + } + return b, err + } + } +} diff --git a/core/port/fingerprint/httpInfo.go b/core/port/fingerprint/httpInfo.go new file mode 100644 index 0000000..2aab11b --- /dev/null +++ b/core/port/fingerprint/httpInfo.go @@ -0,0 +1,108 @@ +package fingerprint + +import ( + "fmt" + "github.com/XinRoom/go-portScan/util" + "net" + "net/http" + "strings" +) + +// HttpInfo Http服务基础信息 +type HttpInfo struct { + StatusCode int // 状态码 + ContentLen int // 相应包大小 + Url string // Url + Location string // 302、301重定向路径 + Title string // 标题 + TlsCN string // tls使用者名称 + TlsDNS []string // tlsDNS列表 +} + +var httpsTopPort = []uint16{443, 4443, 1443, 8443} + +var httpClient *http.Client + +func (hi *HttpInfo) String() string { + if hi == nil { + return "" + } + var buf strings.Builder + buf.WriteString(fmt.Sprintf("Url:%s StatusCode:%d ContentLen:%d Title:%s ", hi.Url, hi.StatusCode, hi.ContentLen, hi.Title)) + if hi.Location != "" { + buf.WriteString("Location:" + hi.Location + " ") + } + if hi.TlsCN != "" { + buf.WriteString("TlsCN:" + hi.TlsCN + " ") + } + if len(hi.TlsDNS) > 0 { + buf.WriteString("TlsDNS:" + strings.Join(hi.TlsDNS, ",") + " ") + } + return buf.String() +} + +func ProbeHttpInfo(ip net.IP, _port uint16) *HttpInfo { + + if httpClient == nil { + httpClient = newHttpClient() + } + + var err error + var rewriteUrl string + var body []byte + var _body []byte + var resp *http.Response + var schemes []string + var httpInfo *HttpInfo + + if util.IsUint16InList(_port, httpsTopPort) { + schemes = []string{"https", "http"} + } else { + schemes = []string{"http", "https"} + } + + for _, scheme := range schemes { + req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s://%s:%d/", scheme, ip.String(), _port), http.NoBody) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") + req.Header.Set("Accept-Encoding", "gzip, deflate") + req.Close = true // disable keepalive + resp, err = httpClient.Do(req) + if err != nil { + continue + } + if resp.Body != http.NoBody && resp.Body != nil { + body, _ = getBody(resp) + _body, err = DecodeData(body, resp.Header) + if err == nil { + body = _body + } + if resp.ContentLength == -1 { + resp.ContentLength = int64(len(body)) + } + rewriteUrl2, _ := resp.Location() + if rewriteUrl2 != nil { + rewriteUrl = rewriteUrl2.String() + } else { + rewriteUrl = "" + } + location := GetLocation(body) + if rewriteUrl == "" && location != "" { + rewriteUrl = location + } + // + httpInfo = new(HttpInfo) + httpInfo.Url = resp.Request.URL.String() + httpInfo.StatusCode = resp.StatusCode + httpInfo.ContentLen = int(resp.ContentLength) + httpInfo.Location = rewriteUrl + httpInfo.Title = ExtractTitle(body) + if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 { + httpInfo.TlsCN = resp.TLS.PeerCertificates[0].Subject.CommonName + httpInfo.TlsDNS = resp.TLS.PeerCertificates[0].DNSNames + } + break + } + } + + return httpInfo +} diff --git a/core/port/fingerprint/httpInfo_test.go b/core/port/fingerprint/httpInfo_test.go new file mode 100644 index 0000000..bb34185 --- /dev/null +++ b/core/port/fingerprint/httpInfo_test.go @@ -0,0 +1,10 @@ +package fingerprint + +import ( + "net" + "testing" +) + +func TestName(t *testing.T) { + t.Log(ProbeHttpInfo(net.ParseIP("14.215.177.39"), 443)) +} diff --git a/core/port/fingerprint/title.go b/core/port/fingerprint/title.go new file mode 100644 index 0000000..0dddad7 --- /dev/null +++ b/core/port/fingerprint/title.go @@ -0,0 +1,96 @@ +package fingerprint + +import ( + "bytes" + "fmt" + "golang.org/x/net/html" + "io" + "regexp" + "strings" +) + +var ( + cutset = "\n\t\v\f\r" + reTitle = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`) + reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`) + reRefresh = regexp.MustCompile(`(?im)\s*content=['"]\d;url=(.*?)['"]`) + reReplace = regexp.MustCompile(`(?im)window\.location\.replace\(['"](.*?)['"]\)`) +) + +// ExtractTitle from a response +func ExtractTitle(body []byte) (title string) { + // Try to parse the DOM + titleDom, err := getTitleWithDom(body) + // In case of error fallback to regex + if err != nil { + for _, match := range reTitle.FindAllString(string(body), -1) { + title = match + break + } + } else { + title = renderNode(titleDom) + } + + title = html.UnescapeString(trimTitleTags(title)) + + // remove unwanted chars + title = strings.TrimSpace(strings.Trim(title, cutset)) + title = strings.ReplaceAll(title, "\n", "") + title = strings.ReplaceAll(title, "\r", "") + + return title +} + +func getTitleWithDom(body []byte) (*html.Node, error) { + var title *html.Node + var crawler func(*html.Node) + crawler = func(node *html.Node) { + if node.Type == html.ElementNode && node.Data == "title" { + title = node + return + } + for child := node.FirstChild; child != nil && title == nil; child = child.NextSibling { + crawler(child) + } + } + htmlDoc, err := html.Parse(bytes.NewReader(body)) + if err != nil { + return nil, err + } + crawler(htmlDoc) + if title != nil { + return title, nil + } + return nil, fmt.Errorf("title not found") +} + +func renderNode(n *html.Node) string { + var buf bytes.Buffer + w := io.Writer(&buf) + html.Render(w, n) //nolint + return buf.String() +} + +func trimTitleTags(title string) string { + // trim * + titleBegin := strings.Index(title, ">") + titleEnd := strings.Index(title, "