Add -httpx, to probe HTTP service information

cmd/go-portScan.go

@@ -32,6 +32,7 @@ var (
 	iL      string
 	devices bool
 	dev     string
+	httpx   bool
 )
 
 func parseFlag(c *cli.Context) {
@@ -46,6 +47,7 @@ func parseFlag(c *cli.Context) {
 	sT = c.Bool("sT")
 	sV = c.Bool("sV")
 	timeout = c.Int("timeout")
+	httpx = c.Bool("httpx")
 }
 
 func run(c *cli.Context) error {
@@ -113,10 +115,32 @@ func run(c *cli.Context) error {
 	single := make(chan struct{})
 	retChan := make(chan port.OpenIpPort, 65535)
 	// port fingerprint
+	var httpxFile *os.File
+	var httpxFileLooker sync.Mutex
+	if httpx {
+		httpxFile, err = os.OpenFile("httpInfo.txt", os.O_APPEND|os.O_CREATE, 0644)
+		if err == nil {
+			defer httpxFile.Close()
+		}
+	}
 	var wgPortIdentify sync.WaitGroup
 	poolPortIdentify, _ := ants.NewPoolWithFunc(500, func(ipPort interface{}) {
 		ret := ipPort.(port.OpenIpPort)
-		fmt.Printf("%s:%d %s\n", ret.Ip, ret.Port, fingerprint.PortIdentify("tcp", ret.Ip, ret.Port))
+		if httpx {
+			_buf := fingerprint.ProbeHttpInfo(ret.Ip, ret.Port)
+			if _buf != nil {
+				buf := fmt.Sprintf("[HttpInfo]%s\n", _buf)
+				if httpxFile != nil {
+					httpxFileLooker.Lock()
+					httpxFile.WriteString(buf)
+					httpxFileLooker.Unlock()
+				}
+				fmt.Print(buf)
+			}
+		} else {
+			fmt.Printf("%s:%d %s\n", ret.Ip, ret.Port, fingerprint.PortIdentify("tcp", ret.Ip, ret.Port))
+		}
+
 		wgPortIdentify.Done()
 	})
 	defer poolPortIdentify.Release()
@@ -128,11 +152,12 @@ func run(c *cli.Context) error {
 					single <- struct{}{}
 					return
 				}
-				if sV {
+				if sV || httpx {
 					// port fingerprint
 					wgPortIdentify.Add(1)
 					poolPortIdentify.Invoke(ret)
-				} else {
+				}
+				if !sV {
 					fmt.Printf("%v:%d\n", ret.Ip, ret.Port)
 				}
 			default:
@@ -310,6 +335,11 @@ func main() {
 				Usage: "port service identify",
 				Value: false,
 			},
+			&cli.BoolFlag{
+				Name:  "httpx",
+				Usage: "http server identify",
+				Value: false,
+			},
 		},
 	}
 

core/port/fingerprint/encodings.go

@@ -0,0 +1,88 @@
+package fingerprint
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/projectdiscovery/stringsutil"
+	"golang.org/x/text/encoding/korean"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/encoding/traditionalchinese"
+	"golang.org/x/text/transform"
+)
+
+// Credits: https://gist.github.com/zhangbaohe/c691e1da5bbdc7f41ca5
+
+// Decodegbk converts GBK to UTF-8
+func Decodegbk(s []byte) ([]byte, error) {
+	I := bytes.NewReader(s)
+	O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder())
+	d, e := ioutil.ReadAll(O)
+	if e != nil {
+		return nil, e
+	}
+	return d, nil
+}
+
+// Decodebig5 converts BIG5 to UTF-8
+func Decodebig5(s []byte) ([]byte, error) {
+	I := bytes.NewReader(s)
+	O := transform.NewReader(I, traditionalchinese.Big5.NewDecoder())
+	d, e := ioutil.ReadAll(O)
+	if e != nil {
+		return nil, e
+	}
+	return d, nil
+}
+
+// Encodebig5 converts UTF-8 to BIG5
+func Encodebig5(s []byte) ([]byte, error) {
+	I := bytes.NewReader(s)
+	O := transform.NewReader(I, traditionalchinese.Big5.NewEncoder())
+	d, e := ioutil.ReadAll(O)
+	if e != nil {
+		return nil, e
+	}
+	return d, nil
+}
+
+func DecodeKorean(s []byte) ([]byte, error) {
+	koreanDecoder := korean.EUCKR.NewDecoder()
+	return koreanDecoder.Bytes(s)
+}
+
+// ExtractTitle from a response
+func DecodeData(data []byte, headers http.Header) ([]byte, error) {
+	// Non UTF-8
+	if contentTypes, ok := headers["Content-Type"]; ok {
+		contentType := strings.ToLower(strings.Join(contentTypes, ";"))
+
+		switch {
+		case stringsutil.ContainsAny(contentType, "charset=gb2312", "charset=gbk"):
+			return Decodegbk([]byte(data))
+		case stringsutil.ContainsAny(contentType, "euc-kr"):
+			return DecodeKorean(data)
+		}
+
+		// Content-Type from head tag
+		var match = reContentType.FindSubmatch(data)
+		var mcontentType = ""
+		if len(match) != 0 {
+			for i, v := range match {
+				if string(v) != "" && i != 0 {
+					mcontentType = string(v)
+				}
+			}
+			mcontentType = strings.ToLower(mcontentType)
+		}
+		switch {
+		case stringsutil.ContainsAny(mcontentType, "gb2312", "gbk"):
+			return Decodegbk(data)
+		}
+	}
+
+	// return as is
+	return data, nil
+}

core/port/fingerprint/http.go

@@ -0,0 +1,95 @@
+package fingerprint
+
+import (
+	"compress/flate"
+	"compress/gzip"
+	"crypto/tls"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"time"
+)
+
+var ErrOverflow = errors.New("OverflowMax")
+
+type Options struct {
+}
+
+func newHttpClient() *http.Client {
+	transport := &http.Transport{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: true,
+		},
+		DialContext: (&net.Dialer{
+			Timeout: 5 * time.Second,
+		}).DialContext,
+		MaxIdleConnsPerHost:   1,
+		IdleConnTimeout:       100 * time.Millisecond,
+		TLSHandshakeTimeout:   3 * time.Second,
+		ExpectContinueTimeout: 3 * time.Second,
+		DisableKeepAlives:     true,
+		ForceAttemptHTTP2:     false,
+	}
+
+	// proxy
+	//if options.ProxyUrl != "" {
+	//	proxyUrl, err := url.Parse(options.ProxyUrl)
+	//	if err != nil {
+	//		log.Fatalln(err)
+	//	}
+	//	transport.Proxy = http.ProxyURL(proxyUrl)
+	//}
+
+	return &http.Client{
+		Timeout:   3 * time.Second,
+		Transport: transport,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse /* 不进入重定向 */
+		},
+	}
+}
+
+// getBody 识别响应Body的编码，读取body数据
+func getBody(resp *http.Response) (body []byte, err error) {
+	if resp.Body == nil || resp.Body == http.NoBody {
+		return
+	}
+	var reader io.Reader
+	switch resp.Header.Get("Content-Encoding") {
+	case "gzip":
+		reader, err = gzip.NewReader(resp.Body)
+	case "deflate":
+		reader = flate.NewReader(resp.Body)
+	//case "br":
+	//	reader = brotli.NewReader(resp.Body)
+	default:
+		reader = resp.Body
+	}
+	if err == nil {
+		body, err = readMaxSize(reader, 300*1024) // Max Size 300kb
+	}
+	return
+}
+
+// readMaxSize 读取io数据，限制最大读取尺寸
+func readMaxSize(r io.Reader, maxsize int) ([]byte, error) {
+	b := make([]byte, 0, 512)
+	for {
+		if len(b) >= maxsize {
+			return b, ErrOverflow
+		}
+		if len(b) == cap(b) {
+			// Add more capacity (let append pick how much).
+			b = append(b, 0)[:len(b)]
+		}
+		n, err := r.Read(b[len(b):cap(b)])
+		b = b[:len(b)+n]
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			return b, err
+		}
+	}
+}

core/port/fingerprint/httpInfo.go

@@ -0,0 +1,108 @@
+package fingerprint
+
+import (
+	"fmt"
+	"github.com/XinRoom/go-portScan/util"
+	"net"
+	"net/http"
+	"strings"
+)
+
+// HttpInfo Http服务基础信息
+type HttpInfo struct {
+	StatusCode int      // 状态码
+	ContentLen int      // 相应包大小
+	Url        string   // Url
+	Location   string   // 302、301重定向路径
+	Title      string   // 标题
+	TlsCN      string   // tls使用者名称
+	TlsDNS     []string // tlsDNS列表
+}
+
+var httpsTopPort = []uint16{443, 4443, 1443, 8443}
+
+var httpClient *http.Client
+
+func (hi *HttpInfo) String() string {
+	if hi == nil {
+		return ""
+	}
+	var buf strings.Builder
+	buf.WriteString(fmt.Sprintf("Url:%s StatusCode:%d ContentLen:%d Title:%s ", hi.Url, hi.StatusCode, hi.ContentLen, hi.Title))
+	if hi.Location != "" {
+		buf.WriteString("Location:" + hi.Location + " ")
+	}
+	if hi.TlsCN != "" {
+		buf.WriteString("TlsCN:" + hi.TlsCN + " ")
+	}
+	if len(hi.TlsDNS) > 0 {
+		buf.WriteString("TlsDNS:" + strings.Join(hi.TlsDNS, ",") + " ")
+	}
+	return buf.String()
+}
+
+func ProbeHttpInfo(ip net.IP, _port uint16) *HttpInfo {
+
+	if httpClient == nil {
+		httpClient = newHttpClient()
+	}
+
+	var err error
+	var rewriteUrl string
+	var body []byte
+	var _body []byte
+	var resp *http.Response
+	var schemes []string
+	var httpInfo *HttpInfo
+
+	if util.IsUint16InList(_port, httpsTopPort) {
+		schemes = []string{"https", "http"}
+	} else {
+		schemes = []string{"http", "https"}
+	}
+
+	for _, scheme := range schemes {
+		req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s://%s:%d/", scheme, ip.String(), _port), http.NoBody)
+		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
+		req.Header.Set("Accept-Encoding", "gzip, deflate")
+		req.Close = true // disable keepalive
+		resp, err = httpClient.Do(req)
+		if err != nil {
+			continue
+		}
+		if resp.Body != http.NoBody && resp.Body != nil {
+			body, _ = getBody(resp)
+			_body, err = DecodeData(body, resp.Header)
+			if err == nil {
+				body = _body
+			}
+			if resp.ContentLength == -1 {
+				resp.ContentLength = int64(len(body))
+			}
+			rewriteUrl2, _ := resp.Location()
+			if rewriteUrl2 != nil {
+				rewriteUrl = rewriteUrl2.String()
+			} else {
+				rewriteUrl = ""
+			}
+			location := GetLocation(body)
+			if rewriteUrl == "" && location != "" {
+				rewriteUrl = location
+			}
+			//
+			httpInfo = new(HttpInfo)
+			httpInfo.Url = resp.Request.URL.String()
+			httpInfo.StatusCode = resp.StatusCode
+			httpInfo.ContentLen = int(resp.ContentLength)
+			httpInfo.Location = rewriteUrl
+			httpInfo.Title = ExtractTitle(body)
+			if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 {
+				httpInfo.TlsCN = resp.TLS.PeerCertificates[0].Subject.CommonName
+				httpInfo.TlsDNS = resp.TLS.PeerCertificates[0].DNSNames
+			}
+			break
+		}
+	}
+
+	return httpInfo
+}

core/port/fingerprint/httpInfo_test.go

@@ -0,0 +1,10 @@
+package fingerprint
+
+import (
+	"net"
+	"testing"
+)
+
+func TestName(t *testing.T) {
+	t.Log(ProbeHttpInfo(net.ParseIP("14.215.177.39"), 443))
+}