diff --git a/README.md b/README.md index a4622530..03e006cd 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ PROBES: -ip display host ip -cname display host cname -asn display host asn information - -cdn display cdn/waf in use + -cdn display cdn/waf in use (default true) -probe display probe status HEADLESS: @@ -139,7 +139,7 @@ MATCHERS: EXTRACTOR: -er, -extract-regex string[] display response content with matched regex - -ep, -extract-preset string[] display response content matched by a pre-defined regex (url,ipv4,mail) + -ep, -extract-preset string[] display response content matched by a pre-defined regex (mail,url,ipv4) FILTERS: -fc, -filter-code string filter response with specified status code (-fc 403,401) @@ -190,9 +190,11 @@ OUTPUT: -include-chain include redirect http chain in JSON output (-json only) -store-chain include http redirect chain in responses (-sr only) -svrc, -store-vision-recon-cluster include visual recon clusters (-ss and -sr only) + -pr, -protocol string protocol to use (unknown, http11) CONFIGURATIONS: -config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml) + -auth configure projectdiscovery cloud (pdcp) api key (default true) -r, -resolvers string[] list of custom resolver (file or comma separated) -allow string[] allowed list of IP/CIDR's to process (file or comma separated) -deny string[] denied list of IP/CIDR's to process (file or comma separated) diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go index 4a06daaf..059fa411 100644 --- a/cmd/integration-test/library.go +++ b/cmd/integration-test/library.go @@ -74,7 +74,7 @@ func (h *httpxLibraryWithStream) Execute() error { RateLimit: 150, Retries: 2, Timeout: 10, - TechDetect: "true", + TechDetect: true, Stream: true, SkipDedupe: true, OnResult: func(r runner.Result) { diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 0a322079..8720d20c 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "net/url" + "os" "strconv" "strings" "time" @@ -152,6 +153,12 @@ func New(options *Options) (*HTTPX, error) { DisableKeepAlives: true, } + if httpx.Options.Protocol == "http11" { + // disable http2 + os.Setenv("GODEBUG", "http2client=0") + transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} + } + if httpx.Options.SniName != "" { transport.TLSClientConfig.ServerName = httpx.Options.SniName } @@ -288,10 +295,12 @@ get_response: // fill metrics resp.StatusCode = httpresp.StatusCode - // number of words - resp.Words = len(strings.Split(respbodystr, " ")) - // number of lines - resp.Lines = len(strings.Split(respbodystr, "\n")) + if respbodystr != "" { + // number of words + resp.Words = len(strings.Split(respbodystr, " ")) + // number of lines + resp.Lines = len(strings.Split(strings.TrimSpace(respbodystr), "\n")) + } if !h.Options.Unsafe && h.Options.TLSGrab { if h.Options.ZTLS { diff --git a/common/httpx/option.go b/common/httpx/option.go index 349966ab..7cb29c34 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -45,6 +45,7 @@ type Options struct { SniName string TlsImpersonate bool NetworkPolicy *networkpolicy.NetworkPolicy + Protocol Proto } // DefaultOptions contains the default options diff --git a/common/httpx/proto.go b/common/httpx/proto.go new file mode 100644 index 00000000..30365298 --- /dev/null +++ b/common/httpx/proto.go @@ -0,0 +1,10 @@ +package httpx + +type Proto string + +const ( + UNKNOWN Proto = "" + HTTP11 Proto = "http11" + HTTP2 Proto = "http2" + HTTP3 Proto = "http3" +) diff --git a/common/httpx/title.go b/common/httpx/title.go index 1b59b1e7..c2766af2 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -9,12 +9,22 @@ import ( stringsutil "github.com/projectdiscovery/utils/strings" "golang.org/x/net/html" + "slices" ) var ( - cutset = "\n\t\v\f\r" - reTitle = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`) - reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`) + cutset = "\n\t\v\f\r" + reTitle = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`) + reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`) + supportedTitleMimeTypes = []string{ + "text/html", + "application/xhtml+xml", + "application/xml", + "application/rss+xml", + "application/atom+xml", + "application/xhtml+xml", + "application/vnd.wap.xhtml+xml", + } ) // ExtractTitle from a response @@ -40,6 +50,10 @@ func ExtractTitle(r *Response) (title string) { return title } +func CanHaveTitleTag(mimeType string) bool { + return slices.Contains(supportedTitleMimeTypes, mimeType) +} + func getTitleWithDom(r *Response) (*html.Node, error) { var title *html.Node var crawler func(*html.Node) diff --git a/go.mod b/go.mod index 08156158..0d61591d 100644 --- a/go.mod +++ b/go.mod @@ -23,22 +23,22 @@ require ( github.com/projectdiscovery/asnmap v1.1.0 github.com/projectdiscovery/cdncheck v1.0.9 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/dsl v0.0.46 - github.com/projectdiscovery/fastdialer v0.0.61 + github.com/projectdiscovery/dsl v0.0.56 + github.com/projectdiscovery/fastdialer v0.0.71 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.41 + github.com/projectdiscovery/goflags v0.1.52 github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/hmap v0.0.41 - github.com/projectdiscovery/mapcidr v1.1.16 + github.com/projectdiscovery/hmap v0.0.42 + github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.0.8 - github.com/projectdiscovery/ratelimit v0.0.27 - github.com/projectdiscovery/rawhttp v0.1.39 - github.com/projectdiscovery/retryablehttp-go v1.0.49 + github.com/projectdiscovery/ratelimit v0.0.40 + github.com/projectdiscovery/rawhttp v0.1.49 + github.com/projectdiscovery/retryablehttp-go v1.0.59 github.com/projectdiscovery/tlsx v1.1.6 - github.com/projectdiscovery/useragent v0.0.38 - github.com/projectdiscovery/utils v0.0.82 - github.com/projectdiscovery/wappalyzergo v0.0.112 + github.com/projectdiscovery/useragent v0.0.48 + github.com/projectdiscovery/utils v0.0.92 + github.com/projectdiscovery/wappalyzergo v0.1.0 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 @@ -47,9 +47,9 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 golang.org/x/exp v0.0.0-20230810033253-352e893a4cad - golang.org/x/net v0.23.0 - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 + golang.org/x/net v0.25.0 + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 ) require ( @@ -81,6 +81,7 @@ require ( github.com/google/certificate-transparency-go v1.1.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -109,7 +110,7 @@ require ( github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/retryabledns v1.0.57 // indirect + github.com/projectdiscovery/retryabledns v1.0.59 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/quic-go/quic-go v0.42.0 // indirect github.com/refraction-networking/utls v1.5.4 // indirect @@ -128,10 +129,9 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect - github.com/tklauser/go-sysconf v0.3.11 // indirect - github.com/tklauser/numcpus v0.6.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/ulikunitz/xz v0.5.11 // indirect - github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect @@ -142,12 +142,14 @@ require ( github.com/ysmood/leakless v0.8.0 // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/term v0.18.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/term v0.20.0 // indirect golang.org/x/tools v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum index 021ab798..1f0fc4c2 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,8 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= @@ -218,48 +220,48 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4= github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.0.46 h1:zBNNzSBA1aakGY44w6KhnjJQx/zO+oW2Wx7TR8xZm/A= -github.com/projectdiscovery/dsl v0.0.46/go.mod h1:eoZEJFCIT5emI00xj8HTQwHz4YwwGAD+grL3G7CDlfs= -github.com/projectdiscovery/fastdialer v0.0.61 h1:z5OzP9lRbn6fSIezgReKC3hkzRh+YX41ST9OgkVEm/s= -github.com/projectdiscovery/fastdialer v0.0.61/go.mod h1:FyxJ0m1MwB69nLmdXYqK32f3a0Pf+5YpC8wBY73baiE= +github.com/projectdiscovery/dsl v0.0.56 h1:iVFIfDdGXkrXTh5sf6hRaxqTTEYiv/LnNjoOHJIfHiY= +github.com/projectdiscovery/dsl v0.0.56/go.mod h1:3UBjPvtiy8c5E8oAUrKobMKjb3kEwQuGB5tqIN964Og= +github.com/projectdiscovery/fastdialer v0.0.71 h1:96j6Y65hDPZ9AzlYpp95hvIH5Yx/0OE2UTx+frWfnm4= +github.com/projectdiscovery/fastdialer v0.0.71/go.mod h1:b/oPPVSoLLD2N4W2/HrXbhQbyJVXqRw8CK1lenCUk64= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.41 h1:rw6NM7ShB7GGQZr5XaM/gkuyoLY3zpf0gb56f9fLNQo= -github.com/projectdiscovery/goflags v0.1.41/go.mod h1:AlZUOcABJTBDzkgquUS00qWTTH80lxzpfHQrYlIzVkU= +github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU= +github.com/projectdiscovery/goflags v0.1.52/go.mod h1:tcBQ0EVGP4Wafza7gx57ZktkGxyfdLn+eQWUUQrV84c= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE= -github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0= +github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM= +github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo= -github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q= +github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= +github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= -github.com/projectdiscovery/ratelimit v0.0.27 h1:McTgnl8CtaEPmPtb9JG7EfgaQ1Rhu0pHa0Kf5Kld6Xs= -github.com/projectdiscovery/ratelimit v0.0.27/go.mod h1:5suG3x1d5+UV4xe2RBE/QCvQkz8CaxPvdwztjab3GzM= -github.com/projectdiscovery/rawhttp v0.1.39 h1:UYtDx5J9wqn7OFte66viag7cL757z1HU94jCya3V70s= -github.com/projectdiscovery/rawhttp v0.1.39/go.mod h1:gn0b2aa2DoxVx7GDzAfoBZPjisDmsXv6h/znGjOwChw= -github.com/projectdiscovery/retryabledns v1.0.57 h1:+DOL9xYSIx74FRrOIKKHVp5R9ci53xHQN3jnncWVds4= -github.com/projectdiscovery/retryabledns v1.0.57/go.mod h1:qIigOcmO9d0Ce/z6mHzLl0Aiz2WJcNk2gUGhRcCQ1k4= -github.com/projectdiscovery/retryablehttp-go v1.0.49 h1:mvlvl2kTN+ctpDIRlusVWui7eyFlElBoKTr8crS7yvY= -github.com/projectdiscovery/retryablehttp-go v1.0.49/go.mod h1:VaJ7Au+1LP8C2u0qmx4NN1IdAxxkhoXpIcc9LAQzFo4= +github.com/projectdiscovery/ratelimit v0.0.40 h1:SURkc7+ezUeGNt9PEQZOtd7nh2o2jKzmqcehrrf6DMU= +github.com/projectdiscovery/ratelimit v0.0.40/go.mod h1:3Z1rK0bs4BSjdYNvIlbmZR1H8Z5MsXRz+QRCXGIB0HE= +github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM= +github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94= +github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= +github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= +github.com/projectdiscovery/retryablehttp-go v1.0.59 h1:jyeBhFG4iKpJPurlQAlVLgCImTMjwUBaJmZoz90SnVw= +github.com/projectdiscovery/retryablehttp-go v1.0.59/go.mod h1:/L7lD+gOM/44r1mWgwf3knG9Q6gUbNEtbd1DNEq1+bE= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= -github.com/projectdiscovery/useragent v0.0.38 h1:E9NB3tG2bOHizF4+1o/wlC3qOndqlQMmhjkriK2Ms/E= -github.com/projectdiscovery/useragent v0.0.38/go.mod h1:uWkTlXQRG8B31OkCZCDOZlFeUqlJhiDKlrONjCDJojY= -github.com/projectdiscovery/utils v0.0.82 h1:U//02floCSFxJluN7MP+rJSwI4Px7o454JL7ukERArI= -github.com/projectdiscovery/utils v0.0.82/go.mod h1:AbmIvy0TTlsfXxPDEMaNPVrxmqDmYiCnbGqh0TTthE4= -github.com/projectdiscovery/wappalyzergo v0.0.112 h1:QPpp5jmj1lqLd5mFdFKQ9VvcYhQNqyU9Mr+IB0US2zA= -github.com/projectdiscovery/wappalyzergo v0.0.112/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= +github.com/projectdiscovery/useragent v0.0.48 h1:ITygElwcY9FlOt0F65kcW/oAALNr1nQOtO3kR9lRzaY= +github.com/projectdiscovery/useragent v0.0.48/go.mod h1:ahQMoWlVNFVQxjHOKqOPHJJQ7Ovz1zJZuJbBsAwIcOw= +github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY= +github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg= +github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0= +github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= @@ -329,16 +331,16 @@ github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= -github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= -github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= -github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db h1:/WcxBne+5CbtbgWd/sV2wbravmr4sT7y52ifQaCgoLs= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= @@ -367,8 +369,11 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= +github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= @@ -396,8 +401,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU= golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -423,8 +428,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= @@ -434,8 +439,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -461,17 +466,19 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -481,8 +488,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= diff --git a/runner/options.go b/runner/options.go index 6c575d82..8a94fceb 100644 --- a/runner/options.go +++ b/runner/options.go @@ -22,11 +22,13 @@ import ( "github.com/projectdiscovery/httpx/common/customlist" customport "github.com/projectdiscovery/httpx/common/customports" fileutilz "github.com/projectdiscovery/httpx/common/fileutil" + "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/httpx/common/slice" "github.com/projectdiscovery/httpx/common/stringz" "github.com/projectdiscovery/utils/auth/pdcp" "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" + stringsutil "github.com/projectdiscovery/utils/strings" updateutils "github.com/projectdiscovery/utils/update" ) @@ -74,7 +76,7 @@ type ScanOptions struct { PreferHTTPS bool NoFallback bool NoFallbackScheme bool - TechDetect string + TechDetect bool StoreChain bool StoreVisionReconClusters bool MaxResponseBodySizeToSave int @@ -229,7 +231,7 @@ type Options struct { OutputResponseTime bool NoFallback bool NoFallbackScheme bool - TechDetect string + TechDetect bool TLSGrab bool protocol string ShowStatistics bool @@ -297,6 +299,7 @@ type Options struct { ScreenshotTimeout int // HeadlessOptionalArguments specifies optional arguments to pass to Chrome HeadlessOptionalArguments goflags.StringSlice + Protocol string } // ParseOptions parses the command line options for application @@ -327,7 +330,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.ExtractTitle, "title", false, "display page title"), flagSet.DynamicVarP(&options.ResponseBodyPreviewSize, "body-preview", "bp", 100, "display first N characters of response body"), flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"), - flagSet.DynamicVarP(&options.TechDetect, "tech-detect", "td", "true", "display technology in use based on wappalyzer dataset"), + flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"), flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"), flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"), flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"), @@ -417,6 +420,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"), flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"), flagSet.BoolVarP(&options.StoreVisionReconClusters, "store-vision-recon-cluster", "svrc", false, "include visual recon clusters (-ss and -sr only)"), + flagSet.StringVarP(&options.Protocol, "protocol", "pr", "", "protocol to use (unknown, http11)"), ) flagSet.CreateGroup("configs", "Configurations", @@ -668,6 +672,10 @@ func (options *Options) ValidateOptions() error { options.OutputCDN = "true" } + if !stringsutil.EqualFoldAny(options.Protocol, string(httpx.UNKNOWN), string(httpx.HTTP11)) { + return fmt.Errorf("invalid protocol: %s", options.Protocol) + } + return nil } diff --git a/runner/runner.go b/runner/runner.go index 1f4f6640..388b73e1 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -106,7 +106,7 @@ func New(options *Options) (*Runner, error) { options: options, } var err error - if options.TechDetect != "false" { + if options.TechDetect { runner.wappalyzer, err = wappalyzer.New() } if err != nil { @@ -1590,8 +1590,12 @@ retry: builder.WriteRune(']') } - title := httpx.ExtractTitle(resp) - if scanopts.OutputTitle { + var title string + if httpx.CanHaveTitleTag(resp.GetHeaderPart("Content-Type", ";")) { + title = httpx.ExtractTitle(resp) + } + + if scanopts.OutputTitle && title != "" { builder.WriteString(" [") if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Cyan(title).String()) @@ -1745,9 +1749,19 @@ retry: if err != nil { onlyHost = URL.Host } - ips, cnames, err := getDNSData(hp, onlyHost) + allIps, cnames, err := getDNSData(hp, onlyHost) if err != nil { - ips = append(ips, ip) + allIps = append(allIps, ip) + } + + var ips4, ips6 []string + for _, ip := range allIps { + switch { + case iputil.IsIPv4(ip): + ips4 = append(ips4, ip) + case iputil.IsIPv6(ip): + ips6 = append(ips6, ip) + } } if scanopts.OutputCName && len(cnames) > 0 { @@ -1764,27 +1778,16 @@ retry: builder.WriteString(fmt.Sprintf(" [%s]", resp.Duration)) } + technologyDetails := make(map[string]wappalyzer.AppInfo) var technologies []string - if scanopts.TechDetect != "false" { - matches := r.wappalyzer.Fingerprint(resp.Headers, resp.Data) - for match := range matches { + if scanopts.TechDetect { + matches := r.wappalyzer.FingerprintWithInfo(resp.Headers, resp.Data) + for match, data := range matches { technologies = append(technologies, match) + technologyDetails[match] = data } } - if scanopts.TechDetect == "true" && len(technologies) > 0 { - sort.Strings(technologies) - technologies := strings.Join(technologies, ",") - - builder.WriteString(" [") - if !scanopts.OutputWithNoColor { - builder.WriteString(aurora.Magenta(technologies).String()) - } else { - builder.WriteString(technologies) - } - builder.WriteRune(']') - } - var extractRegex []string // extract regex var extractResult = map[string][]string{} @@ -1998,6 +2001,15 @@ retry: gologger.Warning().Msgf("%v: %s", err, fullURL) } + // As we now have headless body, we can also use it for detecting + // more technologies in the response. This is a quick trick to get + // more detected technologies. + moreMatches := r.wappalyzer.FingerprintWithInfo(resp.Headers, []byte(headlessBody)) + for match, data := range moreMatches { + technologies = append(technologies, match) + technologyDetails[match] = data + } + technologies = sliceutil.Dedupe(technologies) } if scanopts.NoScreenshotBytes { screenshotBytes = []byte{} @@ -2007,6 +2019,21 @@ retry: } } + if scanopts.TechDetect && len(technologies) > 0 { + sort.Strings(technologies) + technologies := strings.Join(technologies, ",") + + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(technologies).String()) + } else { + builder.WriteString(technologies) + } + builder.WriteRune(']') + } + + // We now have headless body. We can use it for tech detection + result := Result{ Timestamp: time.Now(), Request: request, @@ -2037,7 +2064,8 @@ retry: HTTP2: http2, Method: method, Host: ip, - A: ips, + A: ips4, + AAAA: ips6, CNAMEs: cnames, CDN: isCDN, CDNName: cdnName, @@ -2062,6 +2090,7 @@ retry: "PageType": r.errorPageClassifier.Classify(respData), "pHash": pHash, }, + TechnologyDetails: technologyDetails, } return result } @@ -2106,7 +2135,7 @@ func (r *Runner) handleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, // search in the response of the requested path for element and rel shortcut/mask/apple-touch icon // link with .ico extension (which will be prioritized if available) // if not, any of link from other icons can be requested - potentialURLs, err := extractPotentialFavIconsURLs(req, currentResp) + potentialURLs, err := extractPotentialFavIconsURLs(currentResp) if err != nil { return "", "", err } @@ -2148,7 +2177,7 @@ func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, error) { return fmt.Sprintf("%d", hashNum), nil } -func extractPotentialFavIconsURLs(req *retryablehttp.Request, resp *httpx.Response) ([]string, error) { +func extractPotentialFavIconsURLs(resp *httpx.Response) ([]string, error) { var potentialURLs []string document, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data)) if err != nil { diff --git a/runner/runner_test.go b/runner/runner_test.go index f96d3f9b..08698b20 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -89,6 +89,10 @@ func TestRunner_cidr_targets(t *testing.T) { } func TestRunner_asn_targets(t *testing.T) { + if os.Getenv("PDCP_API_KEY") == "" { + return + } + options := &Options{} r, err := New(options) require.Nil(t, err, "could not create httpx runner") @@ -131,10 +135,12 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) { got = r.countTargetFromRawTarget(input) require.Equal(t, expected, got, "got wrong output") - input = "AS14421" - expected = 256 - got = r.countTargetFromRawTarget(input) - require.Equal(t, expected, got, "got wrong output") + if os.Getenv("PDCP_API_KEY") != "" { + input = "AS14421" + expected = 256 + got = r.countTargetFromRawTarget(input) + require.Equal(t, expected, got, "got wrong output") + } input = "173.0.84.0/24" expected = 256 diff --git a/runner/types.go b/runner/types.go index 37b98c0d..ca7f7759 100644 --- a/runner/types.go +++ b/runner/types.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" mapsutil "github.com/projectdiscovery/utils/maps" + wappalyzer "github.com/projectdiscovery/wappalyzergo" "github.com/projectdiscovery/httpx/common/httpx" ) @@ -66,6 +67,7 @@ type Result struct { Jarm string `json:"jarm,omitempty" csv:"jarm"` ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` A []string `json:"a,omitempty" csv:"a"` + AAAA []string `json:"aaaa,omitempty" csv:"aaaa"` CNAMEs []string `json:"cname,omitempty" csv:"cname"` Technologies []string `json:"tech,omitempty" csv:"tech"` Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` @@ -73,7 +75,7 @@ type Result struct { Words int `json:"words" csv:"words"` Lines int `json:"lines" csv:"lines"` StatusCode int `json:"status_code,omitempty" csv:"status_code"` - ContentLength int `json:"content_length,omitempty" csv:"content_length"` + ContentLength int `json:"content_length" csv:"content_length"` Failed bool `json:"failed" csv:"failed"` VHost bool `json:"vhost,omitempty" csv:"vhost"` WebSocket bool `json:"websocket,omitempty" csv:"websocket"` @@ -86,6 +88,8 @@ type Result struct { ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path"` ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel"` KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"knowledgebase"` + + TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-"` } // function to get dsl variables from result struct