2023-03-07

pkg/common/uncover/uncorver.go

@@ -45,18 +45,22 @@ func GetTargetsFromUncover(delay, limit int, field string, engine, query []strin
 	}
 	_ = loadProvidersFrom(defaultProviderConfigLocation, uncoverOptions)
 
-	//for _, eng := range engine {
-	//	err := loadKeys(eng, uncoverOptions)
-	//	if err != nil {
-	//		gologger.Error().Label("WRN").Msgf(err.Error())
-	//		continue
-	//	}
-	//}
+	for _, eng := range engine {
+		err := loadKeys(eng, uncoverOptions)
+		if err != nil {
+			gologger.Error().Label("WRN").Msgf(err.Error())
+			continue
+		}
+	}
+	if !uncoverOptions.Provider.HasKeys() {
+		return nil, errors.New("no keys provided")
+	}
 	return getTargets(uncoverOptions, field)
 }
 func getTargets(uncoverOptions *ucRunner.Options, field string) (chan string, error) {
 	var rateLimiter *ratelimit.Limiter
 	// create rateLimiter for uncover delay
+
 	if uncoverOptions.Delay > 0 {
 		rateLimiter = ratelimit.New(context.Background(), 1, time.Duration(uncoverOptions.Delay))
 	} else {
@@ -215,6 +219,12 @@ func loadKeys(engine string, options *ucRunner.Options) error {
 		} else {
 			return errors.Errorf("ZONE_API_KEY env variable is not configured")
 		}
+	case "binary":
+		if key, exists := os.LookupEnv("BINARY_API_KEY"); exists {
+			options.Provider.Netlas = append(options.Provider.Binary, key)
+		} else {
+			return errors.Errorf("BINARY_API_KEY env variable is not configured")
+		}
 	default:
 		return errors.Errorf("unknown uncover agent")
 	}

pkg/projectdiscovery/uncover/runner/provider.go

@@ -82,5 +82,6 @@ func (provider *Provider) HasKeys() bool {
 		len(provider.ZoomEye) > 0 ||
 		len(provider.Netlas) > 0 ||
 		len(provider.CriminalIP) > 0 ||
-		len(provider.Zone) > 0
+		len(provider.Zone) > 0 ||
+		len(provider.Binary) > 0
 }

pkg/projectdiscovery/uncover/uncover/agent/quake/quake.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http"
+	"net/url"
 
 	"github.com/wjlin0/pathScan/pkg/projectdiscovery/uncover/uncover"
 )
@@ -47,14 +48,6 @@ func (agent *Agent) Query(session *uncover.Session, query *uncover.Query) (chan
 				Size:        Size,
 				Start:       numberOfResults,
 				IgnoreCache: true,
-				Exclude: []string{"transport", "asn", "org", "service.name", "location.country_cn",
-					"service.http.host", "time", "service.http.title", "service.response", "service.cert",
-					"components.product_catalog", "components.product_type", "components.product_type", "location.country_en",
-					"location.province_en",
-					"location.city_en",
-					"location.district_en",
-					"location.district_cn",
-					"location.province_cn", "location.city_cn", "service.http.host", "service.http.body"},
 			}
 			quakeResponse := agent.query(URL, session, quakeRequest, results)
 			if quakeResponse == nil {
@@ -90,11 +83,26 @@ func (agent *Agent) query(URL string, session *uncover.Session, quakeRequest *Re
 	}
 
 	for _, quakeResult := range quakeResponse.Data {
-
 		result := uncover.Result{Source: agent.Name()}
 		result.IP = quakeResult.IP
 		result.Port = quakeResult.Port
-		result.Host = quakeResult.Domain
+		switch {
+		case quakeResult.Hostname != "":
+			result.Host = quakeResult.Hostname
+		case quakeResult.Domain != "":
+			result.Host = quakeResult.Domain
+		case quakeResult.Service != nil && (quakeResult.Service.Name == "http" || quakeResult.Service.Name == "http/ssl") && quakeResult.Service.Http.Host != "":
+			result.Host = quakeResult.Service.Http.Host
+		case quakeResult.Service != nil && (quakeResult.Service.Name == "http" || quakeResult.Service.Name == "http/ssl") && len(quakeResult.Service.Http.HttpLoadUrl) > 0:
+			parse, err := url.Parse(quakeResult.Service.Http.HttpLoadUrl[0])
+			if err != nil {
+				result.Host = ""
+			} else {
+				result.Host = parse.Host
+			}
+		default:
+			result.Host = ""
+		}
 		raw, _ := json.Marshal(result)
 		result.Raw = raw
 		results <- result

pkg/projectdiscovery/uncover/uncover/agent/quake/request.go

@@ -1,9 +1,8 @@
 package quake
 
 type Request struct {
-	Query       string   `json:"query"`
-	Size        int      `json:"size"`
-	Start       int      `json:"start"`
-	IgnoreCache bool     `json:"ignore_cache"`
-	Exclude     []string `json:"exclude"`
+	Query       string `json:"query"`
+	Size        int    `json:"size"`
+	Start       int    `json:"start"`
+	IgnoreCache bool   `json:"ignore_cache"`
 }

pkg/projectdiscovery/uncover/uncover/agent/quake/response.go

@@ -1,9 +1,11 @@
 package quake
 
 type responseData struct {
-	Domain string `json:"domain"`
-	IP     string `json:"ip"`
-	Port   int    `json:"port"`
+	Domain   string   `json:"domain"`
+	IP       string   `json:"ip"`
+	Port     int      `json:"port"`
+	Hostname string   `json:"hostname"`
+	Service  *service `json:"service"`
 }
 
 type pagination struct {
@@ -16,7 +18,14 @@ type pagination struct {
 type meta struct {
 	Pagination pagination `json:"pagination"`
 }
-
+type service struct {
+	Name string        `json:"name"`
+	Http *httpResponse `json:"http"`
+}
+type httpResponse struct {
+	Host        string   `json:"host"`
+	HttpLoadUrl []string `json:"http_load_url"`
+}
 type Response struct {
 	Code    int            `json:"code"`
 	Data    []responseData `json:"data"`

pkg/projectdiscovery/uncover/uncover/agent/zone/zone.go

@@ -94,14 +94,20 @@ func (agent *Agent) query(session *uncover.Session, zoneRequest *ZoneRequest, re
 		result := uncover.Result{Source: agent.Name()}
 		result.IP = zoneResult.Ip
 		result.Port = zoneResult.Port
-		var host string
-		p, err := url.Parse(zoneResult.Url)
-		if err != nil {
-			host = result.IP + strconv.Itoa(result.Port)
-		} else {
-			host = p.Host
+		switch {
+		case zoneResult.IpAddr != "":
+			result.Host = zoneResult.IpAddr
+		case zoneResult.Url != "":
+			var host string
+			p, err := url.Parse(zoneResult.Url)
+			if err != nil {
+				host = result.IP + strconv.Itoa(result.Port)
+			} else {
+				host = p.Host
+			}
+			result.Host = host
 		}
-		result.Host = host
+
 		raw, _ := json.Marshal(result)
 		result.Raw = raw
 		results <- result

pkg/runner/handlerUrl.go

@@ -70,34 +70,35 @@ func (r *Runner) getAllTargets() []string {
 		}
 	}
 	if r.Cfg.Options.Uncover && r.Cfg.Options.UncoverQuery != nil {
-
 		if r.Cfg.Options.UncoverEngine == nil {
 			r.Cfg.Options.UncoverEngine = []string{"quake", "fofa"}
 		}
 		gologger.Info().Msgf("正在运行: %s", strings.Join(r.Cfg.Options.UncoverEngine, ","))
 		ch, err := uncover.GetTargetsFromUncover(r.Cfg.Options.UncoverDelay, r.Cfg.Options.UncoverLimit, r.Cfg.Options.UncoverField, r.Cfg.Options.UncoverEngine, r.Cfg.Options.UncoverQuery)
 		if err != nil {
 			gologger.Error().Label("WRN").Msg(err.Error())
-		}
-		for c := range ch {
-			c = strings.Trim(c, "\r")
-			c = strings.Trim(c, "\n")
-			if c == "" {
-				continue
-			}
-			if !strings.HasPrefix(c, "http") {
-				c1 := "http://" + c
-				c = "https://" + c
-				if !strings.HasSuffix(c1, "/") {
-					c1, _ = url.JoinPath(c1, "/")
+		} else {
+			for c := range ch {
+				c = strings.Trim(c, "\r")
+				c = strings.Trim(c, "\n")
+				if c == "" {
+					continue
 				}
-				at[c1] = struct{}{}
-			}
-			if !strings.HasSuffix(c, "/") {
-				c, _ = url.JoinPath(c, "/")
+				if !strings.HasPrefix(c, "http") {
+					c1 := "http://" + c
+					c = "https://" + c
+					if !strings.HasSuffix(c1, "/") {
+						c1, _ = url.JoinPath(c1, "/")
+					}
+					at[c1] = struct{}{}
+				}
+				if !strings.HasSuffix(c, "/") {
+					c, _ = url.JoinPath(c, "/")
+				}
+				at[c] = struct{}{}
 			}
-			at[c] = struct{}{}
 		}
+
 	}
 	for _, skip := range r.Cfg.Options.SkipUrl {
 		delete(at, skip)

pkg/runner/options.go

@@ -93,7 +93,7 @@ func ParserOptions() *Options {
 		set.BoolVarP(&options.Uncover, "uncover", "uc", false, "启用打开搜索引擎"),
 		set.StringSliceVarP(&options.UncoverQuery, "uncover-query", "uq", nil, "搜索查询", goflags.FileStringSliceOptions),
 		set.StringSliceVarP(&options.UncoverEngine, "uncover-engine", "ue", nil, fmt.Sprintf("支持的引擎 (%s) (default quake)", uncover.GetUncoverSupportedAgents()), goflags.FileStringSliceOptions),
-		set.StringVarP(&options.UncoverField, "uncover-field", "uf", "host", "uncover fields to return (ip,port,host)"),
+		set.StringVarP(&options.UncoverField, "uncover-field", "uf", "host", "引擎返回字段 (ip,port,host)"),
 		set.IntVarP(&options.UncoverLimit, "uncover-limit", "ul", 200, "发现要返回的结果"),
 		set.IntVarP(&options.UncoverDelay, "uncover-delay", "ucd", 1, "打开查询请求之间的延迟（秒）(0 to disable)"),
 	)