refactor: keys management

cmd/xurlfind3r/main.go

@@ -42,7 +42,7 @@ var (
 
 func init() {
 	// defaults
-	defaultYAMLConfigFile := "~/.huristiq/xurlfind3r/config.yaml"
+	defaultYAMLConfigFile := "~/.hueristiq/xurlfind3r/config.yaml"
 
 	// Handle CLI arguments, flags & help message (pflag)
 	pflag.StringVarP(&YAMLConfigFile, "configuration", "c", defaultYAMLConfigFile, "")
@@ -131,17 +131,14 @@ func main() {
 		hqgolog.Fatal().Msg(err.Error())
 	}
 
-	// Get sources' keys from configuration
-	keys := config.GetKeys()
-
 	// List suported sources
 	if listSources {
 		hqgolog.Info().Msgf("listing %v current supported sources", au.Underline(strconv.Itoa(len(config.Sources))).Bold())
 		hqgolog.Info().Msgf("sources with %v needs a key or token", au.Underline("*").Bold())
 		hqgolog.Print().Msg("")
 
 		needsKey := make(map[string]interface{})
-		keysElem := reflect.ValueOf(&keys).Elem()
+		keysElem := reflect.ValueOf(&config.Keys).Elem()
 
 		for i := 0; i < keysElem.NumField(); i++ {
 			needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface()
@@ -175,7 +172,7 @@ func main() {
 		Domain:             domain,
 		IncludeSubdomains:  includeSubdomains,
 		Sources:            sourcesToUse,
-		Keys:               keys,
+		Keys:               config.Keys,
 		ParseWaybackRobots: !skipWaybackRobots,
 		ParseWaybackSource: !skipWaybackSource,
 		FilterPattern:      filterPattern,

internal/configuration/configuration.go

@@ -1,10 +1,8 @@
 package configuration
 
 import (
-	"math/rand"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"dario.cat/mergo"
 	"github.com/hueristiq/xurlfind3r/pkg/xurlfind3r/sources"
@@ -13,37 +11,9 @@ import (
 )
 
 type Configuration struct {
-	Version string   `yaml:"version"`
-	Sources []string `yaml:"sources"`
-	Keys    Keys     `yaml:"keys"`
-}
-
-func (configuration *Configuration) GetKeys() sources.Keys {
-	keys := sources.Keys{}
-
-	// Github
-	if len(configuration.Keys.Github) > 0 {
-		keys.GitHub = configuration.Keys.Github
-	}
-
-	// IntelX
-	intelxKeysCount := len(configuration.Keys.Intelx)
-	if intelxKeysCount > 0 {
-		intelxKeys := configuration.Keys.Intelx[rand.Intn(intelxKeysCount)] //nolint:gosec // Works perfectly
-		parts := strings.Split(intelxKeys, ":")
-
-		if len(parts) == 2 {
-			keys.IntelXHost = parts[0]
-			keys.IntelXKey = parts[1]
-		}
-	}
-
-	// URLScan
-	if len(configuration.Keys.URLScan) > 0 {
-		keys.URLScan = configuration.Keys.URLScan
-	}
-
-	return keys
+	Version string       `yaml:"version"`
+	Sources []string     `yaml:"sources"`
+	Keys    sources.Keys `yaml:"keys"`
 }
 
 func (configuration *Configuration) Write(path string) (err error) {
@@ -76,12 +46,6 @@ func (configuration *Configuration) Write(path string) (err error) {
 	return
 }
 
-type Keys struct {
-	Github  []string `yaml:"github"`
-	Intelx  []string `yaml:"intelx"`
-	URLScan []string `yaml:"urlscan"`
-}
-
 const (
 	NAME    string = "xurlfind3r"
 	VERSION string = "0.1.0"
@@ -108,8 +72,8 @@ func CreateUpdate(path string) (err error) {
 	defaultConfig := Configuration{
 		Version: VERSION,
 		Sources: sources.List,
-		Keys: Keys{
-			Github:  []string{},
+		Keys: sources.Keys{
+			GitHub:  []string{},
 			Intelx:  []string{},
 			URLScan: []string{},
 		},

pkg/xurlfind3r/sources/configuration.go

@@ -16,9 +16,7 @@ type Configuration struct {
 }
 
 type Keys struct {
-	GitHub     []string `json:"github"`
-	Intelx     []string `json:"intelx"` // unused, add for the purpose of adding an asterisk `*` on listing sources
-	IntelXHost string   `json:"intelXHost"`
-	IntelXKey  string   `json:"intelXKey"`
-	URLScan    []string `json:"urlscan"`
+	GitHub  []string `yaml:"github"`
+	Intelx  []string `yaml:"intelx"`
+	URLScan []string `yaml:"urlscan"`
 }

pkg/xurlfind3r/sources/intelx/intelx.go

@@ -4,6 +4,7 @@ package intelx
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/hueristiq/xurlfind3r/pkg/xurlfind3r/httpclient"
@@ -16,15 +17,15 @@ type searchResponseType struct {
 	Status int    `json:"status"`
 }
 
-type selectorType struct {
-	Selectvalue string `json:"selectorvalue"`
-}
-
 type searchResultType struct {
 	Selectors []selectorType `json:"selectors"`
 	Status    int            `json:"status"`
 }
 
+type selectorType struct {
+	Selectvalue string `json:"selectorvalue"`
+}
+
 type requestBody struct {
 	Term       string        `json:"term"`
 	Timeout    time.Duration `json:"timeout"`
@@ -42,15 +43,24 @@ func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sourc
 
 		var (
 			err  error
-			body []byte
 			res  *fasthttp.Response
+			body []byte
 		)
 
-		if config.Keys.IntelXKey == "" || config.Keys.IntelXHost == "" {
+		key := sources.PickRandom(config.Keys.Intelx)
+		if key == "" {
+			return
+		}
+
+		parts := strings.Split(key, ":")
+		intelXHost := parts[0]
+		intelXKey := parts[1]
+
+		if intelXKey == "" || intelXHost == "" {
 			return
 		}
 
-		searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", config.Keys.IntelXHost, config.Keys.IntelXKey)
+		searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", intelXHost, intelXKey)
 		reqBody := requestBody{
 			Term:       config.Domain,
 			MaxResults: 100000,
@@ -74,7 +84,7 @@ func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sourc
 			return
 		}
 
-		resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", config.Keys.IntelXHost, config.Keys.IntelXKey, response.ID)
+		resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", intelXHost, intelXKey, response.ID)
 		status := 0
 
 		for status == 0 || status == 3 {

pkg/xurlfind3r/sources/urlscan/urlscan.go

@@ -43,8 +43,13 @@ func (source *Source) Run(config *sources.Configuration) (URLsChannel chan sourc
 			}
 		)
 
+		key := sources.PickRandom(config.Keys.URLScan)
+		if key == "" {
+			return
+		}
+
 		if len(config.Keys.URLScan) > 0 {
-			headers["API-Key"] = config.Keys.URLScan[0]
+			headers["API-Key"] = key
 		}
 
 	PAGINATE:

pkg/xurlfind3r/sources/utils.go

@@ -1,11 +1,29 @@
 package sources
 
 import (
+	"math/rand"
 	"net/mail"
+	"time"
 
 	"github.com/hueristiq/hqgourl"
 )
 
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+func PickRandom[T any](v []T) (picked T) {
+	length := len(v)
+
+	if length == 0 {
+		return
+	}
+
+	picked = v[rand.Intn(length)]
+
+	return
+}
+
 func IsValid(URL string) (isValid bool) {
 	var err error