resolved #410 by caching data source responses

README.md

@@ -23,7 +23,7 @@ The OWASP Amass Project performs network mapping of attack surfaces and external
 |:-------------|:-------------|
 | DNS          | Brute forcing, Reverse DNS sweeping, FQDN alterations/permutations, FQDN Similarity-based Guessing, Zone transfers |
 | Scraping     | Ask, Baidu, Bing, BuiltWith, DNSDumpster, DNSTable, HackerOne, RapidDNS, Riddler, SiteDossier, ViewDNS, Yahoo |
-| Certificates | Active pulls (optional), Censys, CertSpotter, Crtsh, Entrust, FacebookCT, GoogleCT |
+| Certificates | Active pulls (optional), Censys, CertSpotter, Crtsh, FacebookCT, GoogleCT |
 | APIs         | AlienVault, BinaryEdge, BufferOver, C99, CIRCL, CommonCrawl, DNSDB, GitHub, HackerTarget, IPToASN, Mnemonic, NetworksDB, PassiveTotal, Pastebin, RADb, Robtex, SecurityTrails, ShadowServer, Shodan, Spyse, Sublist3rAPI, TeamCymru, ThreatCrowd, ThreatMiner, Twitter, Umbrella, URLScan, VirusTotal, WhoisXML, ZETAlytics |
 | Web Archives | ArchiveIt, LoCArchive, UKGovArchive, Wayback |
 

config/config.go

@@ -150,6 +150,7 @@ type APIKey struct {
 	Password string `ini:"password"`
 	Key      string `ini:"apikey"`
 	Secret   string `ini:"secret"`
+	TTL      int    `ini:"ttl"`
 }
 
 // NewConfig returns a default configuration object.

datasrcs/script.go

@@ -101,6 +101,8 @@ func (s *Script) newLuaState(cfg *config.Config) *lua.LState {
 	L.SetGlobal("outputdir", L.NewFunction(s.outputdir))
 	L.SetGlobal("setratelimit", L.NewFunction(s.setRateLimit))
 	L.SetGlobal("checkratelimit", L.NewFunction(s.checkRateLimit))
+	L.SetGlobal("obtain_response", L.NewFunction(s.obtainResponse))
+	L.SetGlobal("cache_response", L.NewFunction(s.cacheResponse))
 	L.SetGlobal("subdomainre", lua.LString(dns.AnySubdomainRegexString()))
 	return L
 }
@@ -125,6 +127,9 @@ func (s *Script) registerAPIKey(L *lua.LState, cfg *config.Config) {
 	if api.Secret != "" {
 		tb.RawSetString("secret", lua.LString(api.Secret))
 	}
+	if api.TTL != 0 {
+		tb.RawSetString("ttl", lua.LNumber(api.TTL))
+	}
 
 	L.SetGlobal("api", tb)
 }

datasrcs/script_exports.go

@@ -608,3 +608,59 @@ func getNumberField(L *lua.LState, t lua.LValue, key string) (float64, bool) {
 	}
 	return 0, false
 }
+
+// Wrapper so that scripts can obtain cached data source responses.
+func (s *Script) obtainResponse(L *lua.LState) int {
+	lv := L.Get(1)
+	u, ok := lv.(lua.LString)
+	if !ok {
+		L.Push(lua.LNil)
+		return 1
+	}
+	url := string(u)
+
+	lv = L.Get(2)
+	t, ok := lv.(lua.LNumber)
+	if !ok {
+		L.Push(lua.LNil)
+		return 1
+	}
+
+	ttl := int(t)
+	if ttl <= 0 {
+		L.Push(lua.LNil)
+		return 1
+	}
+
+	for _, db := range s.sys.GraphDatabases() {
+		if resp, err := db.GetSourceData(s.String(), url, ttl); err == nil {
+			// Allow the data source to accept another request immediately on cache hits
+			s.ClearLast()
+			L.Push(lua.LString(resp))
+			return 1
+		}
+	}
+
+	L.Push(lua.LNil)
+	return 1
+}
+
+// Wrapper so that scripts can cache data source responses.
+func (s *Script) cacheResponse(L *lua.LState) int {
+	lv := L.Get(1)
+	u, ok := lv.(lua.LString)
+	if !ok {
+		return 0
+	}
+
+	lv = L.Get(2)
+	resp, ok := lv.(lua.LString)
+	if !ok {
+		return 0
+	}
+
+	for _, db := range s.sys.GraphDatabases() {
+		db.CacheSourceData(s.String(), s.SourceType, string(u), string(resp))
+	}
+	return 0
+}

datasrcs/sources.go

@@ -38,11 +38,8 @@ func GetAllSources(sys systems.System) []requests.Service {
 		NewCrtsh(sys),
 		NewDNSDB(sys),
 		NewDNSDumpster(sys),
-		NewEntrust(sys),
-		NewGitHub(sys),
 		NewIPToASN(sys),
 		NewNetworksDB(sys),
-		NewPassiveTotal(sys),
 		NewPastebin(sys),
 		NewRADb(sys),
 		NewRobtex(sys),
@@ -52,7 +49,6 @@ func GetAllSources(sys systems.System) []requests.Service {
 		NewUmbrella(sys),
 		NewURLScan(sys),
 		NewViewDNS(sys),
-		NewVirusTotal(sys),
 		NewWhoisXML(sys),
 	}
 

doc/scripting.md

@@ -58,6 +58,7 @@ If the `name` field for a script has a matching entry in the Amass configuration
 | password   | string    |
 | key        | string    |
 | secret     | string    |
+| ttl        | number    |
 
 ### `start` Callback
 

examples/config.ini

@@ -90,16 +90,23 @@ port = 443
 
 #[BinaryEdge]
 #apikey =
+#ttl = 10080
+
+#[BufferOver]
+#ttl = 10080
 
 #[C99]
 #apikey=
+#ttl = 4320
 
 #[Censys]
 #apikey =
 #secret =
+#ttl = 10080
 
 #[Chaos]
 #apikey=
+#ttl = 4320
 
 #[CIRCL]
 #username =
@@ -114,22 +121,30 @@ port = 443
 
 #[GitHub]
 #apikey =
+#ttl = 4320
+
+#[HackerTarget]
+#ttl = 4320
 
 #[NetworksDB]
 #apikey =
 
 #[PassiveTotal]
 #username =
 #apikey =
+#ttl = 10080
 
 #[SecurityTrails]
 #apikey =
+#ttl = 1440
 
 #[Shodan]
 #apikey =
+#ttl = 10080
 
 #[Spyse]
 #apikey =
+#ttl = 4320
 
 # Provide your Twitter App Consumer API key and Consumer API secrety key
 #[Twitter]
@@ -146,9 +161,11 @@ port = 443
 
 #[VirusTotal]
 #apikey =
+#ttl = 10080
 
 #[WhoisXML]
 #apikey= 
 
 #[ZETAlytics]
 #apikey=
+#ttl = 1440

graph/source.go

@@ -5,6 +5,7 @@ package graph
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/OWASP/Amass/v3/graphdb"
 	"github.com/OWASP/Amass/v3/stringset"
@@ -103,3 +104,94 @@ func (g *Graph) NodeSources(node graphdb.Node, events ...string) ([]string, erro
 
 	return sources, nil
 }
+
+// GetSourceData returns the most recent response from the source/tag for the query within the time to live.
+func (g *Graph) GetSourceData(source, query string, ttl int) (string, error) {
+	node, err := g.db.ReadNode(source, "source")
+	if err != nil {
+		return "", err
+	}
+
+	edges, err := g.db.ReadOutEdges(node, query)
+	if err != nil {
+		return "", err
+	}
+
+	var data string
+	for _, edge := range edges {
+		p, err := g.db.ReadProperties(edge.To, "timestamp")
+		if err != nil || len(p) == 0 {
+			continue
+		}
+
+		d := time.Duration(ttl) * time.Minute
+		ts, err := time.Parse(time.RFC3339, p[0].Value)
+		if err != nil || ts.Add(d).Before(time.Now()) {
+			continue
+		}
+
+		p, err = g.db.ReadProperties(edge.To, "response")
+		if err != nil || len(p) == 0 {
+			continue
+		}
+
+		data = p[0].Value
+		break
+	}
+
+	if data == "" {
+		return "", fmt.Errorf("%s: GetSourceData: Failed to obtain a cached response from %s for query %s", g.String(), source, query)
+	}
+
+	return data, nil
+}
+
+// CacheSourceData inserts an updated response from the source/tag for the query.
+func (g *Graph) CacheSourceData(source, tag, query, resp string) error {
+	snode, err := g.InsertSource(source, tag)
+	if err != nil {
+		return err
+	}
+
+	// Remove previously cached responses for the same query
+	g.deleteCachedData(source, query)
+
+	ts := time.Now().Format(time.RFC3339)
+	rnode, err := g.InsertNodeIfNotExist(source+"-response-"+ts, "response")
+	if err != nil {
+		return err
+	}
+
+	if err := g.db.InsertProperty(rnode, "timestamp", ts); err != nil {
+		return err
+	}
+
+	if err := g.db.InsertProperty(rnode, "response", resp); err != nil {
+		return err
+	}
+
+	return g.InsertEdge(&graphdb.Edge{
+		Predicate: query,
+		From:      snode,
+		To:        rnode,
+	})
+}
+
+func (g *Graph) deleteCachedData(source, query string) error {
+	node, err := g.db.ReadNode(source, "source")
+	if err != nil {
+		return err
+	}
+
+	edges, err := g.db.ReadOutEdges(node, query)
+	if err != nil {
+		return err
+	}
+
+	for _, edge := range edges {
+		g.db.DeleteNode(edge.To)
+		g.db.DeleteEdge(edge)
+	}
+
+	return nil
+}

graph/viz.go

@@ -27,7 +27,7 @@ func (g *Graph) VizData(uuid string) ([]viz.Node, []viz.Edge) {
 	return nodes, edges
 }
 
-// Identify unique nodes that should be included in the visualization
+// Identify unique nodes that should be included in the visualization.
 func (g *Graph) vizNodes(uuid string, edges []*graphdb.Edge) ([]viz.Node, map[string]int) {
 	var idx int
 	var nodes []viz.Node
@@ -39,9 +39,11 @@ func (g *Graph) vizNodes(uuid string, edges []*graphdb.Edge) ([]viz.Node, map[st
 			ids.Insert(id)
 
 			properties, err := g.db.ReadProperties(d.To, "type")
-			// We do not print the source or event nodes in the graph visualizations
+			// We do not print the source, event or response nodes in the graph visualizations
 			if err != nil || len(properties) == 0 ||
-				properties[0].Value == "source" || properties[0].Value == "event" {
+				properties[0].Value == "source" ||
+				properties[0].Value == "event" ||
+				properties[0].Value == "response" {
 				continue
 			}
 
@@ -63,7 +65,7 @@ func (g *Graph) vizNodes(uuid string, edges []*graphdb.Edge) ([]viz.Node, map[st
 	return nodes, nodeToIdx
 }
 
-// Identify the edges between nodes that should be included in the visualization
+// Identify the edges between nodes that should be included in the visualization.
 func (g *Graph) vizEdges(nodes []viz.Node, nodeToIdx map[string]int) []viz.Edge {
 	var edges []viz.Edge
 
@@ -131,7 +133,7 @@ func (g *Graph) buildVizNode(node graphdb.Node, ntype, uuid string) *viz.Node {
 	}
 }
 
-// Update the type names for visualization
+// Update the type names for visualization.
 func (g *Graph) convertNodeType(id, ntype string, edges []*graphdb.Edge) string {
 	if ntype == "fqdn" {
 		var pred string

requests/service.go

@@ -37,10 +37,6 @@ type Service interface {
 	Stop() error
 	OnStop() error
 
-	// Methods that enforce the rate limit
-	SetRateLimit(min time.Duration)
-	CheckRateLimit()
-
 	// RequestLen returns the current length of the request queue
 	RequestLen() int
 
@@ -261,6 +257,11 @@ func (bas *BaseService) CheckRateLimit() {
 	bas.last = time.Now()
 }
 
+// ClearLast resets the last value so the service can be utilized immediately.
+func (bas *BaseService) ClearLast() {
+	bas.last.Truncate(time.Hour)
+}
+
 type queuedCall struct {
 	Func reflect.Value
 	Args []reflect.Value