resolved #435 by adding memory consumption monitoring

enum/enum.go

@@ -290,6 +290,10 @@ loop:
 		case <-e.done:
 			break loop
 		case <-t.C:
+			if e.Sys.HighMemoryConsumption() {
+				continue loop
+			}
+
 			num := e.useManagers(secDelay)
 
 			var inactive bool

examples/config.ini

@@ -95,6 +95,9 @@ port = 443
 #[BufferOver]
 #ttl = 10080
 
+#[BuiltWith]
+#ttl = 10080
+
 #[C99]
 #apikey=
 #ttl = 4320
@@ -115,6 +118,9 @@ port = 443
 #[DNSDB]
 #apikey =
 
+#[DNSTable]
+#ttl = 4320
+
 #[FacebookCT]
 #apikey=
 #secret=
@@ -123,6 +129,9 @@ port = 443
 #apikey =
 #ttl = 4320
 
+#[HackerOne]
+#ttl = 4320
+
 #[HackerTarget]
 #ttl = 4320
 
@@ -134,6 +143,12 @@ port = 443
 #apikey =
 #ttl = 10080
 
+#[RapidDNS]
+#ttl = 4320
+
+#[Riddler]
+#ttl = 4320
+
 #[SecurityTrails]
 #apikey =
 #ttl = 1440
@@ -142,6 +157,9 @@ port = 443
 #apikey =
 #ttl = 10080
 
+#[SiteDossier]
+#ttl = 4320
+
 #[Spyse]
 #apikey =
 #ttl = 4320

graph/event.go

@@ -65,7 +65,7 @@ func (g *Graph) InsertEvent(eventID string) (graphdb.Node, error) {
 	return eventNode, nil
 }
 
-// AddNodeToEvent creates an associations between a node in the graph, a data source and a discovery task.
+// AddNodeToEvent creates associations between a node in the graph, a data source and a discovery task.
 func (g *Graph) AddNodeToEvent(node graphdb.Node, source, tag, eventID string) error {
 	if source == "" || tag == "" || eventID == "" {
 		return errors.New("Graph: AddNodeToEvent: Invalid arguments provided")

graph/migrate.go

@@ -5,109 +5,76 @@ package graph
 
 import (
 	"fmt"
-	"sync"
-	"time"
 
 	"github.com/OWASP/Amass/v3/graphdb"
-	"github.com/OWASP/Amass/v3/queue"
-	"github.com/OWASP/Amass/v3/semaphore"
 )
 
-const maxConcurrentDBRequests int = 3
-
 // MigrateEvent copies the nodes and edges related to the Event identified by the uuid from the receiver Graph into another.
 func (g *Graph) MigrateEvent(uuid string, to *Graph) error {
-	q := new(queue.Queue)
-	sem := semaphore.NewSimpleSemaphore(maxConcurrentDBRequests)
+	fnodes := make(map[string]graphdb.Node)
+	tnodes := make(map[string]graphdb.Node)
 
-	idToNode, err := newNodeMap(uuid, g, to)
+	event, err := g.db.ReadNode(uuid, "event")
 	if err != nil {
-		return fmt.Errorf("Graph: Migrate: Failed to setup the node map: %v", err)
+		return fmt.Errorf("Graph: Migrate: Failed to read the event node for %s: %v", uuid, err)
 	}
 
-	event, err := g.db.ReadNode(uuid, "event")
+	edges, err := g.db.ReadOutEdges(event)
 	if err != nil {
-		return fmt.Errorf("Graph: Migrate: Failed to read the event node for %s: %v", uuid, err)
+		return fmt.Errorf("Graph: Migrate: Failed to read the edges from event node %s: %v", uuid, err)
 	}
 
-	if _, _, err := idToNode.getNode(uuid, event); err != nil {
-		return fmt.Errorf("Graph: Migrate: %v", err)
+	// Copy the event node into the graph
+	node, err := g.migrateNode(uuid, event, to)
+	if err != nil {
+		return fmt.Errorf("Graph: Migrate: Failed to copy node %s: %v", uuid, err)
 	}
+	fnodes[uuid] = event
+	tnodes[uuid] = node
 
-	q.Append(event)
-	for {
-		element, ok := q.Next()
-		if !ok {
-			break
-		}
-		cur := element.(graphdb.Node)
+	// Copy all remaining nodes into the graph
+	for _, edge := range edges {
+		id := g.db.NodeToID(edge.To)
 
-		curID := g.db.NodeToID(cur)
-		// Get the same node that is in the graph receiving the copies
-		tocur, found, err := idToNode.getNode(curID, cur)
-		if err != nil || !found {
-			return fmt.Errorf("Graph: Migrate: Failed to read the node %s", curID)
+		// Check if this node has already been migrated
+		if _, found := fnodes[id]; found {
+			continue
 		}
 
-		edges, err := g.db.ReadOutEdges(cur)
+		node, err = g.migrateNode(id, edge.To, to)
 		if err != nil {
-			continue
+			return fmt.Errorf("Graph: Migrate: Failed to copy node %s: %v", id, err)
 		}
 
-		elen := len(edges)
-		ch := make(chan error, elen)
-		for _, edge := range edges {
-			sem.Acquire(1)
-			go g.migrateEdge(tocur, edge, to, q, idToNode, sem, ch)
-		}
-		// Wait for all error values from edge migrations
-		for i := 0; i < elen; i++ {
-			if err := <-ch; err != nil {
-				return err
-			}
-		}
+		fnodes[id] = edge.To
+		tnodes[id] = node
 	}
 
-	return nil
-}
-
-func (g *Graph) migrateEdge(cur graphdb.Node, edge *graphdb.Edge, to *Graph,
-	q *queue.Queue, ids *nodeMap, sem semaphore.Semaphore, ch chan error) {
-	defer sem.Release(1)
-
-	node, found, err := ids.getNode(g.db.NodeToID(edge.To), edge.To)
-	if err != nil {
-		if err.Error() == "out of scope" {
-			err = nil
+	// Create all the edges between the copied nodes
+	for id, node := range fnodes {
+		ins, err := g.db.ReadInEdges(node)
+		if err != nil {
+			continue
 		}
-		ch <- err
-		return
-	}
 
-	if !found {
-		q.Append(edge.To)
-	}
+		for _, edge := range ins {
+			fid := g.db.NodeToID(edge.From)
 
-	for i := 0; i < 3; i++ {
-		err = to.InsertEdge(&graphdb.Edge{
-			Predicate: edge.Predicate,
-			From:      cur,
-			To:        node,
-		})
-		if err == nil {
-			break
+			err = to.InsertEdge(&graphdb.Edge{
+				Predicate: edge.Predicate,
+				From:      tnodes[fid],
+				To:        tnodes[id],
+			})
+			if err != nil {
+				return fmt.Errorf("Graph: Migrate: Failed to create the edge from node %s to node %s: %v", fid, id, err)
+			}
 		}
-		time.Sleep(time.Second)
 	}
-	ch <- err
-}
 
-func (g *Graph) migrateNode(node graphdb.Node, to *Graph) (graphdb.Node, error) {
-	id := g.db.NodeToID(node)
-	if id == "" {
-		return nil, fmt.Errorf("migrateNode: Invalid %s node provided", g.String())
-	}
+	return nil
+}
 
+func (g *Graph) migrateNode(id string, node graphdb.Node, to *Graph) (graphdb.Node, error) {
 	// Obtain the properties of the Node being migrated
 	properties, err := g.db.ReadProperties(node)
 	if err != nil || len(properties) == 0 {
@@ -122,14 +89,12 @@ func (g *Graph) migrateNode(node graphdb.Node, to *Graph) (graphdb.Node, error)
 
 	// Check if this node already exists in the 'to' graph
 	tonode, err := to.db.ReadNode(id, ntype)
-	if err == nil {
-		return tonode, nil
-	}
-
-	// Create the Node in the Graph
-	tonode, err = to.db.InsertNode(id, ntype)
 	if err != nil {
-		return nil, fmt.Errorf("migrateNode: Failed to insert the %s node %s: %v", g.String(), id, err)
+		// Create the Node in the Graph
+		tonode, err = to.db.InsertNode(id, ntype)
+		if err != nil {
+			return nil, fmt.Errorf("migrateNode: Failed to insert the %s node %s: %v", g.String(), id, err)
+		}
 	}
 
 	// Copy all the properties into the Node being migrated
@@ -140,17 +105,6 @@ func (g *Graph) migrateNode(node graphdb.Node, to *Graph) (graphdb.Node, error)
 	return tonode, nil
 }
 
-func (g *Graph) nodeToType(node graphdb.Node) string {
-	// Obtain the properties of the Node being migrated
-	properties, err := g.db.ReadProperties(node)
-	if err != nil || len(properties) == 0 {
-		return ""
-	}
-
-	// Obtain the type of the Node being migrated
-	return getTypeFromProperties(properties)
-}
-
 func (g *Graph) migrateProperties(node graphdb.Node, properties []*graphdb.Property, to *Graph) error {
 	for _, p := range properties {
 		if p.Predicate == "type" {
@@ -177,59 +131,3 @@ func getTypeFromProperties(properties []*graphdb.Property) string {
 
 	return ntype
 }
-
-type nodeMap struct {
-	sync.Mutex
-	uuid     string
-	from, to *Graph
-	nodes    map[string]graphdb.Node
-}
-
-func newNodeMap(uuid string, from, to *Graph) (*nodeMap, error) {
-	// Setup the Event associated with the UUID to start the migration
-	event, err := from.db.ReadNode(uuid, "event")
-	if err != nil {
-		return nil, err
-	}
-
-	m := &nodeMap{
-		uuid:  uuid,
-		from:  from,
-		to:    to,
-		nodes: make(map[string]graphdb.Node),
-	}
-
-	node, err := from.migrateNode(event, to)
-	if err != nil {
-		return nil, err
-	}
-	m.nodes[uuid] = node
-
-	return m, nil
-}
-
-func (n *nodeMap) getNode(id string, fromNode graphdb.Node) (graphdb.Node, bool, error) {
-	n.Lock()
-	defer n.Unlock()
-
-	node, found := n.nodes[id]
-	if found {
-		return node, found, nil
-	}
-
-	if !n.from.InEventScope(fromNode, n.uuid) {
-		return nil, false, fmt.Errorf("out of scope")
-	}
-
-	var err error
-	for i := 0; i < 3; i++ {
-		node, err = n.from.migrateNode(fromNode, n.to)
-		if err == nil {
-			n.nodes[id] = node
-			break
-		}
-		time.Sleep(time.Second)
-	}
-
-	return node, false, err
-}