Store visited IP state in a closure

fierce/fierce.py

@@ -34,6 +34,18 @@ def print_subdomain_result(url, ip, http_connection_headers=None, nearby=None, s
         pprint.pprint(nearby, stream=stream)
 
 
+def unvisited_closure():
+    visited = set()
+
+    def inner(l):
+        nonlocal visited
+        result = l.difference(visited)
+        visited.update(l)
+        return result
+
+    return inner
+
+
 def find_subdomain_list_file(filename):
     # First check the list directory relative to where we are. This
     # will typically happen if they simply cloned the Github repository
@@ -293,7 +305,7 @@ def fierce(**kwargs):
         kwargs["subdomain_file"]
     )
 
-    visited = set()
+    get_unvisited = unvisited_closure()
 
     for subdomain in subdomains:
         url = concatenate_subdomains(domain, [subdomain])
@@ -319,8 +331,7 @@ def fierce(**kwargs):
         else:
             ips = []
 
-        ips = set(ips) - set(visited)
-        visited |= ips
+        ips = get_unvisited(ips)
 
         nearby_ips = find_nearby(resolver, ips, filter_func=filter_func)
 

tests/test_fierce.py

@@ -420,6 +420,33 @@ def test_print_subdomain_result_both(self):
 
         self.assertEqual(expected, result)
 
+    def test_unvisited_closure_empty(self):
+        unvisited = fierce.unvisited_closure()
+        ips = set()
+
+        result = unvisited(ips)
+        expected = set()
+
+        self.assertEqual(expected, result)
+
+    def test_unvisited_closure_empty_intersection(self):
+        unvisited = fierce.unvisited_closure()
+
+        unvisited(set([1, 2, 3]))
+        result = unvisited(set([4, 5, 6]))
+        expected = set([4, 5, 6])
+
+        self.assertEqual(expected, result)
+
+    def test_unvisited_closure_overlapping_intersection(self):
+        unvisited = fierce.unvisited_closure()
+
+        unvisited(set([1, 2, 3]))
+        result = unvisited(set([2, 3, 4]))
+        expected = set([4])
+
+        self.assertEqual(expected, result)
+
 
 if __name__ == "__main__":
     unittest.main()