Added Anubis and SonarSearch as data sources #530

README.md

@@ -28,7 +28,7 @@ The OWASP Amass Project performs network mapping of attack surfaces and external
 | DNS          | Brute forcing, Reverse DNS sweeping, NSEC zone walking, Zone transfers, FQDN alterations/permutations, FQDN Similarity-based Guessing |
 | Scraping     | Ask, Baidu, Bing, BuiltWith, DNSDumpster, HackerOne, RapidDNS, Riddler, SiteDossier, ViewDNS, Yahoo |
 | Certificates | Active pulls (optional), Censys, CertSpotter, Crtsh, FacebookCT, GoogleCT |
-| APIs         | AlienVault, BinaryEdge, BufferOver, C99, CIRCL, Cloudflare, CommonCrawl, DNSDB, GitHub, HackerTarget, Mnemonic, NetworksDB, PassiveTotal, Pastebin, RADb, ReconDev, Robtex, SecurityTrails, ShadowServer, Shodan, Spyse, Sublist3rAPI, TeamCymru, ThreatCrowd, ThreatMiner, Twitter, Umbrella, URLScan, VirusTotal, WhoisXML, ZETAlytics, ZoomEye |
+| APIs         | AlienVault, Anubis, BinaryEdge, BufferOver, C99, CIRCL, Cloudflare, CommonCrawl, DNSDB, GitHub, HackerTarget, Mnemonic, NetworksDB, PassiveTotal, Pastebin, RADb, ReconDev, Robtex, SecurityTrails, ShadowServer, Shodan, SonarSearch, Spyse, Sublist3rAPI, TeamCymru, ThreatCrowd, ThreatMiner, Twitter, Umbrella, URLScan, VirusTotal, WhoisXML, ZETAlytics, ZoomEye |
 | Web Archives | ArchiveIt, LoCArchive, UKGovArchive, Wayback |
 
 ----

resources/scripts/api/anubis.ads

@@ -0,0 +1,17 @@
+-- Copyright 2021 Jeff Foley. All rights reserved.
+-- Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+name = "Anubis"
+type = "api"
+
+function start()
+    setratelimit(1)
+end
+
+function vertical(ctx, domain)
+    scrape(ctx, {url=buildurl(domain)})
+end
+
+function buildurl(domain)
+    return "https://jldc.me/anubis/subdomains/" .. domain
+end

resources/scripts/api/sonarsearch.ads

@@ -0,0 +1,60 @@
+-- Copyright 2021 Jeff Foley. All rights reserved.
+-- Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+local json = require("json")
+
+name = "SonarSearch"
+type = "api"
+
+function start()
+    setratelimit(3)
+end
+
+function vertical(ctx, domain)
+    local p = 0
+    local cfg = datasrc_config()
+
+    while(true) do
+        local resp
+        local vurl = buildurl(domain, p)
+        -- Check if the response data is in the graph database
+        if (cfg.ttl ~= nil and cfg.ttl > 0) then
+            resp = obtain_response(vurl, cfg.ttl)
+        end
+
+        if (resp == nil or resp == "") then
+            resp, err = request({url=vurl})
+            if (err ~= nil and err ~= "") then
+                return
+            end
+
+            if (cfg.ttl ~= nil and cfg.ttl > 0) then
+                cache_response(vurl, resp)
+            end
+        end
+
+        local d = json.decode(resp)
+        if (d == nil or #d == 0) then
+            return
+        end
+
+        sendnames(ctx, resp)
+        checkratelimit()
+        p = p + 1
+    end
+end
+
+function buildurl(domain, page)
+    return "https://sonar.omnisint.io/subdomains/" .. domain .. "?page=" .. page
+end
+
+function sendnames(ctx, content)
+    local names = find(content, subdomainre)
+    if names == nil then
+        return
+    end
+
+    for i, v in pairs(names) do
+        newname(ctx, v)
+    end
+end