-
Notifications
You must be signed in to change notification settings - Fork 167
/
crawl.py
35 lines (29 loc) · 818 Bytes
/
crawl.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
#!/usr/bin/python
import sys
import json
import requests
import argparse
from bs4 import BeautifulSoup
def results(file):
content = open(file, 'r').readlines()
for line in content:
data = json.loads(line.strip())
urls = []
for url in data['results']:
urls.append(url['url'])
return urls
def crawl(url):
r = requests.get(url)
soup = BeautifulSoup(r.text, 'lxml')
links = soup.findAll('a', href=True)
for link in links:
link = link['href']
if link and link != '#':
print '[+] {} : {}'.format(url, link)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("file", help="ffuf results")
args = parser.parse_args()
urls = results(args.file)
for url in urls:
crawl(url)