spider: outputs all of the unique URLs on a domain

robmiller · Feb 13, 2018 · fb3dc29 · fb3dc29
1 parent c773358
commit fb3dc29
Showing 1 changed file with 14 additions and 0 deletions.
diff --git a/bin/spider b/bin/spider
@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# spider
+#
+# Author: Rob Miller <rob@bigfish.co.uk>
+#
+# Outputs all of the HTML pages on a given domain.
+
+
+wget -r -nd --delete-after -w 0.1 "$1" 2>&1 |
+  grep -B3 text/html |
+  grep -B2 '200 OK' | egrep 'https?://' |
+  cut -d' ' -f3- |
+  sort | uniq