spider script: wait longer!

robmiller · Mar 20, 2022 · 6b9090b · 6b9090b
1 parent fe98896
commit 6b9090b
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/bin/spider b/bin/spider
@@ -7,7 +7,7 @@
 # Outputs all of the HTML pages on a given domain.
 
 
-wget -r -nd --delete-after -w 0.1 "$1" 2>&1 |
+wget -r -nd --delete-after -w 1 "$1" 2>&1 |
   grep -B3 text/html |
   grep -B2 '200 OK' | egrep 'https?://' |
   cut -d' ' -f3- |