Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
branch: master
Fetching contributors…

Cannot retrieve contributors at this time

executable file 36 lines (26 sloc) 0.906 kb
#!/bin/bash
# Check to see if robots are being blocked from this website.
domain=$1
disallow_all=0
if echo "$domain" | grep -i '^ftp'; then
pro="ftp"
else
pro="http"
fi
robots_url=""
if [ $pro == "http" ]; then
response=$(curl --write-out %{http_code} --silent --output /dev/null $pro://$domain/robots.txt)
if [ $response == "200" ] || [ $response == "226" ]; then
robots_url="$pro://$domain/robots.txt"
elif [ $response == "301" ] || [ $response == "302" ]; then
response_https=$(curl --write-out %{http_code} --silent --output /dev/null https://$domain/robots.txt)
if [ $response_https == "200" ]; then
robots_url="https://$domain/robots.txt"
fi
fi
fi
#echo "$domain: $response, $robots_url"
if [ "$robots_url" != "" ]; then
disallow_all=$(curl --silent $robots_url | sed 's/^\r?\n?$//g' | grep -i "Disallow: /$" | wc -l | sed 's/ //g')
fi
echo $disallow_all
Jump to Line
Something went wrong with that request. Please try again.