robots-example.txt

# robots.txt file for webtrees
#
# Copyright (C) Greg Roach 2014
#
# This file needs to be placed in the domain root folder,
# such as "www.example.com/robots.txt".  It will not work in a
# subfolder, such as "www.example.com/webtrees/robots.txt"
# If you need to move it, then remember to adjust the paths as well.
# e.g. "Allow: /index.php" becomes "Allow: /webtrees/index.php".
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

# Well-behaved robots understand "Allow" directives.
# Google (plus some others?) understands the "$" pattern match
User-Agent: bingbot       # Bing
User-Agent: Googlebot     # Google
User-Agent: ia_archiver   # Internet Archive "wayback" machine
User-Agent: msnbot        # Microsoft
User-Agent: robozilla     # DMOZ
User-Agent: slurp         # Yahoo!
User-Agent: teoma         # Ask Jeeves
User-Agent: yandex        # Popular Russian search engine
User-Agent: baiduspider   # Popular Chinese search engine
Allow: /family.php
Allow: /index.php
Allow: /indilist.php
Allow: /individual.php
Allow: /mediafirewall.php
Allow: /mediaviewer.php
Allow: /module.php?mod=sitemap
Allow: /note.php
Allow: /repo.php
Allow: /site-offline.php
Allow: /site-unavailable.php
Allow: /source.php
Allow: /$
Disallow: /

# Disallow all other robots
User-Agent: *
Disallow: /

# Put your sitemap here:
#Sitemap: http://www.example.com/module.php?mod=sitemap&mod_action=generate&file=sitemap.xml