robots.cfg

RobotUA <<EOR
    adressendeutschland,
    AdsBot-Google,
    agent,
    AltaVista,
    Apache (internal dummy connection),
    appie,
    AppleSyndication,
    Arachnoidea,
    Aranha,
    Architext,
    archive,
    Argus,
    Ask,
    asterias,
    ATN_Worldwide,
    Atomz,
    AurNet,
    Awasu,
    BackRub,
    bender,
    Bookdog,
    BookmarkSync,
    bot,
    Builder,
    CCBot,
    ccubee,
    cfetch,
    CFNetwork,
    check_http,
    CMC,
    collector,
    complex_network_group,
    Contact,
    crawl,
    Creep,
    Digital*Integrity,
    Directory,
    dogpile,
    Excite,
    EZResult,
    FavOrg,
    FeedDemon,
    FeedFetcher-Google,
    Feedreader,
    FeedValidator,
    Ferret,
    fido,
    find,
    Fireball,
    gazz,
    GetRight,
    gonzo,
    Google-Sitemaps,
    GoogleBot,
    grab,
    griffon,
    Gromit,
    Gulliver,
    H?m?h?kki,
    heritrix,
    HTTrack,
    Harvest,
    holmes,
    HTMLDOC,
    Hubater,
    IncyWincy,
    index,
    INGRID,
    Jack,
    JPluck,
    KIT*Fireball,
    Kototoi,
    larbin,
    Leech,
    legs,
    libwww-perl,
    locator,
    LWP,
    Lycos,
    marvin,
    Mediapartners,
    MegaSheep,
    MEGAUPLOAD,
    Mercator,
    MFC_Tear_Sample,
    Microsoft Data Access,
    Microsoft Office,
    Microsoft URL Control,
    Microsoft-WebDAV,
    MimeLive,
    mirago,
    Miva,
    moget,
    MSFrontPage,
    msnbot,
    Nazilla,
    NetMechanic,
    NetScoop,
    newscan,
    Nutch,
    Ocelli,
    ozelot,
    ozzie,
    pagebull,
    ParaSite,
    pavuk,
    POE-Component,
    Pokey,
    Pompos,
    Refiner,
    retrieve,
    RoboDude,
    Rover,
    Rssbandit,
    RSSOwl,
    Rutgers,
    Scooter,
    search,
    seek,
    shelob,
    ShopWiki,
    silk,
    Slurp,
    sna,
    Snappy,
    Snoopy,
    speedy,
    spider,
    Spyder,
    suke,
    Susie,
    swish,
    T-H-U-N-D-E-R-S-T-O-N-E,
    tarantula,
    topiclink,
    Toutatis,
    TurnitinBot,
    Tv*Merc,
    Twiceler,
    urllib,
    VB Project,
    Valkyrie,
    Voyager,
    W3C_Validator,
    Walker,
    wget,
    WhizBang,
    whowhere,
    Wiki,
    WinInet,
    winona,
    Wire,
    Wombat,
    WordPress,
    worm,
    wwwster,
    WWW-Mechanize,
    xtreme,
    Yahoo,
    Yandex,
    Zeus,
    ZyBorg,
EOR

NotRobotUA <<EOR
    AskTB,
    Seekmo,
EOR

RobotIP <<EOR
    202.9.155.123,
    204.152.191.41,
    208.146.26.19,
    208.146.26.233,
    209.185.141.209,
    209.185.141.211,
    209.202.148.36,
    209.202.148.41,
    216.200.130.207,
    216.35.103.6?,
    216.35.103.70,
    82.99.30.?,		# Munax
    82.99.30.1?,	# Munax
    82.99.30.2?,	# Munax
    82.99.30.3?,	# Munax
    82.99.30.4?,	# Munax
    82.99.30.5?,	# Munax
    82.99.30.6?,	# Munax
    82.99.30.70,	# Munax
    82.99.30.71,	# Munax
    82.99.30.72,	# Munax
    82.99.30.73,	# Munax
EOR

RobotHost <<EOR
    *.analys.google.com,
    *.ask.com,
    *.crawler*.com,
    *.csccorporatedomains.com,
    *.excite.com,
    *.googlebot.com,
    *.infoseek.com,
    *.inktomi.com,
    *.inktomisearch.com,
    *.lycos.com,
    *.pa-x.dec.com,
    *.phx.gbl,
    *.search.live.com,
    add-url.altavista.com,
    msnbot.msn.com,
    westinghouse-rsl-com-usa.NorthRoyalton.cw.net,
EOR