From f5de7e608032d5427855d699187b78b37a74703a Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Sun, 10 May 2020 10:41:50 +0300 Subject: [PATCH] COUNTER_Robots_list.json: Escape literal dots Some patterns are incorrect because they mean to use a literal dot, but we should technically be escaping these to use literal dots in regex. Note that we need to escape the first backslash in the JSON so we can get a literal backslash for escaping the dots in the out- put file. The dot in the "virus.detector" pattern seems to be inte- ntional, as this user agent sometimes appears as virus-detector and other times as virus_detector. Also I've taken the liberty to add URLs and descriptions where app- licable. --- COUNTER_Robots_list.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/COUNTER_Robots_list.json b/COUNTER_Robots_list.json index 45459c1..5169605 100644 --- a/COUNTER_Robots_list.json +++ b/COUNTER_Robots_list.json @@ -661,7 +661,8 @@ "last_changed": "2018-12-13" }, { - "pattern": "mail.ru", + "pattern": "mail\\.ru", + "url": "https://help.mail.ru/webmaster/indexing/robots.txt/rules/user-agent", "last_changed": "2017-08-08" }, { @@ -845,7 +846,7 @@ "last_changed": "2019-11-19" }, { - "pattern": "pear.php.net", + "pattern": "pear\\.php\\.net", "last_changed": "2017-08-08" }, { @@ -1095,7 +1096,9 @@ "last_changed": "2017-08-08" }, { - "pattern": "w3af.org", + "pattern": "w3af\\.org", + "description": "Web Application Attack and Audit Framework", + "url": "https://w3af.org/", "last_changed": "2017-08-08" }, {