From 0f322657775f8e8686b10298868d317a042b3ab2 Mon Sep 17 00:00:00 2001 From: Q_back Date: Fri, 31 Jan 2020 22:15:04 +0100 Subject: [PATCH 1/3] CDN detection plugin added --- w3af/core/data/kb/info_set.py | 3 + w3af/plugins/grep/cdn_providers.py | 109 ++++++++++++++++++ w3af/plugins/tests/grep/test_cdn_providers.py | 48 ++++++++ 3 files changed, 160 insertions(+) create mode 100644 w3af/plugins/grep/cdn_providers.py create mode 100644 w3af/plugins/tests/grep/test_cdn_providers.py diff --git a/w3af/core/data/kb/info_set.py b/w3af/core/data/kb/info_set.py index ae629ddb9f..2a3bdc5aa9 100644 --- a/w3af/core/data/kb/info_set.py +++ b/w3af/core/data/kb/info_set.py @@ -70,6 +70,9 @@ class InfoSet(object): it would have been harder to refactor the whole code and the end result would have been difficult to read. + ITAG defines the name of the key in Info() instance. This key will be used + to group Info() instances together. + Note that: * It can hold both Info and Vuln instances. diff --git a/w3af/plugins/grep/cdn_providers.py b/w3af/plugins/grep/cdn_providers.py new file mode 100644 index 0000000000..85a336139e --- /dev/null +++ b/w3af/plugins/grep/cdn_providers.py @@ -0,0 +1,109 @@ +from w3af.core.controllers.plugins.grep_plugin import GrepPlugin +from w3af.core.data.kb.info import Info +from w3af.core.data.kb.info_set import InfoSet + + +class cdn_providers(GrepPlugin): + """ + Check CDN (Content Delivery Network) providers used by the website. + + :author: Jakub Peczke (qback.contact@gmail.com) + """ + # CDN headers stored in format ['header-name', 'header-value', 'provider's name'] + cdn_headers = ( + ["server", "cloudflare", "Cloudflare"], + ["server", "yunjiasu", "Yunjiasu"], + ["server", "ECS", "Edgecast"], + ["server", "ECAcc", "Edgecast"], + ["server", "ECD", "Edgecast"], + ["server", "NetDNA", "NetDNA"], + ["server", "Airee", "Airee"], + ["X-CDN-Geo", "", "OVH CDN"], + ["X-CDN-Pop", "", "OVH CDN"], + ["X-Px", "", "CDNetworks"], + ["X-Instart-Request-ID", "instart", "Instart Logic"], + ["Via", "CloudFront", "Amazon CloudFront"], + ["X-Edge-IP", "", "CDN"], + ["X-Edge-Location", "", "CDN"], + ["X-HW", "", "Highwinds"], + ["X-Powered-By", "NYI FTW", "NYI FTW"], + ["X-Delivered-By", "NYI FTW", "NYI FTW"], + ["server", "ReSRC", "ReSRC.it"], + ["X-Cdn", "Zenedge", "Zenedge"], + ["server", "leasewebcdn", "LeaseWeb CDN"], + ["Via", "Rev-Cache", "Rev Software"], + ["X-Rev-Cache", "", "Rev Software"], + ["Server", "Caspowa", "Caspowa"], + ["Server", "SurgeCDN", "Surge"], + ["server", "sffe", "Google"], + ["server", "gws", "Google"], + ["server", "GSE", "Google"], + ["server", "Golfe2", "Google"], + ["Via", "google", "Google"], + ["server", "tsa_b", "Twitter"], + ["X-Cache", "cache.51cdn.com", "ChinaNetCenter"], + ["X-CDN", "Incapsula", "Incapsula"], + ["X-Iinfo", "", "Incapsula"], + ["X-Ar-Debug", "", "Aryaka"], + ["server", "gocache", "GoCache"], + ["server", "hiberniacdn", "HiberniaCDN"], + ["server", "UnicornCDN", "UnicornCDN"], + ["server", "Optimal CDN", "Optimal CDN"], + ["server", "Sucuri/Cloudproxy", "Sucuri Firewall"], + ["x-sucuri-id", "", "Sucuri Firewall"], + ["server", "Netlify", "Netlify"], + ["section-io-id", "", "section.io"], + ["server", "Testa/", "Naver"], + ["server", "BunnyCDN", "BunnyCDN"], + ["server", "MNCDN", "Medianova"], + ["server", "Roast.io", "Roast.io"], + ["x-rocket-node", "", "Rocket CDN"], + ) + + def grep(self, request, response): + """ + Check if all responses has CDN header included + """ + headers = response.get_headers() + for cdn_header in self.cdn_headers: + cdn_header_name = cdn_header[0] + if cdn_header_name in headers: + if cdn_header[1] == headers[cdn_header_name]: + detected_cdn_provider = cdn_header[2] + description = 'The URL {} is served using CDN provider: {}'.format( + response.get_url(), + detected_cdn_provider + ) + info = Info( + 'Content Delivery Network Provider detected', + description, + response.id, + self.get_name() + ) + info[CDNProvidersInfoSet.ITAG] = detected_cdn_provider + info.set_url(response.get_url()) + self.kb_append_uniq_group( + self, + 'cdn_providers', + info, + group_klass=CDNProvidersInfoSet + ) + + def get_long_desc(self): + """ + :return: A DETAILED description of the plugin functions and features. + """ + return """ + Check CDN (Content Delivery Network) providers used by the website. + """ + + +class CDNProvidersInfoSet(InfoSet): + ITAG = 'provider' + TEMPLATE = ( + 'The remote web server sent {{ uris|length }} HTTP responses ' + 'recognized as served by {{ provider }}. The first ten URLs are: \n' + '{% for url in uris[:10] %}' + ' - {{ url }}\n' + '{% endfor %}' + ) diff --git a/w3af/plugins/tests/grep/test_cdn_providers.py b/w3af/plugins/tests/grep/test_cdn_providers.py new file mode 100644 index 0000000000..00704963c7 --- /dev/null +++ b/w3af/plugins/tests/grep/test_cdn_providers.py @@ -0,0 +1,48 @@ +import unittest + +from w3af.core.data.dc.headers import Headers +from w3af.core.data.kb.knowledge_base import kb +from w3af.core.data.parsers.doc.url import URL +from w3af.core.data.request.fuzzable_request import FuzzableRequest +from w3af.core.data.url.HTTPResponse import HTTPResponse +from w3af.plugins.grep.cdn_providers import cdn_providers + + +class TestCDNProviders(unittest.TestCase): + def setUp(self): + self.plugin = cdn_providers() + kb.clear('cdn_providers', 'cdn_providers') + + def tearDown(self): + self.plugin.end() + + def test_if_cdn_can_be_detected(self): + url = URL('https://example.com/') + headers = Headers([('server', 'Netlify')]) + response = HTTPResponse(200, '', headers, url, url, _id=1) + request = FuzzableRequest(url, method='GET') + self.plugin.grep(request, response) + self.assertEqual(len(kb.get('cdn_providers', 'cdn_providers')), 1) + + def test_if_cdns_are_grouped_by_provider_name(self): + netlify_header = Headers([('server', 'Netlify')]) + cloudflare_header = Headers([('server', 'cloudflare')]) + + url = URL('https://example.com/') + request = FuzzableRequest(url, method='GET') + response = HTTPResponse(200, '', netlify_header, url, url, _id=1) + self.plugin.grep(request, response) + + # this request should be grouped with the request above + url = URL('https://example.com/another-netflify/') + request = FuzzableRequest(url, method='GET') + response = HTTPResponse(200, '', netlify_header, url, url, _id=2) + self.plugin.grep(request, response) + + # this request should be stored separately in kb as it comes from another provider + url = URL('https://example.com/cloudflare/') + request = FuzzableRequest(url, method='GET') + response = HTTPResponse(200, '', cloudflare_header, url, url, _id=3) + self.plugin.grep(request, response) + + self.assertEqual(len(kb.get('cdn_providers', 'cdn_providers')), 2) From 1fd3ad281270e58c6c8b5c3f0ed83241994fde13 Mon Sep 17 00:00:00 2001 From: Q_back Date: Mon, 3 Feb 2020 14:03:47 +0100 Subject: [PATCH 2/3] added option to detect CDN by URL --- w3af/plugins/grep/cdn_providers.py | 186 ++++++++++++++++-- w3af/plugins/tests/grep/test_cdn_providers.py | 8 + 2 files changed, 174 insertions(+), 20 deletions(-) diff --git a/w3af/plugins/grep/cdn_providers.py b/w3af/plugins/grep/cdn_providers.py index 85a336139e..a3ffc4341a 100644 --- a/w3af/plugins/grep/cdn_providers.py +++ b/w3af/plugins/grep/cdn_providers.py @@ -60,34 +60,180 @@ class cdn_providers(GrepPlugin): ["x-rocket-node", "", "Rocket CDN"], ) + # CDN domains stored in format ['required part of a domain', 'provider's name'] + cdn_domains = [ + [".akamai.net", "Akamai"], + [".akamaized.net", "Akamai"], + [".akamaiedge.net", "Akamai"], + [".akamaihd.net", "Akamai"], + [".edgesuite.net", "Akamai"], + [".edgekey.net", "Akamai"], + [".srip.net", "Akamai"], + [".akamaitechnologies.com", "Akamai"], + [".akamaitechnologies.fr", "Akamai"], + [".tl88.net", "Akamai China CDN"], + [".llnwd.net", "Limelight"], + ["edgecastcdn.net", "Edgecast"], + [".systemcdn.net", "Edgecast"], + [".transactcdn.net", "Edgecast"], + [".v1cdn.net", "Edgecast"], + [".v2cdn.net", "Edgecast"], + [".v3cdn.net", "Edgecast"], + [".v4cdn.net", "Edgecast"], + [".v5cdn.net", "Edgecast"], + ["hwcdn.net", "Highwinds"], + [".simplecdn.net", "Simple CDN"], + [".instacontent.net", "Mirror Image"], + [".footprint.net", "Level 3"], + [".fpbns.net", "Level 3"], + [".ay1.b.yahoo.com", "Yahoo"], + [".yimg.", "Yahoo"], + [".yahooapis.com", "Yahoo"], + [".google.", "Google"], + ["googlesyndication.", "Google"], + ["youtube.", "Google"], + [".googleusercontent.com", "Google"], + ["googlehosted.com", "Google"], + [".gstatic.com", "Google"], + [".doubleclick.net", "Google"], + [".insnw.net", "Instart Logic"], + [".inscname.net", "Instart Logic"], + [".internapcdn.net", "Internap"], + [".cloudfront.net", "Amazon CloudFront"], + [".netdna-cdn.com", "NetDNA"], + [".netdna-ssl.com", "NetDNA"], + [".netdna.com", "NetDNA"], + [".kxcdn.com", "KeyCDN"], + [".cotcdn.net", "Cotendo CDN"], + [".cachefly.net", "Cachefly"], + ["bo.lt", "BO.LT"], + [".cloudflare.com", "Cloudflare"], + [".afxcdn.net", "afxcdn.net"], + [".lxdns.com", "ChinaNetCenter"], + [".wscdns.com", "ChinaNetCenter"], + [".wscloudcdn.com", "ChinaNetCenter"], + [".ourwebpic.com", "ChinaNetCenter"], + [".att-dsa.net", "AT&T"], + [".vo.msecnd.net", "Microsoft Azure"], + [".azureedge.net", "Microsoft Azure"], + [".azure.microsoft.com", "Microsoft Azure"], + [".voxcdn.net", "VoxCDN"], + [".bluehatnetwork.com", "Blue Hat Network"], + [".swiftcdn1.com", "SwiftCDN"], + [".cdngc.net", "CDNetworks"], + [".gccdn.net", "CDNetworks"], + [".panthercdn.com", "CDNetworks"], + [".fastly.net", "Fastly"], + [".fastlylb.net", "Fastly"], + [".nocookie.net", "Fastly"], + [".gslb.taobao.com", "Taobao"], + [".gslb.tbcache.com", "Alimama"], + [".mirror-image.net", "Mirror Image"], + [".yottaa.net", "Yottaa"], + [".cubecdn.net", "cubeCDN"], + [".cdn77.net", "CDN77"], + [".cdn77.org", "CDN77"], + [".incapdns.net", "Incapsula"], + [".bitgravity.com", "BitGravity"], + [".r.worldcdn.net", "OnApp"], + [".r.worldssl.net", "OnApp"], + ["tbcdn.cn", "Taobao"], + [".taobaocdn.com", "Taobao"], + [".ngenix.net", "NGENIX"], + [".pagerain.net", "PageRain"], + [".ccgslb.com", "ChinaCache"], + ["cdn.sfr.net", "SFR"], + [".azioncdn.net", "Azion"], + [".azioncdn.com", "Azion"], + [".azion.net", "Azion"], + [".cdncloud.net.au", "MediaCloud"], + [".rncdn1.com", "Reflected Networks"], + [".cdnsun.net", "CDNsun"], + [".mncdn.com", "Medianova"], + [".mncdn.net", "Medianova"], + [".mncdn.org", "Medianova"], + ["cdn.jsdelivr.net", "jsDelivr"], + [".nyiftw.net", "NYI FTW"], + [".nyiftw.com", "NYI FTW"], + [".resrc.it", "ReSRC.it"], + [".zenedge.net", "Zenedge"], + [".lswcdn.net", "LeaseWeb CDN"], + [".lswcdn.eu", "LeaseWeb CDN"], + [".revcn.net", "Rev Software"], + [".revdn.net", "Rev Software"], + [".caspowa.com", "Caspowa"], + [".twimg.com", "Twitter"], + [".facebook.com", "Facebook"], + [".facebook.net", "Facebook"], + [".fbcdn.net", "Facebook"], + [".cdninstagram.com", "Facebook"], + [".rlcdn.com", "Reapleaf"], + [".wp.com", "WordPress"], + [".aads1.net", "Aryaka"], + [".aads-cn.net", "Aryaka"], + [".aads-cng.net", "Aryaka"], + [".squixa.net", "section.io"], + [".bisongrid.net", "Bison Grid"], + [".cdn.gocache.net", "GoCache"], + [".hiberniacdn.com", "HiberniaCDN"], + [".cdntel.net", "Telenor"], + [".raxcdn.com", "Rackspace"], + [".unicorncdn.net", "UnicornCDN"], + [".optimalcdn.com", "Optimal CDN"], + [".kinxcdn.com", "KINX CDN"], + [".kinxcdn.net", "KINX CDN"], + [".stackpathdns.com", "StackPath"], + [".hosting4cdn.com", "Hosting4CDN"], + [".netlify.com", "Netlify"], + [".b-cdn.net", "BunnyCDN"], + [".pix-cdn.org", "Advanced Hosters CDN"], + [".roast.io", "Roast.io"], + [".cdnvideo.ru", "CDNvideo"], + [".cdnvideo.net", "CDNvideo"], + [".trbcdn.ru", "TRBCDN"], + [".cedexis.net", "Cedexis"], + [".streamprovider.net", "Rocket CDN"], + ] + def grep(self, request, response): """ - Check if all responses has CDN header included + Check if responses are hosted by CDN provider """ headers = response.get_headers() for cdn_header in self.cdn_headers: cdn_header_name = cdn_header[0] if cdn_header_name in headers: if cdn_header[1] == headers[cdn_header_name]: - detected_cdn_provider = cdn_header[2] - description = 'The URL {} is served using CDN provider: {}'.format( - response.get_url(), - detected_cdn_provider - ) - info = Info( - 'Content Delivery Network Provider detected', - description, - response.id, - self.get_name() - ) - info[CDNProvidersInfoSet.ITAG] = detected_cdn_provider - info.set_url(response.get_url()) - self.kb_append_uniq_group( - self, - 'cdn_providers', - info, - group_klass=CDNProvidersInfoSet - ) + self._detect_cdn(response, cdn_header[2]) + + url = request.get_url() + domain = url.get_domain() + for cdn_domain in self.cdn_domains: + if cdn_domain[0] in domain: + self._detect_cdn(response, cdn_domain[1]) + + def _detect_cdn(self, response, provider_name): + """ + Save grep result to knowledge base + """ + description = 'The URL {} is served using CDN provider: {}'.format( + response.get_url(), + provider_name + ) + info = Info( + 'Content Delivery Network Provider detected', + description, + response.id, + self.get_name() + ) + info[CDNProvidersInfoSet.ITAG] = provider_name + info.set_url(response.get_url()) + self.kb_append_uniq_group( + self, + 'cdn_providers', + info, + group_klass=CDNProvidersInfoSet + ) def get_long_desc(self): """ diff --git a/w3af/plugins/tests/grep/test_cdn_providers.py b/w3af/plugins/tests/grep/test_cdn_providers.py index 00704963c7..aab4e31cc4 100644 --- a/w3af/plugins/tests/grep/test_cdn_providers.py +++ b/w3af/plugins/tests/grep/test_cdn_providers.py @@ -46,3 +46,11 @@ def test_if_cdns_are_grouped_by_provider_name(self): self.plugin.grep(request, response) self.assertEqual(len(kb.get('cdn_providers', 'cdn_providers')), 2) + + def test_if_cdn_provider_can_be_detected_by_url(self): + url = URL('https://cdn.jsdelivr.net/npm/bootstrap@3.3.7/dist/css/bootstrap.min.css') + empty_header = Headers() + request = FuzzableRequest(url, method='GET') + response = HTTPResponse(200, '', empty_header, url, url, _id=1) + self.plugin.grep(request, response) + self.assertEqual(len(kb.get('cdn_providers', 'cdn_providers')), 1) From adc49d03ca7757c0df6060114c736477abb989bc Mon Sep 17 00:00:00 2001 From: Q_back Date: Tue, 11 Feb 2020 17:43:42 +0100 Subject: [PATCH 3/3] added copyright, CR changes --- w3af/plugins/grep/cdn_providers.py | 395 +++++++++--------- w3af/plugins/tests/grep/test_cdn_providers.py | 30 ++ 2 files changed, 238 insertions(+), 187 deletions(-) diff --git a/w3af/plugins/grep/cdn_providers.py b/w3af/plugins/grep/cdn_providers.py index a3ffc4341a..01a0fb61a2 100644 --- a/w3af/plugins/grep/cdn_providers.py +++ b/w3af/plugins/grep/cdn_providers.py @@ -1,3 +1,24 @@ +""" +cdn_providers.py + +Copyright 2020 Andres Riancho + +This file is part of w3af, http://w3af.org/ . + +w3af is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation version 2 of the License. + +w3af is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with w3af; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +""" from w3af.core.controllers.plugins.grep_plugin import GrepPlugin from w3af.core.data.kb.info import Info from w3af.core.data.kb.info_set import InfoSet @@ -11,188 +32,188 @@ class cdn_providers(GrepPlugin): """ # CDN headers stored in format ['header-name', 'header-value', 'provider's name'] cdn_headers = ( - ["server", "cloudflare", "Cloudflare"], - ["server", "yunjiasu", "Yunjiasu"], - ["server", "ECS", "Edgecast"], - ["server", "ECAcc", "Edgecast"], - ["server", "ECD", "Edgecast"], - ["server", "NetDNA", "NetDNA"], - ["server", "Airee", "Airee"], - ["X-CDN-Geo", "", "OVH CDN"], - ["X-CDN-Pop", "", "OVH CDN"], - ["X-Px", "", "CDNetworks"], - ["X-Instart-Request-ID", "instart", "Instart Logic"], - ["Via", "CloudFront", "Amazon CloudFront"], - ["X-Edge-IP", "", "CDN"], - ["X-Edge-Location", "", "CDN"], - ["X-HW", "", "Highwinds"], - ["X-Powered-By", "NYI FTW", "NYI FTW"], - ["X-Delivered-By", "NYI FTW", "NYI FTW"], - ["server", "ReSRC", "ReSRC.it"], - ["X-Cdn", "Zenedge", "Zenedge"], - ["server", "leasewebcdn", "LeaseWeb CDN"], - ["Via", "Rev-Cache", "Rev Software"], - ["X-Rev-Cache", "", "Rev Software"], - ["Server", "Caspowa", "Caspowa"], - ["Server", "SurgeCDN", "Surge"], - ["server", "sffe", "Google"], - ["server", "gws", "Google"], - ["server", "GSE", "Google"], - ["server", "Golfe2", "Google"], - ["Via", "google", "Google"], - ["server", "tsa_b", "Twitter"], - ["X-Cache", "cache.51cdn.com", "ChinaNetCenter"], - ["X-CDN", "Incapsula", "Incapsula"], - ["X-Iinfo", "", "Incapsula"], - ["X-Ar-Debug", "", "Aryaka"], - ["server", "gocache", "GoCache"], - ["server", "hiberniacdn", "HiberniaCDN"], - ["server", "UnicornCDN", "UnicornCDN"], - ["server", "Optimal CDN", "Optimal CDN"], - ["server", "Sucuri/Cloudproxy", "Sucuri Firewall"], - ["x-sucuri-id", "", "Sucuri Firewall"], - ["server", "Netlify", "Netlify"], - ["section-io-id", "", "section.io"], - ["server", "Testa/", "Naver"], - ["server", "BunnyCDN", "BunnyCDN"], - ["server", "MNCDN", "Medianova"], - ["server", "Roast.io", "Roast.io"], - ["x-rocket-node", "", "Rocket CDN"], + ['server', 'cloudflare', 'Cloudflare'], + ['server', 'yunjiasu', 'Yunjiasu'], + ['server', 'ECS', 'Edgecast'], + ['server', 'ECAcc', 'Edgecast'], + ['server', 'ECD', 'Edgecast'], + ['server', 'NetDNA', 'NetDNA'], + ['server', 'Airee', 'Airee'], + ['X-CDN-Geo', '', 'OVH CDN'], + ['X-CDN-Pop', '', 'OVH CDN'], + ['X-Px', '', 'CDNetworks'], + ['X-Instart-Request-ID', 'instart', 'Instart Logic'], + ['Via', 'CloudFront', 'Amazon CloudFront'], + ['X-Edge-IP', '', 'CDN'], + ['X-Edge-Location', '', 'CDN'], + ['X-HW', '', 'Highwinds'], + ['X-Powered-By', 'NYI FTW', 'NYI FTW'], + ['X-Delivered-By', 'NYI FTW', 'NYI FTW'], + ['server', 'ReSRC', 'ReSRC.it'], + ['X-Cdn', 'Zenedge', 'Zenedge'], + ['server', 'leasewebcdn', 'LeaseWeb CDN'], + ['Via', 'Rev-Cache', 'Rev Software'], + ['X-Rev-Cache', '', 'Rev Software'], + ['Server', 'Caspowa', 'Caspowa'], + ['Server', 'SurgeCDN', 'Surge'], + ['server', 'sffe', 'Google'], + ['server', 'gws', 'Google'], + ['server', 'GSE', 'Google'], + ['server', 'Golfe2', 'Google'], + ['Via', 'google', 'Google'], + ['server', 'tsa_b', 'Twitter'], + ['X-Cache', 'cache.51cdn.com', 'ChinaNetCenter'], + ['X-CDN', 'Incapsula', 'Incapsula'], + ['X-Iinfo', '', 'Incapsula'], + ['X-Ar-Debug', '', 'Aryaka'], + ['server', 'gocache', 'GoCache'], + ['server', 'hiberniacdn', 'HiberniaCDN'], + ['server', 'UnicornCDN', 'UnicornCDN'], + ['server', 'Optimal CDN', 'Optimal CDN'], + ['server', 'Sucuri/Cloudproxy', 'Sucuri Firewall'], + ['x-sucuri-id', '', 'Sucuri Firewall'], + ['server', 'Netlify', 'Netlify'], + ['section-io-id', '', 'section.io'], + ['server', 'Testa/', 'Naver'], + ['server', 'BunnyCDN', 'BunnyCDN'], + ['server', 'MNCDN', 'Medianova'], + ['server', 'Roast.io', 'Roast.io'], + ['x-rocket-node', '', 'Rocket CDN'], ) # CDN domains stored in format ['required part of a domain', 'provider's name'] cdn_domains = [ - [".akamai.net", "Akamai"], - [".akamaized.net", "Akamai"], - [".akamaiedge.net", "Akamai"], - [".akamaihd.net", "Akamai"], - [".edgesuite.net", "Akamai"], - [".edgekey.net", "Akamai"], - [".srip.net", "Akamai"], - [".akamaitechnologies.com", "Akamai"], - [".akamaitechnologies.fr", "Akamai"], - [".tl88.net", "Akamai China CDN"], - [".llnwd.net", "Limelight"], - ["edgecastcdn.net", "Edgecast"], - [".systemcdn.net", "Edgecast"], - [".transactcdn.net", "Edgecast"], - [".v1cdn.net", "Edgecast"], - [".v2cdn.net", "Edgecast"], - [".v3cdn.net", "Edgecast"], - [".v4cdn.net", "Edgecast"], - [".v5cdn.net", "Edgecast"], - ["hwcdn.net", "Highwinds"], - [".simplecdn.net", "Simple CDN"], - [".instacontent.net", "Mirror Image"], - [".footprint.net", "Level 3"], - [".fpbns.net", "Level 3"], - [".ay1.b.yahoo.com", "Yahoo"], - [".yimg.", "Yahoo"], - [".yahooapis.com", "Yahoo"], - [".google.", "Google"], - ["googlesyndication.", "Google"], - ["youtube.", "Google"], - [".googleusercontent.com", "Google"], - ["googlehosted.com", "Google"], - [".gstatic.com", "Google"], - [".doubleclick.net", "Google"], - [".insnw.net", "Instart Logic"], - [".inscname.net", "Instart Logic"], - [".internapcdn.net", "Internap"], - [".cloudfront.net", "Amazon CloudFront"], - [".netdna-cdn.com", "NetDNA"], - [".netdna-ssl.com", "NetDNA"], - [".netdna.com", "NetDNA"], - [".kxcdn.com", "KeyCDN"], - [".cotcdn.net", "Cotendo CDN"], - [".cachefly.net", "Cachefly"], - ["bo.lt", "BO.LT"], - [".cloudflare.com", "Cloudflare"], - [".afxcdn.net", "afxcdn.net"], - [".lxdns.com", "ChinaNetCenter"], - [".wscdns.com", "ChinaNetCenter"], - [".wscloudcdn.com", "ChinaNetCenter"], - [".ourwebpic.com", "ChinaNetCenter"], - [".att-dsa.net", "AT&T"], - [".vo.msecnd.net", "Microsoft Azure"], - [".azureedge.net", "Microsoft Azure"], - [".azure.microsoft.com", "Microsoft Azure"], - [".voxcdn.net", "VoxCDN"], - [".bluehatnetwork.com", "Blue Hat Network"], - [".swiftcdn1.com", "SwiftCDN"], - [".cdngc.net", "CDNetworks"], - [".gccdn.net", "CDNetworks"], - [".panthercdn.com", "CDNetworks"], - [".fastly.net", "Fastly"], - [".fastlylb.net", "Fastly"], - [".nocookie.net", "Fastly"], - [".gslb.taobao.com", "Taobao"], - [".gslb.tbcache.com", "Alimama"], - [".mirror-image.net", "Mirror Image"], - [".yottaa.net", "Yottaa"], - [".cubecdn.net", "cubeCDN"], - [".cdn77.net", "CDN77"], - [".cdn77.org", "CDN77"], - [".incapdns.net", "Incapsula"], - [".bitgravity.com", "BitGravity"], - [".r.worldcdn.net", "OnApp"], - [".r.worldssl.net", "OnApp"], - ["tbcdn.cn", "Taobao"], - [".taobaocdn.com", "Taobao"], - [".ngenix.net", "NGENIX"], - [".pagerain.net", "PageRain"], - [".ccgslb.com", "ChinaCache"], - ["cdn.sfr.net", "SFR"], - [".azioncdn.net", "Azion"], - [".azioncdn.com", "Azion"], - [".azion.net", "Azion"], - [".cdncloud.net.au", "MediaCloud"], - [".rncdn1.com", "Reflected Networks"], - [".cdnsun.net", "CDNsun"], - [".mncdn.com", "Medianova"], - [".mncdn.net", "Medianova"], - [".mncdn.org", "Medianova"], - ["cdn.jsdelivr.net", "jsDelivr"], - [".nyiftw.net", "NYI FTW"], - [".nyiftw.com", "NYI FTW"], - [".resrc.it", "ReSRC.it"], - [".zenedge.net", "Zenedge"], - [".lswcdn.net", "LeaseWeb CDN"], - [".lswcdn.eu", "LeaseWeb CDN"], - [".revcn.net", "Rev Software"], - [".revdn.net", "Rev Software"], - [".caspowa.com", "Caspowa"], - [".twimg.com", "Twitter"], - [".facebook.com", "Facebook"], - [".facebook.net", "Facebook"], - [".fbcdn.net", "Facebook"], - [".cdninstagram.com", "Facebook"], - [".rlcdn.com", "Reapleaf"], - [".wp.com", "WordPress"], - [".aads1.net", "Aryaka"], - [".aads-cn.net", "Aryaka"], - [".aads-cng.net", "Aryaka"], - [".squixa.net", "section.io"], - [".bisongrid.net", "Bison Grid"], - [".cdn.gocache.net", "GoCache"], - [".hiberniacdn.com", "HiberniaCDN"], - [".cdntel.net", "Telenor"], - [".raxcdn.com", "Rackspace"], - [".unicorncdn.net", "UnicornCDN"], - [".optimalcdn.com", "Optimal CDN"], - [".kinxcdn.com", "KINX CDN"], - [".kinxcdn.net", "KINX CDN"], - [".stackpathdns.com", "StackPath"], - [".hosting4cdn.com", "Hosting4CDN"], - [".netlify.com", "Netlify"], - [".b-cdn.net", "BunnyCDN"], - [".pix-cdn.org", "Advanced Hosters CDN"], - [".roast.io", "Roast.io"], - [".cdnvideo.ru", "CDNvideo"], - [".cdnvideo.net", "CDNvideo"], - [".trbcdn.ru", "TRBCDN"], - [".cedexis.net", "Cedexis"], - [".streamprovider.net", "Rocket CDN"], + ['.akamai.net', 'Akamai'], + ['.akamaized.net', 'Akamai'], + ['.akamaiedge.net', 'Akamai'], + ['.akamaihd.net', 'Akamai'], + ['.edgesuite.net', 'Akamai'], + ['.edgekey.net', 'Akamai'], + ['.srip.net', 'Akamai'], + ['.akamaitechnologies.com', 'Akamai'], + ['.akamaitechnologies.fr', 'Akamai'], + ['.tl88.net', 'Akamai China CDN'], + ['.llnwd.net', 'Limelight'], + ['edgecastcdn.net', 'Edgecast'], + ['.systemcdn.net', 'Edgecast'], + ['.transactcdn.net', 'Edgecast'], + ['.v1cdn.net', 'Edgecast'], + ['.v2cdn.net', 'Edgecast'], + ['.v3cdn.net', 'Edgecast'], + ['.v4cdn.net', 'Edgecast'], + ['.v5cdn.net', 'Edgecast'], + ['hwcdn.net', 'Highwinds'], + ['.simplecdn.net', 'Simple CDN'], + ['.instacontent.net', 'Mirror Image'], + ['.footprint.net', 'Level 3'], + ['.fpbns.net', 'Level 3'], + ['.ay1.b.yahoo.com', 'Yahoo'], + ['.yimg.', 'Yahoo'], + ['.yahooapis.com', 'Yahoo'], + ['.google.', 'Google'], + ['googlesyndication.', 'Google'], + ['youtube.', 'Google'], + ['.googleusercontent.com', 'Google'], + ['googlehosted.com', 'Google'], + ['.gstatic.com', 'Google'], + ['.doubleclick.net', 'Google'], + ['.insnw.net', 'Instart Logic'], + ['.inscname.net', 'Instart Logic'], + ['.internapcdn.net', 'Internap'], + ['.cloudfront.net', 'Amazon CloudFront'], + ['.netdna-cdn.com', 'NetDNA'], + ['.netdna-ssl.com', 'NetDNA'], + ['.netdna.com', 'NetDNA'], + ['.kxcdn.com', 'KeyCDN'], + ['.cotcdn.net', 'Cotendo CDN'], + ['.cachefly.net', 'Cachefly'], + ['bo.lt', 'BO.LT'], + ['.cloudflare.com', 'Cloudflare'], + ['.afxcdn.net', 'afxcdn.net'], + ['.lxdns.com', 'ChinaNetCenter'], + ['.wscdns.com', 'ChinaNetCenter'], + ['.wscloudcdn.com', 'ChinaNetCenter'], + ['.ourwebpic.com', 'ChinaNetCenter'], + ['.att-dsa.net', 'AT&T'], + ['.vo.msecnd.net', 'Microsoft Azure'], + ['.azureedge.net', 'Microsoft Azure'], + ['.azure.microsoft.com', 'Microsoft Azure'], + ['.voxcdn.net', 'VoxCDN'], + ['.bluehatnetwork.com', 'Blue Hat Network'], + ['.swiftcdn1.com', 'SwiftCDN'], + ['.cdngc.net', 'CDNetworks'], + ['.gccdn.net', 'CDNetworks'], + ['.panthercdn.com', 'CDNetworks'], + ['.fastly.net', 'Fastly'], + ['.fastlylb.net', 'Fastly'], + ['.nocookie.net', 'Fastly'], + ['.gslb.taobao.com', 'Taobao'], + ['.gslb.tbcache.com', 'Alimama'], + ['.mirror-image.net', 'Mirror Image'], + ['.yottaa.net', 'Yottaa'], + ['.cubecdn.net', 'cubeCDN'], + ['.cdn77.net', 'CDN77'], + ['.cdn77.org', 'CDN77'], + ['.incapdns.net', 'Incapsula'], + ['.bitgravity.com', 'BitGravity'], + ['.r.worldcdn.net', 'OnApp'], + ['.r.worldssl.net', 'OnApp'], + ['tbcdn.cn', 'Taobao'], + ['.taobaocdn.com', 'Taobao'], + ['.ngenix.net', 'NGENIX'], + ['.pagerain.net', 'PageRain'], + ['.ccgslb.com', 'ChinaCache'], + ['cdn.sfr.net', 'SFR'], + ['.azioncdn.net', 'Azion'], + ['.azioncdn.com', 'Azion'], + ['.azion.net', 'Azion'], + ['.cdncloud.net.au', 'MediaCloud'], + ['.rncdn1.com', 'Reflected Networks'], + ['.cdnsun.net', 'CDNsun'], + ['.mncdn.com', 'Medianova'], + ['.mncdn.net', 'Medianova'], + ['.mncdn.org', 'Medianova'], + ['cdn.jsdelivr.net', 'jsDelivr'], + ['.nyiftw.net', 'NYI FTW'], + ['.nyiftw.com', 'NYI FTW'], + ['.resrc.it', 'ReSRC.it'], + ['.zenedge.net', 'Zenedge'], + ['.lswcdn.net', 'LeaseWeb CDN'], + ['.lswcdn.eu', 'LeaseWeb CDN'], + ['.revcn.net', 'Rev Software'], + ['.revdn.net', 'Rev Software'], + ['.caspowa.com', 'Caspowa'], + ['.twimg.com', 'Twitter'], + ['.facebook.com', 'Facebook'], + ['.facebook.net', 'Facebook'], + ['.fbcdn.net', 'Facebook'], + ['.cdninstagram.com', 'Facebook'], + ['.rlcdn.com', 'Reapleaf'], + ['.wp.com', 'WordPress'], + ['.aads1.net', 'Aryaka'], + ['.aads-cn.net', 'Aryaka'], + ['.aads-cng.net', 'Aryaka'], + ['.squixa.net', 'section.io'], + ['.bisongrid.net', 'Bison Grid'], + ['.cdn.gocache.net', 'GoCache'], + ['.hiberniacdn.com', 'HiberniaCDN'], + ['.cdntel.net', 'Telenor'], + ['.raxcdn.com', 'Rackspace'], + ['.unicorncdn.net', 'UnicornCDN'], + ['.optimalcdn.com', 'Optimal CDN'], + ['.kinxcdn.com', 'KINX CDN'], + ['.kinxcdn.net', 'KINX CDN'], + ['.stackpathdns.com', 'StackPath'], + ['.hosting4cdn.com', 'Hosting4CDN'], + ['.netlify.com', 'Netlify'], + ['.b-cdn.net', 'BunnyCDN'], + ['.pix-cdn.org', 'Advanced Hosters CDN'], + ['.roast.io', 'Roast.io'], + ['.cdnvideo.ru', 'CDNvideo'], + ['.cdnvideo.net', 'CDNvideo'], + ['.trbcdn.ru', 'TRBCDN'], + ['.cedexis.net', 'Cedexis'], + ['.streamprovider.net', 'Rocket CDN'], ] def grep(self, request, response): @@ -200,19 +221,19 @@ def grep(self, request, response): Check if responses are hosted by CDN provider """ headers = response.get_headers() - for cdn_header in self.cdn_headers: - cdn_header_name = cdn_header[0] + for cdn_header_name, cdn_header_value, provider_name in self.cdn_headers: if cdn_header_name in headers: - if cdn_header[1] == headers[cdn_header_name]: - self._detect_cdn(response, cdn_header[2]) + if cdn_header_value == headers[cdn_header_name]: + self._save_found_cdn_to_kb(response, provider_name) + return # We don't want to check the URL also url = request.get_url() domain = url.get_domain() - for cdn_domain in self.cdn_domains: - if cdn_domain[0] in domain: - self._detect_cdn(response, cdn_domain[1]) + for cdn_domain, provider_name in self.cdn_domains: + if cdn_domain in domain: + self._save_found_cdn_to_kb(response, provider_name) - def _detect_cdn(self, response, provider_name): + def _save_found_cdn_to_kb(self, response, provider_name): """ Save grep result to knowledge base """ @@ -240,7 +261,7 @@ def get_long_desc(self): :return: A DETAILED description of the plugin functions and features. """ return """ - Check CDN (Content Delivery Network) providers used by the website. + Identify any CDN (Content Delivery Network) providers used by the target website. """ diff --git a/w3af/plugins/tests/grep/test_cdn_providers.py b/w3af/plugins/tests/grep/test_cdn_providers.py index aab4e31cc4..af4061a1ee 100644 --- a/w3af/plugins/tests/grep/test_cdn_providers.py +++ b/w3af/plugins/tests/grep/test_cdn_providers.py @@ -1,3 +1,24 @@ +""" +cdn_providers.py + +Copyright 2020 Andres Riancho + +This file is part of w3af, http://w3af.org/ . + +w3af is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation version 2 of the License. + +w3af is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with w3af; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +""" import unittest from w3af.core.data.dc.headers import Headers @@ -24,6 +45,15 @@ def test_if_cdn_can_be_detected(self): self.plugin.grep(request, response) self.assertEqual(len(kb.get('cdn_providers', 'cdn_providers')), 1) + def test_if_wrong_cdn_info_is_not_detected(self): + url = URL('https://example.com/') + headers = Headers([('server', 'Netlifo')]) # There's no Netlifo provider, + # so it shouldn't be detected + response = HTTPResponse(200, '', headers, url, url, _id=1) + request = FuzzableRequest(url, method='GET') + self.plugin.grep(request, response) + self.assertEqual(len(kb.get('cdn_providers', 'cdn_providers')), 0) + def test_if_cdns_are_grouped_by_provider_name(self): netlify_header = Headers([('server', 'Netlify')]) cloudflare_header = Headers([('server', 'cloudflare')])