From 0f322657775f8e8686b10298868d317a042b3ab2 Mon Sep 17 00:00:00 2001 From: Q_back Date: Fri, 31 Jan 2020 22:15:04 +0100 Subject: [PATCH] CDN detection plugin added --- w3af/core/data/kb/info_set.py | 3 + w3af/plugins/grep/cdn_providers.py | 109 ++++++++++++++++++ w3af/plugins/tests/grep/test_cdn_providers.py | 48 ++++++++ 3 files changed, 160 insertions(+) create mode 100644 w3af/plugins/grep/cdn_providers.py create mode 100644 w3af/plugins/tests/grep/test_cdn_providers.py diff --git a/w3af/core/data/kb/info_set.py b/w3af/core/data/kb/info_set.py index ae629ddb9f..2a3bdc5aa9 100644 --- a/w3af/core/data/kb/info_set.py +++ b/w3af/core/data/kb/info_set.py @@ -70,6 +70,9 @@ class InfoSet(object): it would have been harder to refactor the whole code and the end result would have been difficult to read. + ITAG defines the name of the key in Info() instance. This key will be used + to group Info() instances together. + Note that: * It can hold both Info and Vuln instances. diff --git a/w3af/plugins/grep/cdn_providers.py b/w3af/plugins/grep/cdn_providers.py new file mode 100644 index 0000000000..85a336139e --- /dev/null +++ b/w3af/plugins/grep/cdn_providers.py @@ -0,0 +1,109 @@ +from w3af.core.controllers.plugins.grep_plugin import GrepPlugin +from w3af.core.data.kb.info import Info +from w3af.core.data.kb.info_set import InfoSet + + +class cdn_providers(GrepPlugin): + """ + Check CDN (Content Delivery Network) providers used by the website. + + :author: Jakub Peczke (qback.contact@gmail.com) + """ + # CDN headers stored in format ['header-name', 'header-value', 'provider's name'] + cdn_headers = ( + ["server", "cloudflare", "Cloudflare"], + ["server", "yunjiasu", "Yunjiasu"], + ["server", "ECS", "Edgecast"], + ["server", "ECAcc", "Edgecast"], + ["server", "ECD", "Edgecast"], + ["server", "NetDNA", "NetDNA"], + ["server", "Airee", "Airee"], + ["X-CDN-Geo", "", "OVH CDN"], + ["X-CDN-Pop", "", "OVH CDN"], + ["X-Px", "", "CDNetworks"], + ["X-Instart-Request-ID", "instart", "Instart Logic"], + ["Via", "CloudFront", "Amazon CloudFront"], + ["X-Edge-IP", "", "CDN"], + ["X-Edge-Location", "", "CDN"], + ["X-HW", "", "Highwinds"], + ["X-Powered-By", "NYI FTW", "NYI FTW"], + ["X-Delivered-By", "NYI FTW", "NYI FTW"], + ["server", "ReSRC", "ReSRC.it"], + ["X-Cdn", "Zenedge", "Zenedge"], + ["server", "leasewebcdn", "LeaseWeb CDN"], + ["Via", "Rev-Cache", "Rev Software"], + ["X-Rev-Cache", "", "Rev Software"], + ["Server", "Caspowa", "Caspowa"], + ["Server", "SurgeCDN", "Surge"], + ["server", "sffe", "Google"], + ["server", "gws", "Google"], + ["server", "GSE", "Google"], + ["server", "Golfe2", "Google"], + ["Via", "google", "Google"], + ["server", "tsa_b", "Twitter"], + ["X-Cache", "cache.51cdn.com", "ChinaNetCenter"], + ["X-CDN", "Incapsula", "Incapsula"], + ["X-Iinfo", "", "Incapsula"], + ["X-Ar-Debug", "", "Aryaka"], + ["server", "gocache", "GoCache"], + ["server", "hiberniacdn", "HiberniaCDN"], + ["server", "UnicornCDN", "UnicornCDN"], + ["server", "Optimal CDN", "Optimal CDN"], + ["server", "Sucuri/Cloudproxy", "Sucuri Firewall"], + ["x-sucuri-id", "", "Sucuri Firewall"], + ["server", "Netlify", "Netlify"], + ["section-io-id", "", "section.io"], + ["server", "Testa/", "Naver"], + ["server", "BunnyCDN", "BunnyCDN"], + ["server", "MNCDN", "Medianova"], + ["server", "Roast.io", "Roast.io"], + ["x-rocket-node", "", "Rocket CDN"], + ) + + def grep(self, request, response): + """ + Check if all responses has CDN header included + """ + headers = response.get_headers() + for cdn_header in self.cdn_headers: + cdn_header_name = cdn_header[0] + if cdn_header_name in headers: + if cdn_header[1] == headers[cdn_header_name]: + detected_cdn_provider = cdn_header[2] + description = 'The URL {} is served using CDN provider: {}'.format( + response.get_url(), + detected_cdn_provider + ) + info = Info( + 'Content Delivery Network Provider detected', + description, + response.id, + self.get_name() + ) + info[CDNProvidersInfoSet.ITAG] = detected_cdn_provider + info.set_url(response.get_url()) + self.kb_append_uniq_group( + self, + 'cdn_providers', + info, + group_klass=CDNProvidersInfoSet + ) + + def get_long_desc(self): + """ + :return: A DETAILED description of the plugin functions and features. + """ + return """ + Check CDN (Content Delivery Network) providers used by the website. + """ + + +class CDNProvidersInfoSet(InfoSet): + ITAG = 'provider' + TEMPLATE = ( + 'The remote web server sent {{ uris|length }} HTTP responses ' + 'recognized as served by {{ provider }}. The first ten URLs are: \n' + '{% for url in uris[:10] %}' + ' - {{ url }}\n' + '{% endfor %}' + ) diff --git a/w3af/plugins/tests/grep/test_cdn_providers.py b/w3af/plugins/tests/grep/test_cdn_providers.py new file mode 100644 index 0000000000..00704963c7 --- /dev/null +++ b/w3af/plugins/tests/grep/test_cdn_providers.py @@ -0,0 +1,48 @@ +import unittest + +from w3af.core.data.dc.headers import Headers +from w3af.core.data.kb.knowledge_base import kb +from w3af.core.data.parsers.doc.url import URL +from w3af.core.data.request.fuzzable_request import FuzzableRequest +from w3af.core.data.url.HTTPResponse import HTTPResponse +from w3af.plugins.grep.cdn_providers import cdn_providers + + +class TestCDNProviders(unittest.TestCase): + def setUp(self): + self.plugin = cdn_providers() + kb.clear('cdn_providers', 'cdn_providers') + + def tearDown(self): + self.plugin.end() + + def test_if_cdn_can_be_detected(self): + url = URL('https://example.com/') + headers = Headers([('server', 'Netlify')]) + response = HTTPResponse(200, '', headers, url, url, _id=1) + request = FuzzableRequest(url, method='GET') + self.plugin.grep(request, response) + self.assertEqual(len(kb.get('cdn_providers', 'cdn_providers')), 1) + + def test_if_cdns_are_grouped_by_provider_name(self): + netlify_header = Headers([('server', 'Netlify')]) + cloudflare_header = Headers([('server', 'cloudflare')]) + + url = URL('https://example.com/') + request = FuzzableRequest(url, method='GET') + response = HTTPResponse(200, '', netlify_header, url, url, _id=1) + self.plugin.grep(request, response) + + # this request should be grouped with the request above + url = URL('https://example.com/another-netflify/') + request = FuzzableRequest(url, method='GET') + response = HTTPResponse(200, '', netlify_header, url, url, _id=2) + self.plugin.grep(request, response) + + # this request should be stored separately in kb as it comes from another provider + url = URL('https://example.com/cloudflare/') + request = FuzzableRequest(url, method='GET') + response = HTTPResponse(200, '', cloudflare_header, url, url, _id=3) + self.plugin.grep(request, response) + + self.assertEqual(len(kb.get('cdn_providers', 'cdn_providers')), 2)