Skip tests that use a site which is failing

Add 'hostname' to TestCase.sites dict, and skip class when
failed to receive a useful response for the hosts homepage.

Re-enable weblib tests on travis-ci, instead using the new
automatic skip functionality to avoid errors due to WebCite
serving invalid XML and having an undecodable homepage.

Bug: T58963
Change-Id: I8fdcdaa0fab3b680d35b81b20a12ff5b786f779d

pywikibot/weblib.py

@@ -36,7 +36,7 @@ def getInternetArchiveURL(url, timestamp=None):
         query['timestamp'] = timestamp
 
     uri = uri + urlencode(query)
-    jsontext = http.request(uri=uri, site=None)
+    jsontext = http.fetch(uri).content
     if "closest" in jsontext:
         data = json.loads(jsontext)
         return data['archived_snapshots']['closest']['url']
@@ -65,7 +65,7 @@ def getWebCitationURL(url, timestamp=None):
         query['date'] = timestamp
 
     uri = uri + urlencode(query)
-    xmltext = http.request(uri=uri, site=None)
+    xmltext = http.fetch(uri).content
     if "success" in xmltext:
         data = ET.fromstring(xmltext)
         return data.find('.//webcite_url').text

tests/aspects.py

@@ -40,6 +40,7 @@
 from pywikibot import config, log, Site
 from pywikibot.site import BaseSite
 from pywikibot.family import WikimediaFamily
+from pywikibot.comms import http
 from pywikibot.data.api import Request as _original_Request
 
 import tests
@@ -382,6 +383,55 @@ def tearDown(self):
         super(CacheInfoMixin, self).tearDown()
 
 
+class CheckHostnameMixin(TestCaseBase):
+
+    """Check the hostname is online before running tests."""
+
+    _checked_hostnames = {}
+
+    @classmethod
+    def setUpClass(cls):
+        """
+        Set up the test class.
+
+        Prevent tests running if the host is down.
+        """
+        super(CheckHostnameMixin, cls).setUpClass()
+
+        if not hasattr(cls, 'sites'):
+            return
+
+        for key, data in cls.sites.items():
+            if 'hostname' not in data:
+                raise Exception('%s: hostname not defined for %s'
+                                % (cls.__name__, key))
+            hostname = data['hostname']
+
+            if hostname in cls._checked_hostnames:
+                if isinstance(cls._checked_hostnames[hostname], Exception):
+                    raise unittest.SkipTest(
+                        '%s: hostname %s failed (cached): %s'
+                        % (cls.__name__, hostname,
+                           cls._checked_hostnames[hostname]))
+                elif cls._checked_hostnames[hostname] is False:
+                    raise unittest.SkipTest('%s: hostname %s failed (cached)'
+                                            % (cls.__name__, hostname))
+                else:
+                    continue
+
+            protocol = 'http'
+            try:
+                r = http.fetch(uri=protocol + '://' + hostname,
+                               default_error_handling=False)
+                r.content  # default decode
+            except Exception as e:
+                cls._checked_hostnames[hostname] = e
+                raise unittest.SkipTest(
+                    '%s: hostname %s failed: %s'
+                    % (cls.__name__, hostname, e))
+            cls._checked_hostnames[hostname] = True
+
+
 class SiteWriteMixin(TestCaseBase):
 
     """
@@ -591,6 +641,8 @@ def wrapped_method(self):
         if 'cached' in dct and dct['cached']:
             bases = tuple([ForceCacheMixin] + list(bases))
 
+        bases = tuple([CheckHostnameMixin] + list(bases))
+
         if 'write' in dct and dct['write']:
             bases = tuple([SiteWriteMixin] + list(bases))
 
@@ -665,16 +717,19 @@ class has declared are needed.
             interface = DrySite
 
         for data in cls.sites.values():
-            if 'site' not in data:
+            if 'site' not in data and 'code' in data and 'family' in data:
                 data['site'] = Site(data['code'], data['family'],
                                     interface=interface)
+            if 'hostname' not in data and 'site' in data:
+                data['hostname'] = data['site'].hostname()
 
         if not hasattr(cls, 'cached') or not cls.cached:
             pywikibot._sites = orig_sites
 
         if len(cls.sites) == 1:
             key = next(iter(cls.sites.keys()))
-            cls.site = cls.sites[key]['site']
+            if 'site' in cls.sites[key]:
+                cls.site = cls.sites[key]['site']
 
     @classmethod
     def get_site(cls, name=None):
@@ -830,19 +885,23 @@ def setUpClass(cls):
         """
         super(WikibaseTestCase, cls).setUpClass()
 
-        for site in cls.sites.values():
-            if not site['site'].has_data_repository:
+        for data in cls.sites.values():
+            if 'site' not in data:
+                continue
+
+            site = data['site']
+            if not site.has_data_repository:
                 raise unittest.SkipTest(
                     u'%s: %r does not have data repository'
-                    % (cls.__name__, site['site']))
+                    % (cls.__name__, site))
 
             if (hasattr(cls, 'repo') and
-                    cls.repo != site['site'].data_repository()):
+                    cls.repo != site.data_repository()):
                 raise Exception(
                     '%s: sites do not all have the same data repository'
                     % cls.__name__)
 
-            cls.repo = site['site'].data_repository()
+            cls.repo = site.data_repository()
 
     @classmethod
     def get_repo(cls):

tests/data_ingestion_tests.py

@@ -14,7 +14,11 @@ class TestPhoto(TestCase):
 
     """Test Photo class."""
 
-    net = True
+    sites = {
+        'wm-upload': {
+            'hostname': 'upload.wikimedia.org',
+        },
+    }
 
     def setUp(self):
         super(TestPhoto, self).setUp()
@@ -54,7 +58,11 @@ class TestCSVReader(TestCase):
 
     """Test CSVReader class."""
 
-    net = False
+    sites = {
+        'wm-upload': {
+            'hostname': 'upload.wikimedia.org',
+        },
+    }
 
     def setUp(self):
         super(TestCSVReader, self).setUp()

tests/http_tests.py

@@ -27,7 +27,20 @@ class HttpTestCase(TestCase):
 
     """Tests for http module."""
 
-    net = True
+    sites = {
+        'www-wp': {
+            'hostname': 'www.wikipedia.org',
+        },
+        'www-wq': {
+            'hostname': 'www.wikiquote.org',
+        },
+        'omegawiki': {
+            'hostname': 'www.omegawiki.org',
+        },
+        'vikidia': {
+            'hostname': 'en.vikidia.org',
+        },
+    }
 
     def test_async(self):
         """Test http request_async function."""
@@ -113,7 +126,14 @@ class ThreadedHttpTestCase(TestCase):
 
     """Tests for threadedhttp module Http class."""
 
-    net = True
+    sites = {
+        'www-wp': {
+            'hostname': 'www.wikipedia.org',
+        },
+        'wikidata': {
+            'hostname': 'test.wikidata.org',
+        },
+    }
 
     def test_http(self):
         o = threadedhttp.Http()

tests/weblib_tests.py

@@ -7,7 +7,6 @@
 #
 __version__ = '$Id$'
 
-import os
 import sys
 if sys.version_info[0] > 2:
     from urllib.parse import urlparse
@@ -18,17 +17,15 @@
 from tests.aspects import unittest, TestCase
 
 
-class TestArchiveSites(TestCase):
+class TestInternetArchive(TestCase):
 
-    """Test weblib methods to access archive websites."""
+    """Test weblib methods to access Internet Archive."""
 
-    net = True
-
-    @classmethod
-    def setUpClass(cls):
-        if os.environ.get('TRAVIS', 'false') == 'true':
-            raise unittest.SkipTest('Weblib tests are disabled on Travis-CI')
-        super(TestArchiveSites, cls).setUpClass()
+    sites = {
+        'archive.org': {
+            'hostname': 'web.archive.org',
+        },
+    }
 
     def testInternetArchiveNewest(self):
         archivedversion = weblib.getInternetArchiveURL('https://google.com')
@@ -45,10 +42,23 @@ def testInternetArchiveOlder(self):
         self.assertTrue(parsed.path.strip('/').endswith('www.google.com'), parsed.path)
         self.assertIn('200606', parsed.path)
 
+
+class TestWebCite(TestCase):
+
+    """Test weblib methods to access WebCite."""
+
+    sites = {
+        'webcite': {
+            'hostname': 'www.webcitation.org',
+        }
+    }
+
+    @unittest.expectedFailure
     def testWebCiteOlder(self):
         archivedversion = weblib.getWebCitationURL('https://google.com', '20130101')
         self.assertEqual(archivedversion, 'http://www.webcitation.org/6DHSeh2L0')
 
+
 if __name__ == '__main__':
     try:
         unittest.main()

tests/wikidataquery_tests.py

@@ -10,7 +10,7 @@
 
 
 import pywikibot.data.wikidataquery as query
-from tests.aspects import unittest, WikidataTestCase, TestCase
+from tests.aspects import unittest, WikibaseTestCase, TestCase
 
 import pywikibot
 from pywikibot.page import ItemPage, PropertyPage, Claim
@@ -19,10 +19,22 @@
 import time
 
 
-class TestApiFunctions(WikidataTestCase):
+class TestApiFunctions(WikibaseTestCase):
 
     """Test WikiDataQuery API functions."""
 
+    sites = {
+        'wikidata': {
+            'family': 'wikidata',
+            'code': 'wikidata',
+        },
+        'wdq': {
+            'hostname': 'wdq.wmflabs.org',
+        },
+    }
+
+    cached = True
+
     def testQueries(self):
         """
         Test Queries and check whether they're behaving correctly.
@@ -217,7 +229,11 @@ class TestApiSlowFunctions(TestCase):
 
     """Test slow WikiDataQuery API functions."""
 
-    net = True
+    sites = {
+        'wdq': {
+            'hostname': 'wdq.wmflabs.org',
+        },
+    }
 
     def testQueryApiGetter(self):
         """Test that we can actually retreive data and that caching works."""

tests/wikistats_tests.py

@@ -22,7 +22,11 @@ class WikiStatsTestCase(TestCase):
 
     """Test WikiStats dump."""
 
-    net = True
+    sites = {
+        'wikistats': {
+            'hostname': 'wikistats.wmflabs.org',
+        },
+    }
 
     def test_sort(self):
         ws = WikiStats()