[mw 1.37] Fix for removed action API token parameters

The legacy API parameters have been finally removed[1], which means
that we can no longer check them using action=paraminfo.

- always use 'csrf' token for outdated token types
  (mw version >= 1.24wmf19) with TokenWallet
- log a message if a given token was replaced inside TokenWallet
- validate tokens by ignoring outdated tokens because they are
  replaced by'csrf' already
- update get_tokens doc string
- update token_tests

[1] https://www.mediawiki.org/wiki/MediaWiki_1.37/Deprecation_of_legacy_API_token_parameters

Bug: T291202
Change-Id: Ic3c0e629f6bd4a609679e0c3a100a75a497713ef

Author: xqt

pywikibot/site/_apisite.py

@@ -1294,28 +1294,14 @@ def getredirtarget(self, page):
 
         return page._redirtarget
 
-    def validate_tokens(self, types: List[str]):
+    def validate_tokens(self, types: List[str]) -> List[str]:
         """Validate if requested tokens are acceptable.
 
         Valid tokens depend on mw version.
         """
-        mw_ver = self.mw_version
-        if mw_ver < '1.24wmf19':
-            types_wiki = self._paraminfo.parameter('tokens', 'type')['type']
-            valid_types = [token for token in types if token in types_wiki]
-        else:
-            types_wiki_old = self._paraminfo.parameter('query+info',
-                                                       'token')['type']
-            types_wiki_action = self._paraminfo.parameter('tokens',
-                                                          'type')['type']
-            types_wiki = self._paraminfo.parameter('query+tokens',
-                                                   'type')['type']
-            valid_types = [token for token in types if token in types_wiki]
-            for token in types:
-                if (token in types_wiki_old or token in types_wiki_action) \
-                   and token not in valid_types:
-                    valid_types.append('csrf')
-        return valid_types
+        query = 'tokens' if self.mw_version < '1.24wmf19' else 'query+tokens'
+        token_types = self._paraminfo.parameter(query, 'type')['type']
+        return [token for token in types if token in token_types]
 
     def get_tokens(self, types: List[str], all: bool = False) -> dict:
         """Preload one or multiple tokens.
@@ -1327,7 +1313,9 @@ def get_tokens(self, types: List[str], all: bool = False) -> dict:
         the parameter is not known it will default to the 'csrf' token.
 
         The other token types available are:
+         - createaccount
          - deleteglobalaccount
+         - login
          - patrol
          - rollback
          - setglobalaccountstatus

pywikibot/site/_tokenwallet.py

@@ -4,6 +4,7 @@
 #
 # Distributed under the terms of the MIT license.
 #
+from pywikibot import log
 from pywikibot.exceptions import Error
 
 
@@ -54,6 +55,14 @@ def __getitem__(self, key):
         # always preload all for users without tokens
         failed_cache_key = (self.site.user(), key)
 
+        # redirect old tokens to be compatible with older MW version
+        # https://www.mediawiki.org/wiki/MediaWiki_1.37/Deprecation_of_legacy_API_token_parameters
+        if self.site.mw_version >= '1.24wmf19' \
+           and key in {'edit', 'delete', 'protect', 'move', 'block', 'unblock',
+                       'email', 'import', 'options'}:
+            log('Token {!r} was replaced by {!r}'.format(key, 'csrf'))
+            key = 'csrf'
+
         try:
             key = self.site.validate_tokens([key])[0]
         except IndexError:

tests/token_tests.py

@@ -7,6 +7,7 @@
 from contextlib import suppress
 
 from pywikibot.exceptions import APIError, Error
+from pywikibot.tools import MediaWikiVersion
 from pywikibot.site import TokenWallet
 from tests.aspects import DefaultSiteTestCase, TestCase, TestCaseBase, unittest
 
@@ -45,31 +46,36 @@ def _test_tokens(self, version, test_version, additional_token):
                 .format(self.mysite, self._version))
 
         self.mysite.version = lambda: test_version
+        del self.mysite._mw_version_time  # remove cached mw_version
 
-        for ttype in ('edit', 'move', 'delete', 'patrol', additional_token):
-            tokentype = self.mysite.validate_tokens([ttype])
+        redirected_tokens = ['edit', 'move', 'delete']
+        for ttype in redirected_tokens + ['patrol', additional_token]:
             try:
                 token = self.mysite.tokens[ttype]
             except Error as error_msg:
-                if tokentype:
-                    self.assertRegex(
-                        str(error_msg),
-                        "Action '[a-z]+' is not allowed "
-                        'for user .* on .* wiki.')
-                    # test __contains__
-                    self.assertNotIn(tokentype[0], self.mysite.tokens)
+                if self.mysite.validate_tokens([ttype]):
+                    pattern = ("Action '[a-z]+' is not allowed "
+                               'for user .* on .* wiki.')
                 else:
-                    self.assertRegex(
-                        str(error_msg),
-                        "Requested token '[a-z]+' is invalid on .* wiki.")
+                    pattern = "Requested token '[a-z]+' is invalid on .* wiki."
+
+                self.assertRegex(str(error_msg), pattern)
+
             else:
                 self.assertIsInstance(token, str)
                 self.assertEqual(token, self.mysite.tokens[ttype])
                 # test __contains__
-                self.assertIn(tokentype[0], self.mysite.tokens)
+                if test_version < '1.24wmf19':
+                    self.assertIn(ttype, self.mysite.tokens)
+                elif ttype in redirected_tokens:
+                    self.assertEqual(self.mysite.tokens[ttype],
+                                     self.mysite.tokens['csrf'])
 
     def test_tokens_in_mw_123_124wmf18(self):
         """Test ability to get page tokens."""
+        if MediaWikiVersion(self.orig_version()) >= '1.37wmf24':
+            self.skipTest('Site {} version {} is too new for this tests.'
+                          .format(self.mysite, self._version))
         self._test_tokens('1.23', '1.24wmf18', 'deleteglobalaccount')
 
     def test_tokens_in_mw_124wmf19(self):