Support for overriding SHA1 with URL rewriting

README.md

@@ -525,12 +525,13 @@ Further details are [here](README-proftool.md).
 ### URL rewriting
 
 Mx includes support for the primary suite to be able to override the source URLs of imported suites.
-The suite level `urlrewrites` attribute allows regular expression URL rewriting. For example:
+The suite level `urlrewrites` attribute allows regular expression URL rewriting, and, optionally, SHA1 rewriting. For example:
 ```
   "urlrewrites" : [
     {
       "https://git.acme.com/(.*).git" : {
         "replacement" : r”https://my.company.com/foo-git-cache/\1.git",
+        "sha1" : "NOCHECK",
       }
     },
     {

mx.py

@@ -8375,11 +8375,11 @@ def __init__(self, suite, name, path, optional, urls, sha1, **kwArgs):
         BaseLibrary.__init__(self, suite, name, optional, None, **kwArgs)
         self.path = _make_absolute(path.replace('/', os.sep), suite.dir) if path else None
         self.sourcePath = None
-        self.urls = urls
-        self.sha1 = sha1
+        self.urls = [self.substVars(url) for url in urls]
+        self.sha1 = mx_urlrewrites.rewritesha1(self.urls, sha1)
 
     def get_urls(self):
-        return [mx_urlrewrites.rewriteurl(self.substVars(url)) for url in self.urls]
+        return mx_urlrewrites.rewriteurls(self.urls)
 
     def getArchivableResults(self, use_relpath=True, single=False):
         path = realpath(self.get_path(False))
@@ -8656,19 +8656,19 @@ def __init__(self, suite, name, path, optional, urls, sha1, sourcePath, sourceUr
         BaseLibrary.__init__(self, suite, name, optional, theLicense, **kwArgs)
         ClasspathDependency.__init__(self, **kwArgs)
         self.path = path.replace('/', os.sep) if path is not None else None
-        self.urls = urls
-        self.sha1 = sha1
+        self.urls = [self.substVars(url) for url in urls]
+        self.sha1 = mx_urlrewrites.rewritesha1(self.urls, sha1)
         self.sourcePath = sourcePath.replace('/', os.sep) if sourcePath else None
         self.sourceUrls = sourceUrls
         if sourcePath == path:
-            assert sourceSha1 is None or sourceSha1 == sha1
-            sourceSha1 = sha1
+            assert sourceSha1 is None or sourceSha1 == self.sha1
+            sourceSha1 = self.sha1
         self.sourceSha1 = sourceSha1
         self.deps = deps
         self.ignore = ignore
         if not optional and not ignore:
             abspath = _make_absolute(path, self.suite.dir)
-            if not exists(abspath) and not len(urls):
+            if not exists(abspath) and not len(self.urls):
                 abort('Non-optional library {0} must either exist at {1} or specify one or more URLs from which it can be retrieved'.format(name, abspath), context=self)
 
             def _checkSha1PropertyCondition(propName, cond, inputPath):
@@ -8678,10 +8678,10 @@ def _checkSha1PropertyCondition(propName, cond, inputPath):
                         abort('Missing "{0}" property for library {1}. Add the following to the definition of {1}:\n{0}={2}'.format(propName, name, sha1OfFile(absInputPath)), context=self)
                     abort('Missing "{0}" property for library {1}'.format(propName, name), context=self)
 
-            _checkSha1PropertyCondition('sha1', sha1, path)
+            _checkSha1PropertyCondition('sha1', self.sha1, path)
             _checkSha1PropertyCondition('sourceSha1', not sourcePath or sourceSha1, sourcePath)
 
-        for url in urls:
+        for url in self.urls:
             if url.endswith('/') != self.path.endswith(os.sep):
                 abort('Path for dependency directory must have a URL ending with "/": path=' + self.path + ' url=' + url, context=self)
 
@@ -8700,7 +8700,7 @@ def _comparison_key(self):
         return (self.sha1, self.name)
 
     def get_urls(self):
-        return [mx_urlrewrites.rewriteurl(self.substVars(url)) for url in self.urls]
+        return mx_urlrewrites.rewriteurls(self.urls)
 
     def is_available(self):
         if not self.path:

mx_urlrewrites.py

@@ -63,6 +63,7 @@ def _error(msg):
     if not isinstance(urlrewrite, dict) or len(urlrewrite) != 1:
         onError('A URL rewrite rule must be a dict with a single entry')
     for pattern, attrs in urlrewrite.items():
+        sha1 = attrs.pop('sha1', None)
         replacement = attrs.pop('replacement', None)
         if replacement is None:
             raise Exception('URL rewrite for pattern "' + pattern + '" is missing "replacement" entry')
@@ -72,7 +73,7 @@ def _error(msg):
             pattern = re.compile(pattern)
         except Exception as e: # pylint: disable=broad-except
             onError('Error parsing URL rewrite pattern "' + pattern + '": ' + str(e))
-        urlrewrite = URLRewrite(pattern, str(replacement))
+        urlrewrite = URLRewrite(pattern, str(replacement), sha1)
         mx.logvv("Registering url rewrite: " + str(urlrewrite))
         _urlrewrites.append(urlrewrite)
 
@@ -110,30 +111,61 @@ def loadJson(jsonValue):
             for rewrite in rewrites:
                 register_urlrewrite(rewrite, raiseError)
 
-def rewriteurl(url):
+def _geturlrewrite(url):
     """
-    Finds the first registered URL rewrite rule that matches `url` and returns the replacement
-    provided by the rule.
+    Finds the first registered URL rewrite rule that matches `url` and returns it.
 
     :param str url: a URL to match against the registered rerwrite rules
-    :return: the value of `url` rewritten according to the first matching rewrite URL or unmodified
-             if no rules match
-    :rtype: str
+    :return: `URLRewrite` rule that matches `url` or `None`
     """
-    original_url = url
     jar_url = mx._JarURL.parse(url)
     if jar_url:
         url = jar_url.base_url
 
     for urlrewrite in _urlrewrites:
         res = urlrewrite._rewrite(url)
         if res:
-            if jar_url is not None:
-                jar_url.base_url = res
-                res = str(jar_url)
-            mx.logvv("Rewrote '{}' to '{}'".format(original_url, res))
-            return res
-    return original_url
+            return urlrewrite
+
+    return None
+
+def _applyurlrewrite(urlrewrite, url):
+    """
+    Applies an URL rewrite rule to `url`.
+    Handles JAR URL references.
+    """
+    jar_url = mx._JarURL.parse(url)
+    if jar_url:
+        jar_url.base_url = urlrewrite._rewrite(jar_url.base_url)
+        res = str(jar_url)
+    else:
+        res = urlrewrite._rewrite(url)
+    mx.logvv("Rewrote '{}' to '{}'".format(url, res))
+    return res
+
+def rewriteurl(url):
+    """
+    Finds the first registered URL rewrite rule that matches `url` and returns the replacement `url`
+    provided by the rule.
+
+    :param str url: a URL to match against the registered rerwrite rules
+    :return: the value of `url` rewritten according to the first matching rewrite URL or unmodified if no rules match
+    :rtype: str
+    """
+    urlrewrite = _geturlrewrite(url)
+    if urlrewrite:
+        return _applyurlrewrite(urlrewrite, url)
+    return url
+
+def rewriteurls(urls):
+    return [rewriteurl(url) for url in urls]
+
+def rewritesha1(urls, sha1):
+    for url in urls:
+        urlrewrite = _geturlrewrite(url)
+        if urlrewrite and urlrewrite.sha1:
+            return urlrewrite.sha1
+    return sha1
 
 def urlrewrite_cli(args):
     """rewrites the given URL using MX_URLREWRITES"""
@@ -148,9 +180,10 @@ class URLRewrite(object):
     :param str replacement: the replacement to use for a URL matched by `pattern`
     """
 
-    def __init__(self, pattern, replacement):
+    def __init__(self, pattern, replacement, sha1):
         self.pattern = pattern
         self.replacement = replacement
+        self.sha1 = sha1
 
     def _rewrite(self, url):
         match = self.pattern.match(url)