Automatically generate a netrc file according to basic http auth in u…

…rl (#306)

* Automatically generate a netrc file according to basic http auth in url

For every mirror url in a pinned json, try to extract netrc credentials. Then, write those
credentials into a file that every generated http_file for the various maven artifacts will use.

Alternative way of implementing would be by allowing you to pass a pre-made netrc and that can be
added in later if we should choose to do so, although it may be somewhat difficult to do so due to
there being at least three different repositories in play making it difficult to use raw paths.

* Remove auth from fetched urls

coursier.bzl

@@ -131,6 +131,72 @@ def _compute_dependency_tree_signature(artifacts):
         signature_inputs.append(":".join(artifact_group))
     return hash(repr(sorted(signature_inputs)))
 
+def extract_netrc_from_auth_url(url):
+    """Return a dict showing the netrc machine, login, and password extracted from a url.
+
+    Returns:
+        A dict that is empty if there were no credentials in the url.
+        A dict that has three keys -- machine, login, password -- with their respective values. These values should be
+        what is needed for the netrc entry of the same name except for password whose value may be empty meaning that
+        there is no password for that login.
+    """
+    if "@" not in url:
+        return {}
+    protocol, url_parts = split_url(url)
+    login_password_host = url_parts[0]
+    if "@" not in login_password_host:
+        return {}
+    login_password, host = login_password_host.rsplit("@", 1)
+    login_password_split = login_password.split(":", 1)
+    login = login_password_split[0]
+    # If password is not provided, then this will be a 1-length split
+    if len(login_password_split) < 2:
+        password = None
+    else:
+        password = login_password_split[1]
+    if not host:
+        fail("Got a blank host from: {}".format(url))
+    if not login:
+        fail("Got a blank login from: {}".format(url))
+    # Do not fail for blank password since that is sometimes a thing
+    return {
+        "machine": host,
+        "login": login,
+        "password": password,
+    }
+
+def add_netrc_entries_from_mirror_urls(netrc_entries, mirror_urls):
+    """Add a url's auth credentials into a netrc dict of form return[machine][login] = password."""
+    for url in mirror_urls:
+        entry = extract_netrc_from_auth_url(url)
+        if not entry:
+            continue
+        machine = entry["machine"]
+        login = entry["login"]
+        password = entry["password"]
+        if machine not in netrc_entries:
+            netrc_entries[machine] = {}
+        if login not in netrc_entries[machine]:
+            if netrc_entries[machine]:
+                print("Received multiple logins for machine '{}'! Only using '{}'".format(
+                    machine, netrc_entries[machine].keys()[0]))
+                continue
+            netrc_entries[machine][login] = password
+        else:
+            if netrc_entries[machine][login] != password:
+                print("Received different passwords for {}@{}! Only using the first".format(login, machine))
+    return netrc_entries
+
+def get_netrc_lines_from_entries(netrc_entries):
+    netrc_lines = []
+    for machine, login_dict in sorted(netrc_entries.items()):
+        for login, password in sorted(login_dict.items()):
+            netrc_lines.append("machine {}".format(machine))
+            netrc_lines.append("login {}".format(login))
+            if password:
+                netrc_lines.append("password {}".format(password))
+    return netrc_lines
+
 def _pinned_coursier_fetch_impl(repository_ctx):
     if not repository_ctx.attr.maven_install_json:
         fail("Please specify the file label to maven_install.json (e.g." +
@@ -197,23 +263,31 @@ def _pinned_coursier_fetch_impl(repository_ctx):
         "load(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_file\")",
         "def pinned_maven_install():",
     ]
+    netrc_entries = {}
     for artifact in dep_tree["dependencies"]:
         if artifact.get("url") != None:
             http_file_repository_name = escape(artifact["coord"])
             http_files.extend([
                 "    http_file(",
                 "        name = \"%s\"," % http_file_repository_name,
                 "        sha256 = \"%s\"," % artifact["sha256"],
+                # repository_ctx should point to external/$repository_ctx.name
+                # The http_file should point to external/$http_file_repository_name
+                # File-path is relative defined from http_file traveling to repository_ctx.
+                "        netrc = \"../%s/netrc\"," % (repository_ctx.name),
             ])
             if artifact.get("mirror_urls") != None:
-                http_files.append("        urls = %s," % repr(artifact["mirror_urls"]))
+                http_files.append("        urls = %s," % repr(
+                    [remove_auth_from_url(url) for url in artifact["mirror_urls"]]))
+                netrc_entries = add_netrc_entries_from_mirror_urls(netrc_entries, artifact["mirror_urls"])
             else:
                 # For backwards compatibility. mirror_urls is a field added in a
                 # later version than the url field, so not all maven_install.json
                 # contains the mirror_urls field.
                 http_files.append("        urls = [\"%s\"]," % artifact["url"])
             http_files.append("    )")
     repository_ctx.file("defs.bzl", "\n".join(http_files), executable = False)
+    repository_ctx.file("netrc", "\n".join(get_netrc_lines_from_entries(netrc_entries)), executable = False)
 
     repository_ctx.report_progress("Generating BUILD targets..")
     (generated_imports, jar_versionless_target_labels) = parser.generate_imports(
@@ -272,13 +346,17 @@ def _pinned_coursier_fetch_impl(repository_ctx):
                 False,  # not executable
             )
 
+def split_url(url):
+    protocol = url[:url.find("://")]
+    url_without_protocol = url[url.find("://") + 3:]
+    url_parts = url_without_protocol.split("/")
+    return protocol, url_parts
+
 def remove_auth_from_url(url):
     """Returns url without `user:pass@` or `user@`."""
     if "@" not in url:
         return url
-    protocol = url[:url.find("://")]
-    url_without_protocol = url[url.find("://") + 3:]
-    url_parts = url_without_protocol.split("/")
+    protocol, url_parts = split_url(url)
     host = url_parts[0]
     if "@" not in host:
         return url

tests/unit/coursier_test.bzl

@@ -1,7 +1,11 @@
 load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest")
 load("//:coursier.bzl",
+    "add_netrc_entries_from_mirror_urls",
+    "extract_netrc_from_auth_url",
+    "get_netrc_lines_from_entries",
     infer = "infer_artifact_path_from_primary_and_repos",
     "remove_auth_from_url",
+    "split_url",
 )
 
 ALL_TESTS = []
@@ -120,6 +124,190 @@ def _remove_auth_noauth_noop_test_impl(ctx):
 
 remove_auth_noauth_noop_test = add_test(_remove_auth_noauth_noop_test_impl)
 
+def _split_url_basic_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        ("https", ["c1"]),
+        split_url("https://c1"))
+    return unittest.end(env)
+
+split_url_basic_test = add_test(_split_url_basic_test_impl)
+
+def _split_url_basic_auth_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        ("https", ["a:b@c1"]),
+        split_url("https://a:b@c1"))
+    asserts.equals(
+        env,
+        ("https", ["a@c1"]),
+        split_url("https://a@c1"))
+    return unittest.end(env)
+
+split_url_basic_auth_test = add_test(_split_url_basic_auth_test_impl)
+
+def _split_url_with_path_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        ("https", ["c1", "some", "path"]),
+        split_url("https://c1/some/path"))
+    return unittest.end(env)
+
+split_url_with_path_test = add_test(_split_url_with_path_test_impl)
+
+def _extract_netrc_from_auth_url_noop_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        {},
+        extract_netrc_from_auth_url("https://c1"))
+    asserts.equals(
+        env,
+        {},
+        extract_netrc_from_auth_url("https://c2/useless@inurl"))
+    return unittest.end(env)
+
+extract_netrc_from_auth_url_noop_test = add_test(_extract_netrc_from_auth_url_noop_test_impl)
+
+def _extract_netrc_from_auth_url_with_auth_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        {"machine": "c", "login": "a", "password": "b"},
+        extract_netrc_from_auth_url("https://a:b@c"))
+    asserts.equals(
+        env,
+        {"machine": "c", "login": "a", "password": "b"},
+        extract_netrc_from_auth_url("https://a:b@c/some/other/stuff@thisplace/for/testing"))
+    asserts.equals(
+        env,
+        {"machine": "c", "login": "a", "password": None},
+        extract_netrc_from_auth_url("https://a@c"))
+    asserts.equals(
+        env,
+        {"machine": "c", "login": "a", "password": None},
+        extract_netrc_from_auth_url("https://a@c/some/other/stuff@thisplace/for/testing"))
+    return unittest.end(env)
+
+extract_netrc_from_auth_url_with_auth_test = add_test(_extract_netrc_from_auth_url_with_auth_test_impl)
+
+def _extract_netrc_from_auth_url_at_in_password_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        {"machine": "c", "login": "a", "password": "p@ssword"},
+        extract_netrc_from_auth_url("https://a:p@ssword@c"))
+    return unittest.end(env)
+
+extract_netrc_from_auth_url_at_in_password_test = add_test(_extract_netrc_from_auth_url_at_in_password_test_impl)
+
+def _add_netrc_entries_from_mirror_urls_noop_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        {},
+        add_netrc_entries_from_mirror_urls({}, ["https://c1", "https://c1/something@there"]))
+    return unittest.end(env)
+
+add_netrc_entries_from_mirror_urls_noop_test = add_test(_add_netrc_entries_from_mirror_urls_noop_test_impl)
+
+def _add_netrc_entries_from_mirror_urls_basic_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        {"c1": {"a": "b"}},
+        add_netrc_entries_from_mirror_urls({}, ["https://a:b@c1"]))
+    asserts.equals(
+        env,
+        {"c1": {"a": "b"}},
+        add_netrc_entries_from_mirror_urls(
+            {"c1": {"a": "b"}},
+            ["https://a:b@c1"]))
+    return unittest.end(env)
+
+add_netrc_entries_from_mirror_urls_basic_test = add_test(_add_netrc_entries_from_mirror_urls_basic_test_impl)
+
+def _add_netrc_entries_from_mirror_urls_multi_login_ignored_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        {"c1": {"a": "b"}},
+        add_netrc_entries_from_mirror_urls({}, ["https://a:b@c1", "https://a:b2@c1", "https://a2:b3@c1"]))
+    asserts.equals(
+        env,
+        {"c1": {"a": "b"}},
+        add_netrc_entries_from_mirror_urls(
+            {"c1": {"a": "b"}},
+            ["https://a:b@c1", "https://a:b2@c1", "https://a2:b3@c1"]))
+    return unittest.end(env)
+
+add_netrc_entries_from_mirror_urls_multi_login_ignored_test = add_test(_add_netrc_entries_from_mirror_urls_multi_login_ignored_test_impl)
+
+def _add_netrc_entries_from_mirror_urls_multi_case_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        {"foo": {"bar": "baz"},
+         "c1": {"a1": "b1"},
+         "c2": {"a2": "b2"}},
+        add_netrc_entries_from_mirror_urls(
+            {"foo": {"bar": "baz"}},
+            ["https://a1:b1@c1", "https://a2:b2@c2", "https://a:b@c1", "https://a:b2@c1", "https://a2:b3@c1"]))
+    return unittest.end(env)
+
+add_netrc_entries_from_mirror_urls_multi_case_test = add_test(_add_netrc_entries_from_mirror_urls_multi_case_test_impl)
+
+def _get_netrc_lines_from_entries_noop_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        [],
+        get_netrc_lines_from_entries({}))
+    return unittest.end(env)
+
+get_netrc_lines_from_entries_noop_test = add_test(_get_netrc_lines_from_entries_noop_test_impl)
+
+def _get_netrc_lines_from_entries_basic_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        ["machine c", "login a", "password b"],
+        get_netrc_lines_from_entries({
+            "c": {"a": "b"}
+        }))
+    return unittest.end(env)
+
+get_netrc_lines_from_entries_basic_test = add_test(_get_netrc_lines_from_entries_basic_test_impl)
+
+def _get_netrc_lines_from_entries_no_pass_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        ["machine c", "login a"],
+        get_netrc_lines_from_entries({
+            "c": {"a": ""}
+        }))
+    return unittest.end(env)
+
+get_netrc_lines_from_entries_no_pass_test = add_test(_get_netrc_lines_from_entries_no_pass_test_impl)
+
+def _get_netrc_lines_from_entries_multi_test_impl(ctx):
+    env = unittest.begin(ctx)
+    asserts.equals(
+        env,
+        ["machine c", "login a", "password b",
+         "machine c2", "login a2", "password p@ssword"],
+        get_netrc_lines_from_entries({
+            "c": {"a": "b"},
+            "c2": {"a2": "p@ssword"}
+        }))
+    return unittest.end(env)
+
+get_netrc_lines_from_entries_multi_test = add_test(_get_netrc_lines_from_entries_multi_test_impl)
+
 def coursier_test_suite():
     unittest.suite(
         "coursier_tests",