Use Cargo.toml to fetch cargo crates

.travis.yml

@@ -28,6 +28,8 @@ install:
  - curl -sSf https://sh.rustup.rs | sh -s -- -y
  - export PATH=$HOME/.cargo/bin:$PATH
  - rustc --version
+ # ./tools/fetch_cargo_crates.py depends on this subcommand (https://github.com/ehuss/cargo-clone-crate)
+ - cargo install cargo-clone-crate
  - ./tools/build_third_party.py
  # ccache needs the custom LLVM to be in PATH and other variables.
  - export PATH=`pwd`/third_party/llvm-build/Release+Asserts/bin:$PATH

Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "deno"
+version = "0.1.0"
+authors = "Ryan Dahl <ry@tinyclouds.org>"
+
+[dependencies]
+libc = "0.2.42"
+url = "1.7.1"
+matches = "0.1.6"
+unicode-bidi = "0.3.4"
+unicode-normalization = "0.1.7"

build_extra/rust/BUILD.gn

@@ -22,7 +22,7 @@ rust_component("libc") {
 }
 
 rust_component("url") {
-  source_root = "$crates/url/src/lib.rs"
+  source_root = "$crates/rust-url/src/lib.rs"
   extern = [
     ":matches",
     ":idna",
@@ -31,15 +31,15 @@ rust_component("url") {
 }
 
 rust_component("percent_encoding") {
-  source_root = "$crates/url/percent_encoding/lib.rs"
+  source_root = "$crates/rust-url/percent_encoding/lib.rs"
 }
 
 rust_component("matches") {
   source_root = "$crates/rust-std-candidates/matches/lib.rs"
 }
 
 rust_component("idna") {
-  source_root = "$crates/url/idna/src/lib.rs"
+  source_root = "$crates/rust-url/idna/src/lib.rs"
   extern = [
     ":matches",
     ":unicode_bidi",

gclient_config.py

@@ -36,30 +36,7 @@
     'flatbuffers'
 }, {
     'url':
-    'https://github.com/rust-lang/libc.git@8a85d662b90c14d458bc4ae9521a05564e20d7ae',
+    'https://github.com/avakar/pytoml.git@cb92445ca769b3966b3976cd69f5140761f15843',
     'name':
-    'rust_crates/libc'
-}, {
-    'url':
-    'https://github.com/servo/rust-url.git@fbe5e50316105482dcd53d2dabb148c445a5f4cd',
-    'name':
-    'rust_crates/url'
-}, {
-    # Needed for url.
-    'url':
-    'https://github.com/SimonSapin/rust-std-candidates.git@88a017b79ea146d6fde389c96982fc7518ba98bf',
-    'name':
-    'rust_crates/rust-std-candidates'
-}, {
-    # Needed for url.
-    'url':
-    'https://github.com/servo/unicode-bidi.git@32c81729db0ac90289ebeca9e0d4886f264e724d',
-    'name':
-    'rust_crates/unicode-bidi'
-}, {
-    # Needed for url.
-    'url':
-    'https://github.com/behnam/rust-unicode-normalization.git@3898e77b110246cb7243bf29b896c58d8975304a',
-    'name':
-    'rust_crates/unicode-normalization'
+    'pytoml'
 }]

tools/build_third_party.py

@@ -14,13 +14,15 @@
 
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 third_party_path = join(root_path, "third_party")
+tools_path = join(root_path, "tools")
 
 try:
     os.makedirs(third_party_path)
 except:
     pass
 os.chdir(third_party_path)
 remove_and_symlink(join("..", "gclient_config.py"), ".gclient")
+remove_and_symlink(join("..", "Cargo.toml"), "Cargo.toml")
 remove_and_symlink(join("..", "package.json"), "package.json")
 remove_and_symlink(join("..", "yarn.lock"), "yarn.lock")
 remove_and_symlink(join("v8", "third_party", "googletest"), "googletest")
@@ -29,3 +31,4 @@
 remove_and_symlink(join("v8", "third_party", "markupsafe"), "markupsafe")
 run(["gclient", "sync", "--shallow", "--no-history"])
 run(["yarn"])
+run(["python", join(tools_path, "fetch_cargo_crates.py")])

tools/fetch_cargo_crates.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# Inspired by
+# https://fuchsia.googlesource.com/build/+/master/rust/list_3p_crates.py
+# https://fuchsia.googlesource.com/build/+/master/rust/compile_3p_crates.py
+# Copyright 2018 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+from os.path import join, dirname, realpath
+import sys
+import subprocess
+
+root_path = dirname(dirname(realpath(__file__)))
+third_party_path = join(root_path, "third_party")
+
+sys.path += [join(third_party_path, "pytoml")]
+import pytoml
+
+def run_command(args, env, cwd):
+    job = subprocess.Popen(args, env=env, cwd=cwd, stdout=subprocess.PIPE,
+                           stderr=subprocess.PIPE)
+    stdout, stderr = job.communicate()
+    return (job.returncode, stdout, stderr)
+
+def clone_crate(name, version):
+    rust_crates_path = join(third_party_path, "rust_crates")
+    if not os.path.exists(rust_crates_path):
+        os.makedirs(rust_crates_path)
+    call_args = [
+        "cargo", "clone",
+        "%s:%s" % (name, version)
+    ]
+    env = os.environ.copy()
+    _, _, stderr = run_command(
+        call_args,
+        env,
+        join(third_party_path, "rust_crates")
+    )
+    print(stderr)
+
+def main():
+    cargo_toml_path = join(third_party_path, "Cargo.toml")
+    with open(cargo_toml_path, "r") as file:
+        cargo_toml = pytoml.load(file)
+        for key, value in cargo_toml["dependencies"].items():
+            if type(value) is dict:
+                git = value.get("git")
+                clone_crate(key, git)
+            else:
+                clone_crate(key, value)
+
+if __name__ == '__main__':
+    sys.exit(main())