Auto merge of rust-lang#41268 - mmatyas:test_on_device, r=alexcrichton

Run non-native tests on real device

After rust-lang#40733, I've made some hacks to the QEMU client-server tools to allow running the tests on a real device when cross compiling Rust. The address and port of the remote server can be set using an environment variable.

I've made this mainly for local testing purposes, if you're interested in merging this, I'd clean it a bit more (eg. renaming the functions from `qemu-` to something else). I'm not asking for CI integration or adding ARM boards to the build system; it's just that I used these modifications and I was wondering if you'd find them useful too.

src/bootstrap/check.rs

@@ -529,7 +529,7 @@ fn find_tests(dir: &Path,
     }
 }
 
-pub fn emulator_copy_libs(build: &Build, compiler: &Compiler, target: &str) {
+pub fn remote_copy_libs(build: &Build, compiler: &Compiler, target: &str) {
     if !build.remote_tested(target) {
         return
     }

src/bootstrap/step.rs

@@ -307,7 +307,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
                  .dep(|s| s.name("libtest"))
                  .dep(|s| s.name("tool-compiletest").target(s.host).stage(0))
                  .dep(|s| s.name("test-helpers"))
-                 .dep(|s| s.name("emulator-copy-libs"))
+                 .dep(|s| s.name("remote-copy-libs"))
                  .default(mode != "pretty") // pretty tests don't run everywhere
                  .run(move |s| {
                      check::compiletest(build, &s.compiler(), s.target, mode, dir)
@@ -346,7 +346,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
              .dep(|s| s.name("tool-compiletest").target(s.host).stage(0))
              .dep(|s| s.name("test-helpers"))
              .dep(|s| s.name("debugger-scripts"))
-             .dep(|s| s.name("emulator-copy-libs"))
+             .dep(|s| s.name("remote-copy-libs"))
              .run(move |s| check::compiletest(build, &s.compiler(), s.target,
                                          "debuginfo-gdb", "debuginfo"));
         let mut rule = rules.test("check-debuginfo", "src/test/debuginfo");
@@ -400,14 +400,14 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     for (krate, path, _default) in krates("std") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("emulator-copy-libs"))
+             .dep(|s| s.name("remote-copy-libs"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Libstd, TestKind::Test,
                                         Some(&krate.name)));
     }
     rules.test("check-std-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .dep(|s| s.name("emulator-copy-libs"))
+         .dep(|s| s.name("remote-copy-libs"))
          .default(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
                                     Mode::Libstd, TestKind::Test, None));
@@ -416,44 +416,44 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     for (krate, path, _default) in krates("std") {
         rules.bench(&krate.bench_step, path)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("emulator-copy-libs"))
+             .dep(|s| s.name("remote-copy-libs"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Libstd, TestKind::Bench,
                                         Some(&krate.name)));
     }
     rules.bench("bench-std-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .dep(|s| s.name("emulator-copy-libs"))
+         .dep(|s| s.name("remote-copy-libs"))
          .default(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
                                     Mode::Libstd, TestKind::Bench, None));
 
     for (krate, path, _default) in krates("test") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("emulator-copy-libs"))
+             .dep(|s| s.name("remote-copy-libs"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Libtest, TestKind::Test,
                                         Some(&krate.name)));
     }
     rules.test("check-test-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .dep(|s| s.name("emulator-copy-libs"))
+         .dep(|s| s.name("remote-copy-libs"))
          .default(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
                                     Mode::Libtest, TestKind::Test, None));
     for (krate, path, _default) in krates("rustc-main") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("librustc"))
-             .dep(|s| s.name("emulator-copy-libs"))
+             .dep(|s| s.name("remote-copy-libs"))
              .host(true)
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Librustc, TestKind::Test,
                                         Some(&krate.name)));
     }
     rules.test("check-rustc-all", "path/to/nowhere")
          .dep(|s| s.name("librustc"))
-         .dep(|s| s.name("emulator-copy-libs"))
+         .dep(|s| s.name("remote-copy-libs"))
          .default(true)
          .host(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
@@ -500,17 +500,17 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     rules.build("openssl", "path/to/nowhere")
          .run(move |s| native::openssl(build, s.target));
 
-    // Some test suites are run inside emulators, and most of our test binaries
-    // are linked dynamically which means we need to ship the standard library
-    // and such to the emulator ahead of time. This step represents this and is
-    // a dependency of all test suites.
+    // Some test suites are run inside emulators or on remote devices, and most
+    // of our test binaries are linked dynamically which means we need to ship
+    // the standard library and such to the emulator ahead of time. This step
+    // represents this and is a dependency of all test suites.
     //
     // Most of the time this step is a noop (the `check::emulator_copy_libs`
     // only does work if necessary). For some steps such as shipping data to
     // QEMU we have to build our own tools so we've got conditional dependencies
-    // on those programs as well. Note that the QEMU client is built for the
-    // build target (us) and the server is built for the target.
-    rules.test("emulator-copy-libs", "path/to/nowhere")
+    // on those programs as well. Note that the remote test client is built for
+    // the build target (us) and the server is built for the target.
+    rules.test("remote-copy-libs", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
          .dep(move |s| {
              if build.remote_tested(s.target) {
@@ -526,7 +526,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
                  Step::noop()
              }
          })
-         .run(move |s| check::emulator_copy_libs(build, &s.compiler(), s.target));
+         .run(move |s| check::remote_copy_libs(build, &s.compiler(), s.target));
 
     rules.test("check-bootstrap", "src/bootstrap")
          .default(true)

src/tools/remote-test-client/src/main.rs

@@ -25,6 +25,8 @@ use std::process::{Command, Stdio};
 use std::thread;
 use std::time::Duration;
 
+const REMOTE_ADDR_ENV: &'static str = "TEST_DEVICE_ADDR";
+
 macro_rules! t {
     ($e:expr) => (match $e {
         Ok(e) => e,
@@ -56,7 +58,11 @@ fn spawn_emulator(target: &str,
                   server: &Path,
                   tmpdir: &Path,
                   rootfs: Option<PathBuf>) {
-    if target.contains("android") {
+    let device_address = env::var(REMOTE_ADDR_ENV).unwrap_or("127.0.0.1:12345".to_string());
+
+    if env::var(REMOTE_ADDR_ENV).is_ok() {
+        println!("Connecting to remote device {} ...", device_address);
+    } else if target.contains("android") {
         start_android_emulator(server);
     } else {
         let rootfs = rootfs.as_ref().expect("need rootfs on non-android");
@@ -66,7 +72,7 @@ fn spawn_emulator(target: &str,
     // Wait for the emulator to come online
     loop {
         let dur = Duration::from_millis(100);
-        if let Ok(mut client) = TcpStream::connect("127.0.0.1:12345") {
+        if let Ok(mut client) = TcpStream::connect(&device_address) {
             t!(client.set_read_timeout(Some(dur)));
             t!(client.set_write_timeout(Some(dur)));
             if client.write_all(b"ping").is_ok() {
@@ -162,7 +168,8 @@ fn start_qemu_emulator(rootfs: &Path, server: &Path, tmpdir: &Path) {
 }
 
 fn push(path: &Path) {
-    let client = t!(TcpStream::connect("127.0.0.1:12345"));
+    let device_address = env::var(REMOTE_ADDR_ENV).unwrap_or("127.0.0.1:12345".to_string());
+    let client = t!(TcpStream::connect(device_address));
     let mut client = BufWriter::new(client);
     t!(client.write_all(b"push"));
     send(path, &mut client);
@@ -178,7 +185,8 @@ fn push(path: &Path) {
 }
 
 fn run(files: String, args: Vec<String>) {
-    let client = t!(TcpStream::connect("127.0.0.1:12345"));
+    let device_address = env::var(REMOTE_ADDR_ENV).unwrap_or("127.0.0.1:12345".to_string());
+    let client = t!(TcpStream::connect(device_address));
     let mut client = BufWriter::new(client);
     t!(client.write_all(b"run "));
 

src/tools/remote-test-server/src/main.rs

@@ -8,10 +8,10 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-/// This is a small server which is intended to run inside of an emulator. This
-/// server pairs with the `remote-test-client` program in this repository. The
-/// `remote-test-client` connects to this server over a TCP socket and performs
-/// work such as:
+/// This is a small server which is intended to run inside of an emulator or
+/// on a remote test device. This server pairs with the `remote-test-client`
+/// program in this repository. The `remote-test-client` connects to this
+/// server over a TCP socket and performs work such as:
 ///
 /// 1. Pushing shared libraries to the server
 /// 2. Running tests through the server
@@ -21,6 +21,7 @@
 /// basically custom format suiting our needs.
 
 use std::cmp;
+use std::env;
 use std::fs::{self, File, Permissions};
 use std::io::prelude::*;
 use std::io::{self, BufReader};
@@ -42,12 +43,54 @@ macro_rules! t {
 
 static TEST: AtomicUsize = ATOMIC_USIZE_INIT;
 
+struct Config {
+    pub remote: bool,
+    pub verbose: bool,
+}
+
+impl Config {
+    pub fn default() -> Config {
+        Config {
+            remote: false,
+            verbose: false,
+        }
+    }
+
+    pub fn parse_args() -> Config {
+        let mut config = Config::default();
+
+        let args = env::args().skip(1);
+        for argument in args {
+            match &argument[..] {
+                "remote" => {
+                    config.remote = true;
+                },
+                "verbose" | "-v" => {
+                    config.verbose = true;
+                }
+                arg => panic!("unknown argument: {}", arg),
+            }
+        }
+
+        config
+    }
+}
+
 fn main() {
     println!("starting test server");
+
+    let config = Config::parse_args();
+
+    let bind_addr = if cfg!(target_os = "android") || config.remote {
+        "0.0.0.0:12345"
+    } else {
+        "10.0.2.15:12345"
+    };
+
     let (listener, work) = if cfg!(target_os = "android") {
-        (t!(TcpListener::bind("0.0.0.0:12345")), "/data/tmp/work")
+        (t!(TcpListener::bind(bind_addr)), "/data/tmp/work")
     } else {
-        (t!(TcpListener::bind("10.0.2.15:12345")), "/tmp/work")
+        (t!(TcpListener::bind(bind_addr)), "/tmp/work")
     };
     println!("listening!");