Add random port support to Python services

Signed-off-by: Ryan Levick <ryan.levick@fermyon.com>

tests/runtime-tests/README.md

@@ -53,6 +53,20 @@ The following service types are supported:
 
 When looking to add a new service, always prefer the Python based service as it's generally much quicker and lighter weight to run a Python script than a Docker container. Only use Docker when the service you require is not possible to achieve in cross platform way as a Python script.
 
+### Signaling Service Readiness
+
+Services can signal that they are ready so that tests aren't run against them until they are ready:
+
+* Python: Python services signal they are ready by printing `READY` to stdout.
+* Docker: Docker services signal readiness by exposing a Docker health check in the Dockerfile (e.g., `HEALTHCHECK --start-period=4s --interval=1s CMD  /usr/bin/mysqladmin ping --silent`)
+
+### Exposing Ports
+
+Both Docker and Python based services can expose some logical port number that will be mapped to a random free port number at runtime.
+
+* Python: Python based services can do this by printing `PORT=($PORT1, $PORT2)` to stdout where the $PORT1 is the logical port the service exposes and $PORT2 is the random port actually being exposed (e.g., `PORT=(80, 59392)`)
+* Docker: Docker services can do this by exposing the port in their Dockerfile (e.g., `EXPOSE 3306`)
+
 ## When do tests pass?
 
 A test will pass in the following conditions:

tests/runtime-tests/services/http-echo.py

@@ -20,10 +20,13 @@ def do_POST(self):
         self.wfile.write(body)
 
 
-def run(port=8080):
-    server_address = ('', port)
+def run():
+    server_address = ('', 0)
     httpd = HTTPServer(server_address, EchoHandler)
-    print(f'Starting server on port {port}...')
+    print(f'Starting http server...')
+    port = selected_port = httpd.server_address[1]
+    print(f'PORT=(80,{port})')
+    print(f'READY',  flush=True)
     httpd.serve_forever()
 
 

tests/runtime-tests/services/tcp-echo.py

@@ -2,6 +2,7 @@
 import threading
 import os
 
+
 def handle_client(client_socket):
     while True:
         data = client_socket.recv(1024)
@@ -11,27 +12,31 @@ def handle_client(client_socket):
         client_socket.send(data)
     client_socket.close()
 
+
 def echo_server():
-    host = "127.0.0.1" 
+    host = "127.0.0.1"
     server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    server_socket.bind((host, 6001))
+    server_socket.bind((host, 0))
     server_socket.listen(5)
     _, port = server_socket.getsockname()
-    print(f"Listening on {host}:{port}")
+    print(f"Listening on {host}...")
+    print(f"PORT=(5000,{port})")
+    print(f"READY", flush=True)
 
     try:
         while True:
             client_socket, client_address = server_socket.accept()
             print(f"Accepted connection from {client_address}")
             # Handle the client in a separate thread
-            client_handler = threading.Thread(target=handle_client, args=(client_socket,))
+            client_handler = threading.Thread(
+                target=handle_client, args=(client_socket,))
             client_handler.start()
     except KeyboardInterrupt:
         print("Server shutting down.")
     finally:
         # Close the server socket
         server_socket.close()
 
+
 if __name__ == "__main__":
-    # Run the echo server
     echo_server()

tests/runtime-tests/src/lib.rs

@@ -59,7 +59,7 @@ pub fn bootstrap_and_run(test_path: &Path, config: &Config) -> Result<(), anyhow
         .context("failed to produce a temporary directory to run the test in")?;
     log::trace!("Temporary directory: {}", temp.path().display());
     let mut services = services::start_services(test_path)?;
-    copy_manifest(test_path, &temp, &services)?;
+    copy_manifest(test_path, &temp, &mut services)?;
     let spin = Spin::start(&config.spin_binary_path, temp.path(), &mut services)?;
     log::debug!("Spin started on port {}.", spin.port());
     run_test(test_path, spin, config.on_error);
@@ -156,7 +156,7 @@ static TEMPLATE: OnceLock<regex::Regex> = OnceLock::new();
 fn copy_manifest(
     test_dir: &Path,
     temp: &temp_dir::TempDir,
-    services: &Services,
+    services: &mut Services,
 ) -> anyhow::Result<()> {
     let manifest_path = test_dir.join("spin.toml");
     let mut manifest = std::fs::read_to_string(manifest_path).with_context(|| {
@@ -346,10 +346,11 @@ impl OutputStream {
         std::thread::spawn(move || {
             let mut buffer = vec![0; 1024];
             loop {
-                if tx
-                    .send(stream.read(&mut buffer).map(|n| buffer[..n].to_vec()))
-                    .is_err()
-                {
+                let msg = stream.read(&mut buffer).map(|n| buffer[..n].to_vec());
+                if let Err(e) = tx.send(msg) {
+                    if let Err(e) = e.0 {
+                        eprintln!("Error reading from stream: {e}");
+                    }
                     break;
                 }
             }
@@ -369,6 +370,8 @@ impl OutputStream {
     }
 
     /// Get the output of the stream so far
+    ///
+    /// Returns None if the output is not valid utf8
     fn output_as_str(&mut self) -> Option<&str> {
         std::str::from_utf8(self.output()).ok()
     }

tests/runtime-tests/src/services.rs

@@ -37,7 +37,7 @@ pub fn start_services(test_path: &Path) -> anyhow::Result<Services> {
         let service_definition_extension = service_definitions
             .get(required_service)
             .map(|e| e.as_str());
-        let service: Box<dyn Service> = match service_definition_extension {
+        let mut service: Box<dyn Service> = match service_definition_extension {
             Some("py") => Box::new(PythonService::start(
                 required_service,
                 &service_definitions_path,
@@ -92,9 +92,9 @@ impl Services {
     }
 
     /// Get the host port that a service exposes a guest port on.
-    pub(crate) fn get_port(&self, guest_port: u16) -> anyhow::Result<Option<u16>> {
+    pub(crate) fn get_port(&mut self, guest_port: u16) -> anyhow::Result<Option<u16>> {
         let mut result = None;
-        for service in &self.services {
+        for service in &mut self.services {
             let host_port = service.ports().unwrap().get(&guest_port);
             match result {
                 None => result = host_port.copied(),
@@ -122,11 +122,11 @@ pub trait Service {
     fn name(&self) -> &str;
 
     /// Block until the service is ready.
-    fn await_ready(&self) -> anyhow::Result<()>;
+    fn await_ready(&mut self) -> anyhow::Result<()>;
 
     /// Check if the service is in an error state.
     fn error(&mut self) -> anyhow::Result<()>;
 
     /// Get a mapping of ports that the service exposes.
-    fn ports(&self) -> anyhow::Result<&HashMap<u16, u16>>;
+    fn ports(&mut self) -> anyhow::Result<&HashMap<u16, u16>>;
 }

tests/runtime-tests/src/services/docker.rs

@@ -62,8 +62,13 @@ impl Container {
                 let (guest, host) = s
                     .split_once(" -> ")
                     .context("failed to parse port mapping")?;
-                let (guest_port, _) = guest.split_once('/').context("TODO")?;
-                let host_port = host.rsplit(':').next().context("TODO")?;
+                let (guest_port, _) = guest
+                    .split_once('/')
+                    .context("guest mapping does not contain '/'")?;
+                let host_port = host
+                    .rsplit(':')
+                    .next()
+                    .expect("`rsplit` should always return one element but somehow did not");
                 Ok((guest_port.parse()?, host_port.parse()?))
             })
             .collect()
@@ -81,7 +86,7 @@ impl Service for DockerService {
         "docker"
     }
 
-    fn await_ready(&self) -> anyhow::Result<()> {
+    fn await_ready(&mut self) -> anyhow::Result<()> {
         // docker container inspect -f '{{.State.Health.Status}}'
         loop {
             let output = Command::new("docker")
@@ -111,7 +116,7 @@ impl Service for DockerService {
         Ok(())
     }
 
-    fn ports(&self) -> anyhow::Result<&HashMap<u16, u16>> {
+    fn ports(&mut self) -> anyhow::Result<&HashMap<u16, u16>> {
         match self.ports.get() {
             Some(p) => Ok(p),
             None => {

tests/runtime-tests/src/services/python.rs

@@ -1,13 +1,18 @@
+use crate::OutputStream;
+
 use super::Service;
 use anyhow::Context as _;
 use std::{
+    cell::OnceCell,
     collections::HashMap,
     path::Path,
     process::{Command, Stdio},
 };
 
 pub struct PythonService {
     child: std::process::Child,
+    stdout: OutputStream,
+    ports: OnceCell<HashMap<u16, u16>>,
     _lock: fslock::LockFile,
 }
 
@@ -17,18 +22,28 @@ impl PythonService {
             fslock::LockFile::open(&service_definitions_path.join(format!("{name}.lock")))
                 .context("failed to open service file lock")?;
         lock.lock().context("failed to obtain service file lock")?;
-        let child = python()
+        let mut child = python()
             .arg(
                 service_definitions_path
                     .join(format!("{name}.py"))
                     .display()
                     .to_string(),
             )
-            // Ignore stdout
-            .stdout(Stdio::null())
+            .stdout(Stdio::piped())
             .spawn()
             .context("service failed to spawn")?;
-        Ok(Self { child, _lock: lock })
+        std::thread::sleep(std::time::Duration::from_millis(1000));
+        Ok(Self {
+            stdout: OutputStream::new(
+                child
+                    .stdout
+                    .take()
+                    .expect("child process somehow does not have stdout"),
+            ),
+            child,
+            ports: OnceCell::new(),
+            _lock: lock,
+        })
     }
 }
 
@@ -37,7 +52,16 @@ impl Service for PythonService {
         "python"
     }
 
-    fn await_ready(&self) -> anyhow::Result<()> {
+    fn await_ready(&mut self) -> anyhow::Result<()> {
+        loop {
+            let stdout = self
+                .stdout
+                .output_as_str()
+                .context("stdout is not valid utf8")?;
+            if stdout.contains("READY") {
+                break;
+            }
+        }
         Ok(())
     }
 
@@ -53,8 +77,35 @@ impl Service for PythonService {
         Ok(())
     }
 
-    fn ports(&self) -> anyhow::Result<&HashMap<u16, u16>> {
-        todo!()
+    fn ports(&mut self) -> anyhow::Result<&HashMap<u16, u16>> {
+        let stdout = self
+            .stdout
+            .output_as_str()
+            .context("stdout is not valid utf8")?;
+        match self.ports.get() {
+            Some(ports) => Ok(ports),
+            None => {
+                let ports = stdout
+                    .lines()
+                    .filter_map(|l| l.trim().split_once('='))
+                    .map(|(k, v)| -> anyhow::Result<_> {
+                        let k = k.trim();
+                        let v = v.trim();
+                        if k == "PORT" {
+                            let err = "malformed service port pair - PORT values should be in the form PORT=(80,8080)";
+                            let (port_in, port_out) = v.split_once(',').context(err)?;
+                            let port_in = port_in.trim().strip_prefix('(').context(err)?;
+                            let port_out = port_out.trim().strip_suffix(')').context(err)?;
+                            Ok(Some((port_in.parse::<u16>().context("port number was not a number")?, port_out.parse::<u16>().context("port number was not a number")?)))
+                        } else {
+                            Ok(None)
+                        }
+                    })
+                    .filter_map(|r| r.transpose())
+                    .collect::<anyhow::Result<HashMap<_, _>>>()?;
+                Ok(self.ports.get_or_init(|| ports))
+            }
+        }
     }
 }
 

tests/runtime-tests/tests/tcp-sockets-ip-range/spin.toml

@@ -11,4 +11,5 @@ component = "test"
 
 [component.test]
 source = "%{source=tcp-sockets}"
-allowed_outbound_hosts = ["*://127.0.0.0/24:6001"]
+environment = { ADDRESS = "127.0.0.1:%{port=5000}" }
+allowed_outbound_hosts = ["*://127.0.0.0/24:%{port=5000}"]

tests/runtime-tests/tests/tcp-sockets-no-ip-permission/spin.toml

@@ -11,5 +11,6 @@ component = "test"
 
 [component.test]
 source = "%{source=tcp-sockets}"
+environment = { ADDRESS = "127.0.0.1:6001" }
 # Component expects 127.0.0.1 but we only allow 127.0.0.2
 allowed_outbound_hosts = ["*://127.0.0.2:6001"]

tests/runtime-tests/tests/tcp-sockets-no-port-permission/spin.toml

@@ -11,5 +11,6 @@ component = "test"
 
 [component.test]
 source = "%{source=tcp-sockets}"
-# Component expects port 5001 but we allow 6002
+environment = { ADDRESS = "127.0.0.1:6001" }
+# Component expects port 6001 but we allow 6002
 allowed_outbound_hosts = ["*://127.0.0.1:6002"]

tests/runtime-tests/tests/tcp-sockets/spin.toml

@@ -11,4 +11,5 @@ component = "test"
 
 [component.test]
 source = "%{source=tcp-sockets}"
-allowed_outbound_hosts = ["*://127.0.0.1:6001"]
+environment = { ADDRESS = "127.0.0.1:%{port=5000}" }
+allowed_outbound_hosts = ["*://127.0.0.1:%{port=5000}"]

tests/runtime-tests/tests/wasi-http/services

@@ -1 +1 @@
-http-echo.py
+http-echo

tests/runtime-tests/tests/wasi-http/spin.toml

@@ -10,5 +10,6 @@ route = "/"
 component = "test"
 
 [component.test]
-source = "{{wasi-http-v0.2.0-rc-2023-11-10}}"
-allowed_outbound_hosts = ["http://localhost:8080"]
+source = "%{source=wasi-http-v0.2.0-rc-2023-11-10}"
+environment = { URL = "http://localhost:%{port=80}" }
+allowed_outbound_hosts = ["http://localhost:%{port=80}"]

tests/test-components/components/tcp-sockets/README.md

@@ -5,4 +5,5 @@ Tests the `wasi:sockets` TCP related interfaces
 ## Expectations
 
 This test component expects the following to be true:
-* It has access to a TCP echo server on 127.0.0.1:6001
+* It is provided the env variable `ADDRESS`
+* It has access to a TCP echo server on the address supplied in `ADDRESS`

tests/test-components/components/tcp-sockets/src/lib.rs

@@ -9,13 +9,13 @@ use bindings::wasi::{
     },
 };
 use helper::{ensure_eq, ensure_ok};
-use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::net::SocketAddr;
 
 helper::define_component!(Component);
 
 impl Component {
     fn main() -> Result<(), String> {
-        let address = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 6001);
+        let address = ensure_ok!(ensure_ok!(std::env::var("ADDRESS")).parse());
 
         let client = ensure_ok!(tcp_create_socket::create_tcp_socket(IpAddressFamily::Ipv4));
 

tests/test-components/components/wasi-http-v0.2.0-rc-2023-11-10/README.md

@@ -7,4 +7,5 @@ The `wit` directory was copied from https://github.com/bytecodealliance/wasmtime
 ## Expectations
 
 This test component expects the following to be true:
-* It has access to an HTTP server on localhost:8080 that accepts POST requests and returns the same bytes in the response body as in the request body.
+* It is provided the env variable `URL`
+* It has access to an HTTP server at $URL (where $URL is the url provided above) that accepts POST requests and returns the same bytes in the response body as in the request body.

tests/test-components/components/wasi-http-v0.2.0-rc-2023-11-10/src/lib.rs

@@ -12,7 +12,7 @@ helper::define_component!(Component);
 
 impl Component {
     fn main() -> Result<(), String> {
-        let url = url::Url::parse("http://localhost:8080").unwrap();
+        let url = ensure_ok!(url::Url::parse(&ensure_ok!(std::env::var("URL"))));
 
         let headers = Headers::new();
         headers