Use Opendal driver in Http driver

docs/source/api-reference/drivers/opendal.md

@@ -13,11 +13,9 @@ The Opendal driver is a driver for interacting with storages attached to the exp
 ```{doctest}
 >>> from tempfile import NamedTemporaryFile
 >>> opendal.create_dir("test/directory/")
->>> remote_file = opendal.open("test/directory/file", "wb")
->>> with NamedTemporaryFile() as local_file:
-...     assert local_file.write(b"hello") == 5
-...     remote_file.write(local_file.name)
->>> remote_file.close()
+>>> opendal.write_bytes("test/directory/file", b"hello")
+>>> assert opendal.hash("test/directory/file", "md5") == "5d41402abc4b2a76b9719d911017c592"
+>>> opendal.remove_all("test/")
 ```
 
 ```{testsetup} *

packages/jumpstarter-driver-http/jumpstarter_driver_http/client.py

@@ -1,14 +1,13 @@
 from dataclasses import dataclass
-from pathlib import Path
 
-from jumpstarter_driver_opendal.adapter import OpendalAdapter
+from jumpstarter_driver_composite.client import CompositeClient
+from jumpstarter_driver_opendal.common import PathBuf
 from opendal import Operator
-
-from jumpstarter.client import DriverClient
+from yarl import URL
 
 
 @dataclass(kw_only=True)
-class HttpServerClient(DriverClient):
+class HttpServerClient(CompositeClient):
     """Client for the HTTP server driver"""
 
     def start(self):
@@ -30,59 +29,6 @@ def stop(self):
         """
         self.call("stop")
 
-    def list_files(self) -> list[str]:
-        """
-        List all files in the HTTP server's root directory.
-
-        Returns:
-            list[str]: A list of filenames present in the HTTP server's root directory
-        """
-        return self.call("list_files")
-
-    def put_file(self, filename: str, src_stream):
-        """
-        Upload a file to the HTTP server using a streamed source.
-
-        Args:
-            filename (str): Name to save the file as on the server.
-            src_stream: Stream/source to read the file data from.
-
-        Returns:
-            str: URL of the uploaded file
-        """
-        return self.call("put_file", filename, src_stream)
-
-    def put_local_file(self, filepath: str) -> str:
-        """
-        Upload a file from the local filesystem to the HTTP server.
-
-        Note: This doesn't use HTTP to upload; it streams the file content directly.
-
-        Args:
-            filepath (str): Path to the local file to upload.
-
-        Returns:
-            str: Name of the uploaded file
-
-        Example:
-            >>> client.put_local_file("/path/to/local/file.txt")
-        """
-        absolute = Path(filepath).resolve()
-        with OpendalAdapter(client=self, operator=Operator("fs", root="/"), path=str(absolute), mode="rb") as handle:
-            return self.call("put_file", absolute.name, handle)
-
-    def delete_file(self, filename: str) -> str:
-        """
-        Delete a file from the HTTP server.
-
-        Args:
-            filename (str): Name of the file to delete.
-
-        Returns:
-            str: Name of the deleted file
-        """
-        return self.call("delete_file", filename)
-
     def get_host(self) -> str:
         """
         Get the host IP address the HTTP server is listening on.
@@ -109,3 +55,19 @@ def get_url(self) -> str:
             str: The base URL of the server
         """
         return self.call("get_url")
+
+    def put_file(self, dst: PathBuf, src: PathBuf, operator: Operator | None = None) -> str:
+        """
+        Upload a file to the HTTP server using a opendal operator as source.
+
+        Args:
+            dst (PathBuf): Name to save the file as on the server.
+            src (PathBuf): Name to read the file from opendal operator.
+            operator (Operator): opendal operator to read the file from, defaults to local fs.
+
+        Returns:
+            str: URL of the uploaded file
+        """
+        self.storage.write_from_path(dst, src, operator)
+
+        return str(URL(self.get_url()).joinpath(dst))

packages/jumpstarter-driver-http/jumpstarter_driver_http/driver.py

@@ -1,11 +1,10 @@
 import os
 from dataclasses import dataclass, field
-from pathlib import Path
 from typing import Optional
 
 import anyio
 from aiohttp import web
-from anyio.streams.file import FileWriteStream
+from jumpstarter_driver_opendal.driver import Opendal
 
 from jumpstarter.driver import Driver, export
 
@@ -34,6 +33,8 @@ def __post_init__(self):
             super().__post_init__()
 
         os.makedirs(self.root_dir, exist_ok=True)
+
+        self.children["storage"] = Opendal(scheme="fs", kwargs={"root": self.root_dir})
         self.app.router.add_routes(
             [
                 web.static("/", self.root_dir),
@@ -58,84 +59,6 @@ def client(cls) -> str:
         """Return the import path of the corresponding client"""
         return "jumpstarter_driver_http.client.HttpServerClient"
 
-    @export
-    async def put_file(self, filename: str, src_stream) -> str:
-        """
-        Upload a file to the HTTP server.
-
-        Args:
-            filename (str): Name of the file to upload.
-            src_stream: Stream of file content.
-
-        Returns:
-            str: Name of the uploaded file.
-
-        Raises:
-            HttpServerError: If the target path is invalid.
-            FileWriteError: If the file upload fails.
-        """
-        try:
-            file_path = os.path.join(self.root_dir, filename)
-
-            if not Path(file_path).resolve().is_relative_to(Path(self.root_dir).resolve()):
-                raise HttpServerError("Invalid target path")
-
-            async with await FileWriteStream.from_path(file_path) as dst:
-                async with self.resource(src_stream, timeout=self.timeout) as src:
-                    async for chunk in src:
-                        await dst.send(chunk)
-
-            self.logger.info(f"File '{filename}' written to '{file_path}'")
-            return f"{self.get_url()}/{filename}"
-
-        except Exception as e:
-            self.logger.error(f"Failed to upload file '{filename}': {e}")
-            raise FileWriteError(f"Failed to upload file '{filename}': {e}") from e
-
-    @export
-    async def delete_file(self, filename: str) -> str:
-        """
-        Delete a file from the HTTP server.
-
-        Args:
-            filename (str): Name of the file to delete.
-
-        Returns:
-            str: Name of the deleted file.
-
-        Raises:
-            HttpServerError: If the file does not exist or deletion fails.
-        """
-        file_path = Path(self.root_dir) / filename
-        if not file_path.exists():
-            raise HttpServerError(f"File '{filename}' does not exist.")
-        try:
-            file_path.unlink()
-            self.logger.info(f"File '{filename}' has been deleted.")
-            return filename
-        except Exception as e:
-            self.logger.error(f"Failed to delete file '{filename}': {e}")
-            raise HttpServerError(f"Failed to delete file '{filename}': {e}") from e
-
-    @export
-    def list_files(self) -> list[str]:
-        """
-        List all files in the root directory.
-
-        Returns:
-            list[str]: List of filenames in the root directory.
-
-        Raises:
-            HttpServerError: If listing files fails.
-        """
-        try:
-            files = os.listdir(self.root_dir)
-            files = [f for f in files if os.path.isfile(os.path.join(self.root_dir, f))]
-            return files
-        except Exception as e:
-            self.logger.error(f"Failed to list files: {e}")
-            raise HttpServerError(f"Failed to list files: {e}") from e
-
     @export
     async def start(self):
         """

packages/jumpstarter-driver-http/jumpstarter_driver_http/driver_test.py

@@ -1,14 +1,8 @@
-import os
-import uuid
-from tempfile import TemporaryDirectory
-
 import aiohttp
-import anyio
 import pytest
-from anyio import create_memory_object_stream
 
 from .driver import HttpServer
-from jumpstarter.common.resources import ClientStreamResource
+from jumpstarter.common.utils import serve
 
 
 @pytest.fixture
@@ -17,44 +11,27 @@ def anyio_backend():
 
 
 @pytest.fixture
-def temp_dir():
-    with TemporaryDirectory() as tmpdir:
-        yield tmpdir
-
-
-@pytest.fixture
-async def server(temp_dir):
-    server = HttpServer(root_dir=temp_dir)
-    await server.start()
-    try:
-        yield server
-    finally:
-        await server.stop()
+def http(tmp_path):
+    with serve(HttpServer(root_dir=str(tmp_path))) as client:
+        client.start()
+        try:
+            yield client
+        finally:
+            client.stop()
 
 
 @pytest.mark.anyio
-async def test_http_server(server):
+async def test_http_server(http, tmp_path):
     filename = "test.txt"
     test_content = b"test content"
 
-    send_stream, receive_stream = create_memory_object_stream(max_buffer_size=1024)
-
-    resource_uuid = uuid.uuid4()
-    server.resources[resource_uuid] = receive_stream
-
-    resource_handle = ClientStreamResource(uuid=resource_uuid).model_dump(mode="json")
-
-    async def send_data():
-        await send_stream.send(test_content)
-        await send_stream.aclose()
+    (tmp_path / "src").write_bytes(test_content)
 
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(send_data)
+    uploaded_url = http.put_file(filename, tmp_path / "src")
 
-        uploaded_url = await server.put_file(filename, resource_handle)
-        assert uploaded_url == f"{server.get_url()}/{filename}"
+    print(http.storage.stat(filename))
 
-    files = server.list_files()
+    files = list(http.storage.list("/"))
     assert filename in files
 
     async with aiohttp.ClientSession() as session:
@@ -63,20 +40,19 @@ async def send_data():
             retrieved_content = await response.read()
             assert retrieved_content == test_content
 
-    deleted_filename = await server.delete_file(filename)
-    assert deleted_filename == filename
+    http.storage.delete(filename)
 
-    files_after_deletion = server.list_files()
+    files_after_deletion = list(http.storage.list("/"))
     assert filename not in files_after_deletion
 
 
-def test_http_server_host_config(temp_dir):
+def test_http_server_host_config(tmp_path):
     custom_host = "192.168.1.1"
-    server = HttpServer(root_dir=temp_dir, host=custom_host)
+    server = HttpServer(root_dir=str(tmp_path), host=custom_host)
     assert server.get_host() == custom_host
 
 
-def test_http_server_root_directory_creation(temp_dir):
-    new_dir = os.path.join(temp_dir, "new_http_root")
-    _ = HttpServer(root_dir=new_dir)
-    assert os.path.exists(new_dir)
+def test_http_server_root_directory_creation(tmp_path):
+    new_dir = tmp_path / "new_http_root"
+    _ = HttpServer(root_dir=str(new_dir))
+    assert new_dir.exists()

packages/jumpstarter-driver-http/pyproject.toml

@@ -10,7 +10,9 @@ requires-python = ">=3.12"
 dependencies = [
     "anyio>=4.6.2.post1",
     "jumpstarter",
-    "jumpstarter-driver-opendal"
+    "jumpstarter-driver-composite",
+    "jumpstarter-driver-opendal",
+    "yarl>=1.18.3",
 ]
 
 [tool.hatch.version]