From 553108ea9be2f9b9c95f07069134d31121237a6c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 04:16:50 +0000 Subject: [PATCH 01/12] Add preheat function to download OCI image blobs through Dragonfly proxy Add `oci-client` and `oci-spec` dependencies to workspace and util crate. Create `request/preheat.rs` with `PreheatRequest` struct and `preheat` function that: - Parses OCI image reference and authenticates with registry - Pulls manifest (with multi-platform index support) - Downloads all blobs (config + layers) via Dragonfly proxy - Discards content (preheat only caches in P2P network) Agent-Logs-Url: https://github.com/dragonflyoss/client/sessions/f89011d2-1920-46d0-9bc5-48e1f8e086e8 Co-authored-by: gaius-qi <15955374+gaius-qi@users.noreply.github.com> --- Cargo.lock | 395 ++++++++++++++++--- Cargo.toml | 2 + dragonfly-client-util/Cargo.toml | 3 + dragonfly-client-util/src/request/mod.rs | 1 + dragonfly-client-util/src/request/preheat.rs | 266 +++++++++++++ 5 files changed, 612 insertions(+), 55 deletions(-) create mode 100644 dragonfly-client-util/src/request/preheat.rs diff --git a/Cargo.lock b/Cargo.lock index d1eabed4..caaf9bd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -334,6 +334,28 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-lc-rs" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" version = "0.8.8" @@ -574,9 +596,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.41" +version = "1.2.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" dependencies = [ "find-msvc-tools", "jobserver", @@ -727,6 +749,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -831,6 +862,26 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "const_format" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -985,6 +1036,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -1066,6 +1152,37 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", +] + [[package]] name = "digest" version = "0.10.7" @@ -1164,7 +1281,7 @@ dependencies = [ "rand 0.9.2", "rcgen", "regex", - "reqwest", + "reqwest 0.12.28", "rolling-file", "rustls", "rustls-pki-types", @@ -1216,7 +1333,7 @@ dependencies = [ "opendal", "percent-encoding", "rcgen", - "reqwest", + "reqwest 0.12.28", "reqwest-middleware", "reqwest-retry", "reqwest-tracing", @@ -1253,7 +1370,7 @@ dependencies = [ "local-ip-address", "rcgen", "regex", - "reqwest", + "reqwest 0.12.28", "rustls-pki-types", "serde", "serde_json", @@ -1276,7 +1393,7 @@ dependencies = [ "hyper 1.9.0", "hyper-util", "opendal", - "reqwest", + "reqwest 0.12.28", "reqwest-middleware", "thiserror 2.0.18", "tokio", @@ -1340,7 +1457,7 @@ dependencies = [ "num_cpus", "prost-wkt-types", "quinn", - "reqwest", + "reqwest 0.12.28", "rocksdb", "rustls", "rustls-pki-types", @@ -1381,13 +1498,15 @@ dependencies = [ "lru", "mocktail", "num_cpus", + "oci-client", + "oci-spec", "openssl", "parking_lot", "pnet", "protobuf 3.7.2", "rcgen", "regex", - "reqwest", + "reqwest 0.12.28", "reqwest-middleware", "reqwest-tracing", "ringbuf", @@ -1396,6 +1515,7 @@ dependencies = [ "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", + "serde_json", "sha2", "sysinfo", "tempfile", @@ -1409,6 +1529,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.9.0" @@ -1555,9 +1681,9 @@ checksum = "a043dc74da1e37d6afe657061213aa6f425f855399a11d3463c6ecccc4dfda1f" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "findshlibs" @@ -1639,6 +1765,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.32" @@ -1791,6 +1923,18 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "getset" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "gimli" version = "0.28.1" @@ -2089,6 +2233,15 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-auth" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" +dependencies = [ + "memchr", +] + [[package]] name = "http-body" version = "0.4.6" @@ -2426,6 +2579,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.4.0" @@ -2683,10 +2842,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.81" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -2706,6 +2867,20 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "jsonwebtoken" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" +dependencies = [ + "base64 0.22.1", + "getrandom 0.2.12", + "js-sys", + "serde", + "serde_json", + "signature", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -3261,6 +3436,49 @@ dependencies = [ "memchr", ] +[[package]] +name = "oci-client" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b7f8deaffcd3b0e3baf93dddcab3d18b91d46dc37d38a8b170089b234de5bb3" +dependencies = [ + "bytes", + "chrono", + "futures-util", + "http 1.4.0", + "http-auth", + "jsonwebtoken 10.3.0", + "lazy_static", + "oci-spec", + "olpc-cjson", + "regex", + "reqwest 0.13.2", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.18", + "tokio", + "tracing", + "unicase", +] + +[[package]] +name = "oci-spec" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8445a2631507cec628a15fdd6154b54a3ab3f20ed4fe9d73a3b8b7a4e1ba03a" +dependencies = [ + "const_format", + "derive_builder", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror 2.0.18", +] + [[package]] name = "oid-registry" version = "0.6.1" @@ -3270,6 +3488,17 @@ dependencies = [ "asn1-rs", ] +[[package]] +name = "olpc-cjson" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083" +dependencies = [ + "serde", + "serde_json", + "unicode-normalization", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -3313,7 +3542,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.38.3", "reqsign", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "sha2", @@ -3399,7 +3628,7 @@ dependencies = [ "bytes", "http 1.4.0", "opentelemetry", - "reqwest", + "reqwest 0.12.28", ] [[package]] @@ -3414,7 +3643,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost 0.14.3", - "reqwest", + "reqwest 0.12.28", "thiserror 2.0.18", "tokio", "tonic", @@ -4238,6 +4467,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ + "aws-lc-rs", "bytes", "fastbloom", "getrandom 0.3.1", @@ -4424,13 +4654,13 @@ dependencies = [ "hmac", "home", "http 1.4.0", - "jsonwebtoken", + "jsonwebtoken 9.3.0", "log", "once_cell", "percent-encoding", "quick-xml 0.37.5", "rand 0.8.5", - "reqwest", + "reqwest 0.12.28", "rsa", "rust-ini", "serde", @@ -4485,11 +4715,52 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.0", "web-sys", "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.9.0", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + [[package]] name = "reqwest-middleware" version = "0.4.2" @@ -4499,7 +4770,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.4.0", - "reqwest", + "reqwest 0.12.28", "serde", "thiserror 1.0.69", "tower-service", @@ -4517,7 +4788,7 @@ dependencies = [ "getrandom 0.2.12", "http 1.4.0", "hyper 1.9.0", - "reqwest", + "reqwest 0.12.28", "reqwest-middleware", "retry-policies", "thiserror 2.0.18", @@ -4537,7 +4808,7 @@ dependencies = [ "getrandom 0.2.12", "http 1.4.0", "matchit", - "reqwest", + "reqwest 0.12.28", "reqwest-middleware", "tracing", ] @@ -4719,6 +4990,7 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring 0.17.7", @@ -4801,6 +5073,7 @@ version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ + "aws-lc-rs", "ring 0.17.7", "rustls-pki-types", "untrusted 0.9.0", @@ -5163,6 +5436,24 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "subtle" version = "2.5.0" @@ -5967,12 +6258,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.7.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" [[package]] name = "unicode-bidi" @@ -6218,9 +6506,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.104" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" dependencies = [ "cfg-if", "once_cell", @@ -6229,37 +6517,21 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.117", - "wasm-bindgen-shared", -] - [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" dependencies = [ - "cfg-if", "js-sys", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.104" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6267,22 +6539,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.104" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" dependencies = [ + "bumpalo", "proc-macro2", "quote", "syn 2.0.117", - "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.104" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" dependencies = [ "unicode-ident", ] @@ -6300,6 +6572,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmtimer" version = "0.4.3" @@ -6316,9 +6601,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 2777ab33..9429f562 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,6 +120,8 @@ cgroups-rs = "0.5" num_cpus = "1.17" async-trait = "0.1" humantime-serde = "1.1.1" +oci-client = { version = "0.16.1", default-features = false, features = ["rustls-tls"] } +oci-spec = "0.9" # TODO(Gaius): Remove the git dependency after the next release of mocktail, refer to https://github.com/IBM/mocktail/issues/65. mocktail = { version = "0.3.0", git = "https://github.com/IBM/mocktail", rev = "860e75e171a8eb818083813dbeed4401d1a40b3b" } diff --git a/dragonfly-client-util/Cargo.toml b/dragonfly-client-util/Cargo.toml index ddfb47d4..e7ef8e38 100644 --- a/dragonfly-client-util/Cargo.toml +++ b/dragonfly-client-util/Cargo.toml @@ -49,6 +49,9 @@ reqwest-tracing.workspace = true reqwest-middleware.workspace = true num_cpus.workspace = true humantime-serde.workspace = true +oci-client.workspace = true +oci-spec.workspace = true +serde_json.workspace = true rustix = { version = "1.1.3", features = ["fs"] } base64 = "0.22.1" pnet = "0.35.0" diff --git a/dragonfly-client-util/src/request/mod.rs b/dragonfly-client-util/src/request/mod.rs index e8c7fa42..e13674aa 100644 --- a/dragonfly-client-util/src/request/mod.rs +++ b/dragonfly-client-util/src/request/mod.rs @@ -15,6 +15,7 @@ */ pub mod errors; +pub mod preheat; mod selector; use crate::http::{headermap_to_hashmap, query_params::default_proxy_rule_filtered_query_params}; diff --git a/dragonfly-client-util/src/request/preheat.rs b/dragonfly-client-util/src/request/preheat.rs new file mode 100644 index 00000000..3cc8d39b --- /dev/null +++ b/dragonfly-client-util/src/request/preheat.rs @@ -0,0 +1,266 @@ +/* + * Copyright 2025 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use super::errors::Error; +use super::{GetRequest, Request, Result}; +use oci_client::client::ClientConfig; +use oci_client::manifest::ImageIndexEntry; +use oci_client::secrets::RegistryAuth; +use oci_client::{Client as OciClient, Reference, RegistryOperation}; +use oci_spec::image::{Arch, Os}; +use reqwest::header::{HeaderMap, HeaderValue}; +use rustls_pki_types::CertificateDer; +use std::time::Duration; +use tracing::{debug, info}; + +/// PreheatRequest represents a request to preheat an OCI image through the +/// Dragonfly seed client. The preheat downloads all blobs (config and layers) +/// of the specified image via the Dragonfly proxy, effectively caching them +/// in the P2P network for faster subsequent access. +pub struct PreheatRequest { + /// image is the OCI image reference (e.g., "docker.io/library/nginx:latest"). + pub image: String, + + /// username for registry authentication. If not provided, anonymous access is used. + pub username: Option, + + /// password for registry authentication. If not provided, anonymous access is used. + pub password: Option, + + /// platform specifies the target platform in the format "os/arch" + /// (e.g., "linux/amd64", "linux/arm64"). This is used to select the correct + /// manifest from a multi-platform image index. + pub platform: String, + + /// piece_length is the optional piece length for the Dragonfly task. + pub piece_length: Option, + + /// tag identifies different tasks for the same URL. + pub tag: Option, + + /// application identifies different tasks for the same URL. + pub application: Option, + + /// timeout is the timeout for each blob download request. + pub timeout: Duration, + + /// client_cert is the optional client certificates for the request. + pub client_cert: Option>>, +} + +/// Preheats an OCI image by downloading all its blobs through the Dragonfly proxy. +/// +/// This function performs the following steps: +/// 1. Parses the image reference and authenticates with the OCI registry. +/// 2. Pulls the image manifest (handling multi-platform image indexes). +/// 3. Downloads each blob (config + layers) through the Dragonfly proxy using the +/// provided `Request` implementation, discarding the content. +/// +/// The downloads go through the Dragonfly seed client's proxy, which caches the +/// content in the P2P network for faster subsequent access. +pub async fn preheat(request: &R, preheat_req: &PreheatRequest) -> Result<()> { + // Parse image reference. + let reference: Reference = + preheat_req + .image + .parse() + .map_err(|err: oci_client::ParseError| { + Error::InvalidArgument(format!("invalid image reference: {}", err)) + })?; + + // Create registry authentication. + let auth = match (&preheat_req.username, &preheat_req.password) { + (Some(username), Some(password)) => RegistryAuth::Basic(username.clone(), password.clone()), + _ => RegistryAuth::Anonymous, + }; + + // Parse platform (os/arch). + let (os, arch) = parse_platform(&preheat_req.platform)?; + + // Create OCI client with a platform resolver that matches the requested os/arch. + let oci_config = ClientConfig { + platform_resolver: Some(Box::new(move |manifests: &[ImageIndexEntry]| { + manifests + .iter() + .find(|entry| { + entry + .platform + .as_ref() + .is_some_and(|p| p.os == os && p.architecture == arch) + }) + .map(|e| e.digest.clone()) + })), + ..Default::default() + }; + let oci_client = OciClient::new(oci_config); + + // Authenticate with the registry and get a bearer token if available. + let token = oci_client + .auth(&reference, &auth, RegistryOperation::Pull) + .await + .map_err(|err| Error::Internal(format!("failed to authenticate with registry: {}", err)))?; + + // Pull image manifest. This handles multi-platform image index manifests + // by selecting the platform-specific manifest using our resolver. + let (manifest, digest) = oci_client + .pull_image_manifest(&reference, &auth) + .await + .map_err(|err| Error::Internal(format!("failed to pull image manifest: {}", err)))?; + + info!( + "pulled manifest for image {} with digest {}, layers: {}", + preheat_req.image, + digest, + manifest.layers.len() + ); + + // Build authorization header for blob downloads through the Dragonfly proxy. + let mut auth_headers = HeaderMap::new(); + if let Some(ref token) = token { + auth_headers.insert( + "Authorization", + HeaderValue::from_str(&format!("Bearer {}", token)) + .map_err(|err| Error::Internal(format!("invalid auth token: {}", err)))?, + ); + } else if let (Some(username), Some(password)) = (&preheat_req.username, &preheat_req.password) + { + let credentials = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + format!("{}:{}", username, password), + ); + auth_headers.insert( + "Authorization", + HeaderValue::from_str(&format!("Basic {}", credentials)) + .map_err(|err| Error::Internal(format!("invalid credentials: {}", err)))?, + ); + } + + // Construct blob download URLs and download through Dragonfly. + let registry = reference.resolve_registry(); + let repository = reference.repository(); + + // Collect all blob digests: config + layers. + let config_digest = manifest.config.digest.clone(); + let mut blob_digests: Vec = vec![config_digest]; + for layer in &manifest.layers { + blob_digests.push(layer.digest.clone()); + } + + for blob_digest in &blob_digests { + let blob_url = format!( + "https://{}/v2/{}/blobs/{}", + registry, repository, blob_digest + ); + + debug!("preheating blob: {}", blob_url); + + let get_request = GetRequest { + url: blob_url.clone(), + header: Some(auth_headers.clone()), + piece_length: preheat_req.piece_length, + tag: preheat_req.tag.clone(), + application: preheat_req.application.clone(), + filtered_query_params: vec![], + content_for_calculating_task_id: None, + priority: None, + timeout: preheat_req.timeout, + client_cert: preheat_req.client_cert.clone(), + }; + + let response = request.get(get_request).await?; + + // Read and discard the body to complete the download through Dragonfly. + if let Some(mut reader) = response.reader { + tokio::io::copy(&mut reader, &mut tokio::io::sink()) + .await + .map_err(|err| { + Error::Internal(format!("failed to read blob {}: {}", blob_digest, err)) + })?; + } + + info!("preheated blob: {}", blob_url); + } + + info!("preheat completed for image: {}", preheat_req.image); + Ok(()) +} + +/// Parses a platform string in the format "os/arch" into Os and Arch types. +fn parse_platform(platform: &str) -> Result<(Os, Arch)> { + let parts: Vec<&str> = platform.split('/').collect(); + if parts.len() != 2 { + return Err(Error::InvalidArgument(format!( + "invalid platform format '{}', expected 'os/arch' (e.g., 'linux/amd64')", + platform + ))); + } + + let os: Os = serde_json::from_str(&format!("\"{}\"", parts[0])) + .map_err(|_| Error::InvalidArgument(format!("unsupported OS: {}", parts[0])))?; + let arch: Arch = serde_json::from_str(&format!("\"{}\"", parts[1])) + .map_err(|_| Error::InvalidArgument(format!("unsupported architecture: {}", parts[1])))?; + + Ok((os, arch)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_platform_valid() { + let (os, arch) = parse_platform("linux/amd64").unwrap(); + assert_eq!(os, Os::Linux); + assert_eq!(arch, Arch::Amd64); + } + + #[test] + fn test_parse_platform_arm64() { + let (os, arch) = parse_platform("linux/arm64").unwrap(); + assert_eq!(os, Os::Linux); + assert_eq!(arch, Arch::ARM64); + } + + #[test] + fn test_parse_platform_invalid_format() { + let result = parse_platform("linux"); + assert!(result.is_err()); + assert!(matches!(result, Err(Error::InvalidArgument(_)))); + } + + #[test] + fn test_parse_platform_unknown_os() { + // Unknown OS values are accepted as Os::Other. + let (os, arch) = parse_platform("unknown_os/amd64").unwrap(); + assert_eq!(os, Os::Other("unknown_os".to_string())); + assert_eq!(arch, Arch::Amd64); + } + + #[test] + fn test_parse_platform_unknown_arch() { + // Unknown architecture values are accepted as Arch::Other. + let (os, arch) = parse_platform("linux/unknown_arch").unwrap(); + assert_eq!(os, Os::Linux); + assert_eq!(arch, Arch::Other("unknown_arch".to_string())); + } + + #[test] + fn test_parse_platform_too_many_parts() { + let result = parse_platform("linux/amd64/extra"); + assert!(result.is_err()); + assert!(matches!(result, Err(Error::InvalidArgument(_)))); + } +} From e9dc2be8c134bb37a670ac8e1f44ac7c90f2abd4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 04:22:05 +0000 Subject: [PATCH 02/12] Address review: use AUTHORIZATION constant, clean base64 encoding, add warn on missing reader Agent-Logs-Url: https://github.com/dragonflyoss/client/sessions/f89011d2-1920-46d0-9bc5-48e1f8e086e8 Co-authored-by: gaius-qi <15955374+gaius-qi@users.noreply.github.com> --- dragonfly-client-util/src/request/preheat.rs | 47 +++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/dragonfly-client-util/src/request/preheat.rs b/dragonfly-client-util/src/request/preheat.rs index 3cc8d39b..c4f79679 100644 --- a/dragonfly-client-util/src/request/preheat.rs +++ b/dragonfly-client-util/src/request/preheat.rs @@ -16,15 +16,16 @@ use super::errors::Error; use super::{GetRequest, Request, Result}; +use base64::Engine; use oci_client::client::ClientConfig; use oci_client::manifest::ImageIndexEntry; use oci_client::secrets::RegistryAuth; use oci_client::{Client as OciClient, Reference, RegistryOperation}; use oci_spec::image::{Arch, Os}; -use reqwest::header::{HeaderMap, HeaderValue}; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; use rustls_pki_types::CertificateDer; use std::time::Duration; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; /// PreheatRequest represents a request to preheat an OCI image through the /// Dragonfly seed client. The preheat downloads all blobs (config and layers) @@ -131,35 +132,29 @@ pub async fn preheat(request: &R, preheat_req: &PreheatRequest) -> R let mut auth_headers = HeaderMap::new(); if let Some(ref token) = token { auth_headers.insert( - "Authorization", + AUTHORIZATION, HeaderValue::from_str(&format!("Bearer {}", token)) .map_err(|err| Error::Internal(format!("invalid auth token: {}", err)))?, ); } else if let (Some(username), Some(password)) = (&preheat_req.username, &preheat_req.password) { - let credentials = base64::Engine::encode( - &base64::engine::general_purpose::STANDARD, - format!("{}:{}", username, password), - ); + let credentials = + base64::engine::general_purpose::STANDARD.encode(format!("{}:{}", username, password)); auth_headers.insert( - "Authorization", + AUTHORIZATION, HeaderValue::from_str(&format!("Basic {}", credentials)) .map_err(|err| Error::Internal(format!("invalid credentials: {}", err)))?, ); } - // Construct blob download URLs and download through Dragonfly. + // Download each blob (config + layers) through Dragonfly proxy. let registry = reference.resolve_registry(); let repository = reference.repository(); - // Collect all blob digests: config + layers. - let config_digest = manifest.config.digest.clone(); - let mut blob_digests: Vec = vec![config_digest]; - for layer in &manifest.layers { - blob_digests.push(layer.digest.clone()); - } + let blob_digests = + std::iter::once(&manifest.config.digest).chain(manifest.layers.iter().map(|l| &l.digest)); - for blob_digest in &blob_digests { + for blob_digest in blob_digests { let blob_url = format!( "https://{}/v2/{}/blobs/{}", registry, repository, blob_digest @@ -183,12 +178,20 @@ pub async fn preheat(request: &R, preheat_req: &PreheatRequest) -> R let response = request.get(get_request).await?; // Read and discard the body to complete the download through Dragonfly. - if let Some(mut reader) = response.reader { - tokio::io::copy(&mut reader, &mut tokio::io::sink()) - .await - .map_err(|err| { - Error::Internal(format!("failed to read blob {}: {}", blob_digest, err)) - })?; + match response.reader { + Some(mut reader) => { + tokio::io::copy(&mut reader, &mut tokio::io::sink()) + .await + .map_err(|err| { + Error::Internal(format!("failed to read blob {}: {}", blob_digest, err)) + })?; + } + None => { + warn!( + "no response body for blob {}, download may not have completed", + blob_digest + ); + } } info!("preheated blob: {}", blob_url); From 80508cd389cdcfa1c13486d5eadd31737b2ba884 Mon Sep 17 00:00:00 2001 From: Gaius Date: Thu, 9 Apr 2026 17:42:50 +0800 Subject: [PATCH 03/12] refactor(request): move preheat logic into Request trait as a method Signed-off-by: Gaius --- dragonfly-client-util/src/request/mod.rs | 222 ++++++++++++++- dragonfly-client-util/src/request/preheat.rs | 269 ------------------- 2 files changed, 215 insertions(+), 276 deletions(-) delete mode 100644 dragonfly-client-util/src/request/preheat.rs diff --git a/dragonfly-client-util/src/request/mod.rs b/dragonfly-client-util/src/request/mod.rs index f5f78e06..5ef3177d 100644 --- a/dragonfly-client-util/src/request/mod.rs +++ b/dragonfly-client-util/src/request/mod.rs @@ -14,10 +14,6 @@ * limitations under the License. */ -pub mod errors; -pub mod preheat; -mod selector; - use crate::digest::is_blob_url; use crate::http::{headermap_to_hashmap, query_params::default_proxy_rule_filtered_query_params}; use crate::id_generator::{IDGenerator, TaskIDParameter}; @@ -30,7 +26,15 @@ use dragonfly_api::scheduler::v2::scheduler_client::SchedulerClient; use errors::{BackendError, DfdaemonError, Error, ProxyError}; use futures::TryStreamExt; use hostname; -use reqwest::{header::HeaderMap, header::HeaderValue, Client}; +use oci_client::client::ClientConfig; +use oci_client::manifest::ImageIndexEntry; +use oci_client::secrets::RegistryAuth; +use oci_client::{Client as OciClient, Reference, RegistryOperation}; +use oci_spec::image::{Arch, Os}; +use reqwest::{ + header::{HeaderMap, HeaderValue, AUTHORIZATION}, + Client, +}; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_tracing::TracingMiddleware; use rustix::path::Arg; @@ -44,7 +48,10 @@ use std::time::Duration; use tokio::io::AsyncRead; use tokio_util::io::StreamReader; use tonic::transport::Endpoint; -use tracing::{debug, error}; +use tracing::{debug, error, warn}; + +pub mod errors; +mod selector; /// POOL_MAX_IDLE_PER_HOST is the max idle connections per host. const POOL_MAX_IDLE_PER_HOST: usize = 1024; @@ -94,6 +101,15 @@ pub trait Request { /// `BytesMut` buffer is used to store the response content, and the response metadata (e.g., status /// and headers) is returned separately. async fn get_into(&self, request: GetRequest, buf: &mut BytesMut) -> Result; + + /// Preheats an OCI image by downloading all its blobs via the Dragonfly. + /// + /// This method is designed for scenarios where OCI image content needs to be pre-cached in + /// the seed client before actual consumption, ensuring faster subsequent access across the + /// cluster. It parses the image reference, authenticates with the OCI registry, resolves + /// the image manifest (including multi-platform image indexes), and downloads each blob + /// (config and layers) through the seed client. + async fn preheat(&self, request: &PreheatRequest) -> Result<()>; } /// GetRequest represents a GET request to be sent via the Dragonfly. @@ -160,6 +176,63 @@ where pub reader: Option, } +/// PreheatRequest represents a request to preheat an OCI image through the +/// Dragonfly seed client. The preheat downloads all blobs (config and layers) +/// of the specified image via the Dragonfly proxy, effectively caching them +/// in the P2P network for faster downloading. +pub struct PreheatRequest { + /// Image is the OCI image reference (e.g., "docker.io/library/nginx:latest"). + pub image: String, + + /// Username for registry authentication. If not provided, anonymous access is used. + pub username: Option, + + /// Password for registry authentication. If not provided, anonymous access is used. + pub password: Option, + + /// Platform specifies the target platform in the format "os/arch" + /// (e.g., "linux/amd64", "linux/arm64"). This is used to select the correct + /// manifest from a multi-platform image index, default is current platform. + pub platform: Option, + + /// Piece length is the optional piece length for the Dragonfly task. + pub piece_length: Option, + + /// Tag identifies different tasks for the same URL. + pub tag: Option, + + /// Application identifies different tasks for the same URL. + pub application: Option, + + /// Filtered query params to generate the task id. + /// When filter is ["Signature", "Expires", "ns"], for example: + /// http://example.com/xyz?Expires=e1&Signature=s1&ns=docker.io and http://example.com/xyz?Expires=e2&Signature=s2&ns=docker.io + /// will generate the same task id. + /// Default value includes the filtered query params of s3, gcs, oss, obs, cos. + pub filtered_query_params: Vec, + + /// Content for calculating task id. This is used when the task ID cannot be calculated based + /// on URL and other parameters, such as when the URL contains dynamic query parameters that + /// cannot be filtered out. + pub content_for_calculating_task_id: Option, + + /// Enable task id based blob digest. It indicates whether to use the blob digest for task ID calculation + /// when downloading from OCI registries. When enabled for OCI blob URLs (e.g., /v2//blobs/sha256:), + /// the task ID is derived from the blob digest rather than the full URL. This enables deduplication across + /// registries - the same blob from different registries shares one task ID, eliminating redundant downloads + /// and storage. + pub enable_task_id_based_blob_digest: bool, + + /// Refer to https://github.com/dragonflyoss/api/blob/main/proto/common.proto#L67 + pub priority: Option, + + /// Timeout is the timeout for each blob download request. + pub timeout: Duration, + + /// Client cert is the optional client certificates for the request. + pub client_cert: Option>>, +} + /// Factory for creating HTTPClient instances. #[derive(Debug, Clone, Default)] struct HTTPClientFactory {} @@ -418,6 +491,106 @@ impl Request for Proxy { .await .map_err(|err| Error::RequestTimeout(err.to_string()))? } + + /// Preheats an OCI image by downloading all its blobs via the Dragonfly. + /// + /// This method is designed for scenarios where OCI image content needs to be pre-cached in + /// the seed client before actual consumption, ensuring faster subsequent access across the + /// cluster. It parses the image reference, authenticates with the OCI registry, resolves + /// the image manifest (including multi-platform image indexes), and downloads each blob + /// (config and layers) through the seed client. + async fn preheat(&self, request: &PreheatRequest) -> Result<()> { + let oci_client = Self::oci_client(request.platform.clone())?; + + // Parse image reference. + let reference: Reference = request + .image + .parse() + .map_err(|err| Error::InvalidArgument(format!("invalid image reference: {}", err)))?; + + // Create registry authentication. + let auth = match (&request.username, &request.password) { + (Some(username), Some(password)) => { + RegistryAuth::Basic(username.clone(), password.clone()) + } + _ => RegistryAuth::Anonymous, + }; + + // Pull image manifest. This handles multi-platform image index manifests + // by selecting the platform-specific manifest using our resolver. + let (manifest, digest) = oci_client + .pull_image_manifest(&reference, &auth) + .await + .map_err(|err| Error::Internal(format!("failed to pull image manifest: {}", err)))?; + debug!( + "pulled manifest for image {} with digest {}, layers: {}", + request.image, + digest, + manifest.layers.len() + ); + + // Authenticate with the registry and get a bearer token if available. + let token = oci_client + .auth(&reference, &auth, RegistryOperation::Pull) + .await + .map_err(|err| { + Error::Internal(format!("failed to authenticate with registry: {}", err)) + })? + .ok_or_else(|| { + Error::Internal("registry did not return authentication token".to_string()) + })?; + + // Build authorization header for blob downloads through the Dragonfly. + let mut header = HeaderMap::new(); + header.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", token)) + .map_err(|err| Error::Internal(format!("invalid auth token: {}", err)))?, + ); + + let registry = reference.resolve_registry(); + let repository = reference.repository(); + for digest in std::iter::once(&manifest.config.digest) + .chain(manifest.layers.iter().map(|layer| &layer.digest)) + { + let url = Self::build_blob_url(registry, repository, digest); + let get_request = GetRequest { + url: url.clone(), + header: Some(header.clone()), + piece_length: request.piece_length, + tag: request.tag.clone(), + application: request.application.clone(), + filtered_query_params: request.filtered_query_params.clone(), + content_for_calculating_task_id: request.content_for_calculating_task_id.clone(), + enable_task_id_based_blob_digest: request.enable_task_id_based_blob_digest, + priority: request.priority, + timeout: request.timeout, + client_cert: request.client_cert.clone(), + }; + + let response = self.get(get_request).await?; + match response.reader { + Some(mut reader) => { + tokio::io::copy(&mut reader, &mut tokio::io::sink()) + .await + .map_err(|err| { + Error::Internal(format!("failed to read blob {}: {}", digest, err)) + })?; + } + None => { + warn!( + "no response body for blob {}, download may not have completed", + digest + ); + } + } + + debug!("preheated blob: {}", url); + } + + debug!("preheat completed for image: {}", request.image); + Ok(()) + } } /// Proxy implements proxy request logic. @@ -568,7 +741,7 @@ impl Proxy { } } - /// make_request_headers applies p2p related headers to the request headers. + /// Make request headers applies p2p related headers to the request headers. fn make_request_headers(&self, request: &GetRequest) -> Result { let mut headers = request.header.clone().unwrap_or_default(); @@ -653,6 +826,41 @@ impl Proxy { headers.insert("X-Dragonfly-Use-P2P", HeaderValue::from_static("true")); Ok(headers) } + + /// Helper function to check if a URL is an OCI blob URL (e.g., /v2//blobs/sha256: + /// ). + fn build_blob_url(registry: &str, repository: &str, digest: &str) -> String { + format!("https://{}/v2/{}/blobs/{}", registry, repository, digest) + } + + /// Builds an OCI client with a platform resolver that matches the requested os/arch. + fn oci_client(platform: Option) -> Result { + let mut oci_config = ClientConfig::default(); + if let Some(platform) = platform { + let (os, arch) = platform + .split_once('/') + .map(|(os, arch)| (Os::from(os), Arch::from(arch))) + .ok_or_else(|| { + Error::InvalidArgument(format!( + "invalid platform format '{}', expected 'os/arch' (e.g., 'linux/amd64')", + platform + )) + })?; + + oci_config.platform_resolver = Some(Box::new(move |manifests: &[ImageIndexEntry]| { + manifests + .iter() + .find(|entry| { + entry.platform.as_ref().is_some_and(|platform| { + platform.os == os && platform.architecture == arch + }) + }) + .map(|entry| entry.digest.clone()) + })) + }; + + Ok(OciClient::new(oci_config)) + } } #[cfg(test)] diff --git a/dragonfly-client-util/src/request/preheat.rs b/dragonfly-client-util/src/request/preheat.rs deleted file mode 100644 index c4f79679..00000000 --- a/dragonfly-client-util/src/request/preheat.rs +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright 2025 The Dragonfly Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use super::errors::Error; -use super::{GetRequest, Request, Result}; -use base64::Engine; -use oci_client::client::ClientConfig; -use oci_client::manifest::ImageIndexEntry; -use oci_client::secrets::RegistryAuth; -use oci_client::{Client as OciClient, Reference, RegistryOperation}; -use oci_spec::image::{Arch, Os}; -use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; -use rustls_pki_types::CertificateDer; -use std::time::Duration; -use tracing::{debug, info, warn}; - -/// PreheatRequest represents a request to preheat an OCI image through the -/// Dragonfly seed client. The preheat downloads all blobs (config and layers) -/// of the specified image via the Dragonfly proxy, effectively caching them -/// in the P2P network for faster subsequent access. -pub struct PreheatRequest { - /// image is the OCI image reference (e.g., "docker.io/library/nginx:latest"). - pub image: String, - - /// username for registry authentication. If not provided, anonymous access is used. - pub username: Option, - - /// password for registry authentication. If not provided, anonymous access is used. - pub password: Option, - - /// platform specifies the target platform in the format "os/arch" - /// (e.g., "linux/amd64", "linux/arm64"). This is used to select the correct - /// manifest from a multi-platform image index. - pub platform: String, - - /// piece_length is the optional piece length for the Dragonfly task. - pub piece_length: Option, - - /// tag identifies different tasks for the same URL. - pub tag: Option, - - /// application identifies different tasks for the same URL. - pub application: Option, - - /// timeout is the timeout for each blob download request. - pub timeout: Duration, - - /// client_cert is the optional client certificates for the request. - pub client_cert: Option>>, -} - -/// Preheats an OCI image by downloading all its blobs through the Dragonfly proxy. -/// -/// This function performs the following steps: -/// 1. Parses the image reference and authenticates with the OCI registry. -/// 2. Pulls the image manifest (handling multi-platform image indexes). -/// 3. Downloads each blob (config + layers) through the Dragonfly proxy using the -/// provided `Request` implementation, discarding the content. -/// -/// The downloads go through the Dragonfly seed client's proxy, which caches the -/// content in the P2P network for faster subsequent access. -pub async fn preheat(request: &R, preheat_req: &PreheatRequest) -> Result<()> { - // Parse image reference. - let reference: Reference = - preheat_req - .image - .parse() - .map_err(|err: oci_client::ParseError| { - Error::InvalidArgument(format!("invalid image reference: {}", err)) - })?; - - // Create registry authentication. - let auth = match (&preheat_req.username, &preheat_req.password) { - (Some(username), Some(password)) => RegistryAuth::Basic(username.clone(), password.clone()), - _ => RegistryAuth::Anonymous, - }; - - // Parse platform (os/arch). - let (os, arch) = parse_platform(&preheat_req.platform)?; - - // Create OCI client with a platform resolver that matches the requested os/arch. - let oci_config = ClientConfig { - platform_resolver: Some(Box::new(move |manifests: &[ImageIndexEntry]| { - manifests - .iter() - .find(|entry| { - entry - .platform - .as_ref() - .is_some_and(|p| p.os == os && p.architecture == arch) - }) - .map(|e| e.digest.clone()) - })), - ..Default::default() - }; - let oci_client = OciClient::new(oci_config); - - // Authenticate with the registry and get a bearer token if available. - let token = oci_client - .auth(&reference, &auth, RegistryOperation::Pull) - .await - .map_err(|err| Error::Internal(format!("failed to authenticate with registry: {}", err)))?; - - // Pull image manifest. This handles multi-platform image index manifests - // by selecting the platform-specific manifest using our resolver. - let (manifest, digest) = oci_client - .pull_image_manifest(&reference, &auth) - .await - .map_err(|err| Error::Internal(format!("failed to pull image manifest: {}", err)))?; - - info!( - "pulled manifest for image {} with digest {}, layers: {}", - preheat_req.image, - digest, - manifest.layers.len() - ); - - // Build authorization header for blob downloads through the Dragonfly proxy. - let mut auth_headers = HeaderMap::new(); - if let Some(ref token) = token { - auth_headers.insert( - AUTHORIZATION, - HeaderValue::from_str(&format!("Bearer {}", token)) - .map_err(|err| Error::Internal(format!("invalid auth token: {}", err)))?, - ); - } else if let (Some(username), Some(password)) = (&preheat_req.username, &preheat_req.password) - { - let credentials = - base64::engine::general_purpose::STANDARD.encode(format!("{}:{}", username, password)); - auth_headers.insert( - AUTHORIZATION, - HeaderValue::from_str(&format!("Basic {}", credentials)) - .map_err(|err| Error::Internal(format!("invalid credentials: {}", err)))?, - ); - } - - // Download each blob (config + layers) through Dragonfly proxy. - let registry = reference.resolve_registry(); - let repository = reference.repository(); - - let blob_digests = - std::iter::once(&manifest.config.digest).chain(manifest.layers.iter().map(|l| &l.digest)); - - for blob_digest in blob_digests { - let blob_url = format!( - "https://{}/v2/{}/blobs/{}", - registry, repository, blob_digest - ); - - debug!("preheating blob: {}", blob_url); - - let get_request = GetRequest { - url: blob_url.clone(), - header: Some(auth_headers.clone()), - piece_length: preheat_req.piece_length, - tag: preheat_req.tag.clone(), - application: preheat_req.application.clone(), - filtered_query_params: vec![], - content_for_calculating_task_id: None, - priority: None, - timeout: preheat_req.timeout, - client_cert: preheat_req.client_cert.clone(), - }; - - let response = request.get(get_request).await?; - - // Read and discard the body to complete the download through Dragonfly. - match response.reader { - Some(mut reader) => { - tokio::io::copy(&mut reader, &mut tokio::io::sink()) - .await - .map_err(|err| { - Error::Internal(format!("failed to read blob {}: {}", blob_digest, err)) - })?; - } - None => { - warn!( - "no response body for blob {}, download may not have completed", - blob_digest - ); - } - } - - info!("preheated blob: {}", blob_url); - } - - info!("preheat completed for image: {}", preheat_req.image); - Ok(()) -} - -/// Parses a platform string in the format "os/arch" into Os and Arch types. -fn parse_platform(platform: &str) -> Result<(Os, Arch)> { - let parts: Vec<&str> = platform.split('/').collect(); - if parts.len() != 2 { - return Err(Error::InvalidArgument(format!( - "invalid platform format '{}', expected 'os/arch' (e.g., 'linux/amd64')", - platform - ))); - } - - let os: Os = serde_json::from_str(&format!("\"{}\"", parts[0])) - .map_err(|_| Error::InvalidArgument(format!("unsupported OS: {}", parts[0])))?; - let arch: Arch = serde_json::from_str(&format!("\"{}\"", parts[1])) - .map_err(|_| Error::InvalidArgument(format!("unsupported architecture: {}", parts[1])))?; - - Ok((os, arch)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_platform_valid() { - let (os, arch) = parse_platform("linux/amd64").unwrap(); - assert_eq!(os, Os::Linux); - assert_eq!(arch, Arch::Amd64); - } - - #[test] - fn test_parse_platform_arm64() { - let (os, arch) = parse_platform("linux/arm64").unwrap(); - assert_eq!(os, Os::Linux); - assert_eq!(arch, Arch::ARM64); - } - - #[test] - fn test_parse_platform_invalid_format() { - let result = parse_platform("linux"); - assert!(result.is_err()); - assert!(matches!(result, Err(Error::InvalidArgument(_)))); - } - - #[test] - fn test_parse_platform_unknown_os() { - // Unknown OS values are accepted as Os::Other. - let (os, arch) = parse_platform("unknown_os/amd64").unwrap(); - assert_eq!(os, Os::Other("unknown_os".to_string())); - assert_eq!(arch, Arch::Amd64); - } - - #[test] - fn test_parse_platform_unknown_arch() { - // Unknown architecture values are accepted as Arch::Other. - let (os, arch) = parse_platform("linux/unknown_arch").unwrap(); - assert_eq!(os, Os::Linux); - assert_eq!(arch, Arch::Other("unknown_arch".to_string())); - } - - #[test] - fn test_parse_platform_too_many_parts() { - let result = parse_platform("linux/amd64/extra"); - assert!(result.is_err()); - assert!(matches!(result, Err(Error::InvalidArgument(_)))); - } -} From 1251590bf89aba76f8cec0184bac8f04af3e861f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Apr 2026 10:21:30 +0000 Subject: [PATCH 04/12] fix: install ring CryptoProvider in test to fix --all-features CI failure When CI runs tests with --all-features, both ring and aws-lc-rs features are enabled for rustls, which prevents auto-detection of the CryptoProvider. Explicitly install ring as the default provider at the start of the test. Agent-Logs-Url: https://github.com/dragonflyoss/client/sessions/78ab4119-bb29-4144-99cd-b6a6b3bb7e4c Co-authored-by: gaius-qi <15955374+gaius-qi@users.noreply.github.com> --- dragonfly-client/src/resource/piece.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dragonfly-client/src/resource/piece.rs b/dragonfly-client/src/resource/piece.rs index 851dfe29..fe807a4a 100644 --- a/dragonfly-client/src/resource/piece.rs +++ b/dragonfly-client/src/resource/piece.rs @@ -1161,6 +1161,12 @@ mod tests { #[tokio::test] async fn test_calculate_interested() { + // Install the ring CryptoProvider as the process-level default. This is + // required when both `ring` and `aws-lc-rs` features are enabled (e.g., + // `--all-features` in CI), because rustls cannot auto-detect which + // provider to use. + let _ = rustls::crypto::ring::default_provider().install_default(); + let temp_dir = tempdir().unwrap(); let config = Config::default(); From 07d5dbecc90eaa5e7b2850bdaf73c022540d9986 Mon Sep 17 00:00:00 2001 From: Gaius Date: Fri, 10 Apr 2026 11:19:12 +0800 Subject: [PATCH 05/12] fix(deps): switch oci-client from rustls-tls to native-tls feature Signed-off-by: Gaius --- Cargo.lock | 54 ++------------------------ Cargo.toml | 2 +- dragonfly-client/src/resource/piece.rs | 6 --- 3 files changed, 4 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index caaf9bd3..e5b2e9dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -334,28 +334,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "aws-lc-rs" -version = "1.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.39.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - [[package]] name = "axum" version = "0.8.8" @@ -749,15 +727,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - [[package]] name = "colorchoice" version = "1.0.0" @@ -1529,12 +1498,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "either" version = "1.9.0" @@ -1765,12 +1728,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.32" @@ -4467,7 +4424,6 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ - "aws-lc-rs", "bytes", "fastbloom", "getrandom 0.3.1", @@ -4734,22 +4690,20 @@ dependencies = [ "http-body 1.0.0", "http-body-util", "hyper 1.9.0", - "hyper-rustls", + "hyper-tls", "hyper-util", "js-sys", "log", + "native-tls", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", "rustls-pki-types", - "rustls-platform-verifier", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-native-tls", "tokio-util", "tower", "tower-http", @@ -4990,7 +4944,6 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ - "aws-lc-rs", "log", "once_cell", "ring 0.17.7", @@ -5073,7 +5026,6 @@ version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ - "aws-lc-rs", "ring 0.17.7", "rustls-pki-types", "untrusted 0.9.0", diff --git a/Cargo.toml b/Cargo.toml index 9429f562..e0949622 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,7 +120,7 @@ cgroups-rs = "0.5" num_cpus = "1.17" async-trait = "0.1" humantime-serde = "1.1.1" -oci-client = { version = "0.16.1", default-features = false, features = ["rustls-tls"] } +oci-client = { version = "0.16.1", default-features = false, features = ["native-tls"] } oci-spec = "0.9" # TODO(Gaius): Remove the git dependency after the next release of mocktail, refer to https://github.com/IBM/mocktail/issues/65. diff --git a/dragonfly-client/src/resource/piece.rs b/dragonfly-client/src/resource/piece.rs index fe807a4a..851dfe29 100644 --- a/dragonfly-client/src/resource/piece.rs +++ b/dragonfly-client/src/resource/piece.rs @@ -1161,12 +1161,6 @@ mod tests { #[tokio::test] async fn test_calculate_interested() { - // Install the ring CryptoProvider as the process-level default. This is - // required when both `ring` and `aws-lc-rs` features are enabled (e.g., - // `--all-features` in CI), because rustls cannot auto-detect which - // provider to use. - let _ = rustls::crypto::ring::default_provider().install_default(); - let temp_dir = tempdir().unwrap(); let config = Config::default(); From 082f1a97746d889e8aec1eaf944ed60f6f84879a Mon Sep 17 00:00:00 2001 From: Gaius Date: Fri, 10 Apr 2026 11:24:01 +0800 Subject: [PATCH 06/12] chore(dragonfly-client-util): remove unused serde_json dependency Signed-off-by: Gaius --- Cargo.lock | 1 - dragonfly-client-util/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5b2e9dc..0d68824c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1484,7 +1484,6 @@ dependencies = [ "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", - "serde_json", "sha2", "sysinfo", "tempfile", diff --git a/dragonfly-client-util/Cargo.toml b/dragonfly-client-util/Cargo.toml index e7ef8e38..d5edd098 100644 --- a/dragonfly-client-util/Cargo.toml +++ b/dragonfly-client-util/Cargo.toml @@ -51,7 +51,6 @@ num_cpus.workspace = true humantime-serde.workspace = true oci-client.workspace = true oci-spec.workspace = true -serde_json.workspace = true rustix = { version = "1.1.3", features = ["fs"] } base64 = "0.22.1" pnet = "0.35.0" From 6196d0aa86a464aa42be16e102eb83817ca30c9a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Apr 2026 03:32:40 +0000 Subject: [PATCH 07/12] Add get and preheat examples for dragonfly-client-util/request Agent-Logs-Url: https://github.com/dragonflyoss/client/sessions/920339da-3afc-4501-a403-f914dd1c69a5 Co-authored-by: gaius-qi <15955374+gaius-qi@users.noreply.github.com> --- Cargo.lock | 1 + dragonfly-client-util/Cargo.toml | 1 + dragonfly-client-util/examples/get.rs | 86 ++++++++++++++++++++++ dragonfly-client-util/examples/preheat.rs | 87 +++++++++++++++++++++++ 4 files changed, 175 insertions(+) create mode 100644 dragonfly-client-util/examples/get.rs create mode 100644 dragonfly-client-util/examples/preheat.rs diff --git a/Cargo.lock b/Cargo.lock index 0d68824c..350fb4ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1444,6 +1444,7 @@ dependencies = [ name = "dragonfly-client-util" version = "1.3.2" dependencies = [ + "anyhow", "async-trait", "base64 0.22.1", "bytes", diff --git a/dragonfly-client-util/Cargo.toml b/dragonfly-client-util/Cargo.toml index d5edd098..c4a6559f 100644 --- a/dragonfly-client-util/Cargo.toml +++ b/dragonfly-client-util/Cargo.toml @@ -65,3 +65,4 @@ cgroups-rs.workspace = true [dev-dependencies] tempfile.workspace = true mocktail.workspace = true +anyhow.workspace = true diff --git a/dragonfly-client-util/examples/get.rs b/dragonfly-client-util/examples/get.rs new file mode 100644 index 00000000..da3f63c4 --- /dev/null +++ b/dragonfly-client-util/examples/get.rs @@ -0,0 +1,86 @@ +/* + * Copyright 2025 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use dragonfly_client_util::request::{GetRequest, Proxy, Request}; +use std::time::Duration; +use tokio::io::AsyncReadExt; + +/// This example demonstrates how to use the `get` method of the Dragonfly request module +/// to download a file via the Dragonfly P2P network. +/// +/// Prerequisites: +/// 1. A running Dragonfly scheduler service. +/// 2. At least one seed peer registered with the scheduler. +/// +/// Usage: +/// Set the `DRAGONFLY_SCHEDULER_ENDPOINT` environment variable to the scheduler's gRPC endpoint, +/// then run the example: +/// +/// ```shell +/// export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" +/// cargo run -p dragonfly-client-util --example get +/// ``` +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Read the scheduler endpoint from an environment variable, falling back to a default. + let scheduler_endpoint = std::env::var("DRAGONFLY_SCHEDULER_ENDPOINT") + .unwrap_or_else(|_| "http://127.0.0.1:8002".to_string()); + + // Build a Proxy client that communicates with the Dragonfly scheduler and seed peers. + let proxy = Proxy::builder() + .scheduler_endpoint(scheduler_endpoint) + .scheduler_request_timeout(Duration::from_secs(5)) + .health_check_interval(Duration::from_secs(60)) + .max_retries(3) + .build() + .await + .map_err(|err| anyhow::anyhow!("failed to build proxy: {}", err))?; + + // Create a GET request for the target URL. The request is routed through the Dragonfly + // seed peer proxy so that the content is cached in the P2P network. + let request = GetRequest { + url: "https://example.com/path/to/file".to_string(), + header: None, + piece_length: None, + tag: None, + application: None, + filtered_query_params: Vec::new(), + content_for_calculating_task_id: None, + enable_task_id_based_blob_digest: false, + priority: None, + timeout: Duration::from_secs(300), + client_cert: None, + }; + + // Send the GET request and receive a streaming response. + let response = proxy + .get(request) + .await + .map_err(|err| anyhow::anyhow!("get request failed: {}", err))?; + + println!("Response success: {}", response.success); + println!("Response status: {:?}", response.status_code); + println!("Response headers: {:?}", response.header); + + // Read the response body from the streaming reader. + if let Some(mut reader) = response.reader { + let mut body = Vec::new(); + reader.read_to_end(&mut body).await?; + println!("Downloaded {} bytes", body.len()); + } + + Ok(()) +} diff --git a/dragonfly-client-util/examples/preheat.rs b/dragonfly-client-util/examples/preheat.rs new file mode 100644 index 00000000..acce155c --- /dev/null +++ b/dragonfly-client-util/examples/preheat.rs @@ -0,0 +1,87 @@ +/* + * Copyright 2025 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use dragonfly_client_util::request::{PreheatRequest, Proxy, Request}; +use std::time::Duration; + +/// This example demonstrates how to use the `preheat` method of the Dragonfly request module +/// to pre-cache an OCI image via the Dragonfly P2P network. +/// +/// The example preheats the `dragonflyoss/scheduler:v2.4.3` image for the `linux/amd64` platform. +/// All blobs (config and layers) are downloaded through the Dragonfly seed peer proxy and cached +/// in the P2P network, ensuring fast access for subsequent pulls across the cluster. +/// +/// Prerequisites: +/// 1. A running Dragonfly scheduler service. +/// 2. At least one seed peer registered with the scheduler. +/// +/// Usage: +/// Set the `DRAGONFLY_SCHEDULER_ENDPOINT` environment variable to the scheduler's gRPC endpoint, +/// then run the example: +/// +/// ```shell +/// export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" +/// cargo run -p dragonfly-client-util --example preheat +/// ``` +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Read the scheduler endpoint from an environment variable, falling back to a default. + let scheduler_endpoint = std::env::var("DRAGONFLY_SCHEDULER_ENDPOINT") + .unwrap_or_else(|_| "http://127.0.0.1:8002".to_string()); + + // Build a Proxy client that communicates with the Dragonfly scheduler and seed peers. + let proxy = Proxy::builder() + .scheduler_endpoint(scheduler_endpoint) + .scheduler_request_timeout(Duration::from_secs(5)) + .health_check_interval(Duration::from_secs(60)) + .max_retries(3) + .build() + .await + .map_err(|err| anyhow::anyhow!("failed to build proxy: {}", err))?; + + // Create a preheat request for the dragonflyoss/scheduler:v2.4.3 image. + // Anonymous authentication is used here since the image is public. For private + // registries, set `username` and `password` to the appropriate credentials. + let request = PreheatRequest { + image: "docker.io/dragonflyoss/scheduler:v2.4.3".to_string(), + username: None, + password: None, + platform: Some("linux/amd64".to_string()), + piece_length: None, + tag: None, + application: None, + filtered_query_params: Vec::new(), + content_for_calculating_task_id: None, + enable_task_id_based_blob_digest: false, + priority: None, + timeout: Duration::from_secs(600), + client_cert: None, + }; + + // Preheat the image. This downloads all blobs (config + layers) through the Dragonfly + // seed peer proxy, caching them in the P2P network. + proxy + .preheat(&request) + .await + .map_err(|err| anyhow::anyhow!("preheat failed: {}", err))?; + + println!( + "Successfully preheated image: {}", + "docker.io/dragonflyoss/scheduler:v2.4.3" + ); + + Ok(()) +} From f05c75a375bf39c83c32cf0264b9d7faf987712a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Apr 2026 03:48:09 +0000 Subject: [PATCH 08/12] Restructure get/preheat examples as separate workspace crates following plugin example pattern Agent-Logs-Url: https://github.com/dragonflyoss/client/sessions/efd37f6c-3bcc-4270-929b-ad5fd79cb709 Co-authored-by: gaius-qi <15955374+gaius-qi@users.noreply.github.com> --- Cargo.lock | 19 ++++++++++++++++++- Cargo.toml | 2 ++ dragonfly-client-util/Cargo.toml | 1 - dragonfly-client-util/examples/get/Cargo.toml | 16 ++++++++++++++++ dragonfly-client-util/examples/get/README.md | 15 +++++++++++++++ .../examples/{get.rs => get/src/main.rs} | 2 +- .../examples/preheat/Cargo.toml | 16 ++++++++++++++++ .../examples/preheat/README.md | 17 +++++++++++++++++ .../{preheat.rs => preheat/src/main.rs} | 7 ++----- 9 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 dragonfly-client-util/examples/get/Cargo.toml create mode 100644 dragonfly-client-util/examples/get/README.md rename dragonfly-client-util/examples/{get.rs => get/src/main.rs} (98%) create mode 100644 dragonfly-client-util/examples/preheat/Cargo.toml create mode 100644 dragonfly-client-util/examples/preheat/README.md rename dragonfly-client-util/examples/{preheat.rs => preheat/src/main.rs} (94%) diff --git a/Cargo.lock b/Cargo.lock index 350fb4ca..d277ea6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1444,7 +1444,6 @@ dependencies = [ name = "dragonfly-client-util" version = "1.3.2" dependencies = [ - "anyhow", "async-trait", "base64 0.22.1", "bytes", @@ -1853,6 +1852,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "get" +version = "1.3.2" +dependencies = [ + "anyhow", + "dragonfly-client-util", + "tokio", +] + [[package]] name = "getrandom" version = "0.2.12" @@ -4056,6 +4064,15 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "preheat" +version = "1.3.2" +dependencies = [ + "anyhow", + "dragonfly-client-util", + "tokio", +] + [[package]] name = "prettyplease" version = "0.2.17" diff --git a/Cargo.toml b/Cargo.toml index e0949622..1fffccb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,8 @@ members = [ "dragonfly-client-storage", "dragonfly-client-util", "dragonfly-client-backend/examples/plugin", + "dragonfly-client-util/examples/get", + "dragonfly-client-util/examples/preheat", "dragonfly-client-metric", ] diff --git a/dragonfly-client-util/Cargo.toml b/dragonfly-client-util/Cargo.toml index c4a6559f..d5edd098 100644 --- a/dragonfly-client-util/Cargo.toml +++ b/dragonfly-client-util/Cargo.toml @@ -65,4 +65,3 @@ cgroups-rs.workspace = true [dev-dependencies] tempfile.workspace = true mocktail.workspace = true -anyhow.workspace = true diff --git a/dragonfly-client-util/examples/get/Cargo.toml b/dragonfly-client-util/examples/get/Cargo.toml new file mode 100644 index 00000000..3beac43b --- /dev/null +++ b/dragonfly-client-util/examples/get/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "get" +description = "An example of using the Dragonfly client to download a file via the P2P network" +version.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +keywords.workspace = true +license.workspace = true +edition.workspace = true +publish = false + +[dependencies] +dragonfly-client-util.workspace = true +tokio.workspace = true +anyhow.workspace = true diff --git a/dragonfly-client-util/examples/get/README.md b/dragonfly-client-util/examples/get/README.md new file mode 100644 index 00000000..41a5bb12 --- /dev/null +++ b/dragonfly-client-util/examples/get/README.md @@ -0,0 +1,15 @@ +# Example of GET Request + +An example of downloading a file via the Dragonfly P2P network using the `get` method. + +## Prerequisites + +1. A running Dragonfly scheduler service. +2. At least one seed peer registered with the scheduler. + +## Run Example + +```shell +export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" +cargo run -p get +``` diff --git a/dragonfly-client-util/examples/get.rs b/dragonfly-client-util/examples/get/src/main.rs similarity index 98% rename from dragonfly-client-util/examples/get.rs rename to dragonfly-client-util/examples/get/src/main.rs index da3f63c4..4d2a039f 100644 --- a/dragonfly-client-util/examples/get.rs +++ b/dragonfly-client-util/examples/get/src/main.rs @@ -31,7 +31,7 @@ use tokio::io::AsyncReadExt; /// /// ```shell /// export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" -/// cargo run -p dragonfly-client-util --example get +/// cargo run -p get /// ``` #[tokio::main] async fn main() -> anyhow::Result<()> { diff --git a/dragonfly-client-util/examples/preheat/Cargo.toml b/dragonfly-client-util/examples/preheat/Cargo.toml new file mode 100644 index 00000000..bfb07a09 --- /dev/null +++ b/dragonfly-client-util/examples/preheat/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "preheat" +description = "An example of using the Dragonfly client to preheat an OCI image via the P2P network" +version.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +keywords.workspace = true +license.workspace = true +edition.workspace = true +publish = false + +[dependencies] +dragonfly-client-util.workspace = true +tokio.workspace = true +anyhow.workspace = true diff --git a/dragonfly-client-util/examples/preheat/README.md b/dragonfly-client-util/examples/preheat/README.md new file mode 100644 index 00000000..a3c58a8c --- /dev/null +++ b/dragonfly-client-util/examples/preheat/README.md @@ -0,0 +1,17 @@ +# Example of Preheat Request + +An example of preheating an OCI image (`dragonflyoss/scheduler:v2.4.3`) via the Dragonfly P2P network +using the `preheat` method. All blobs (config and layers) are downloaded through the Dragonfly +seed peer proxy and cached in the P2P network. + +## Prerequisites + +1. A running Dragonfly scheduler service. +2. At least one seed peer registered with the scheduler. + +## Run Example + +```shell +export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" +cargo run -p preheat +``` diff --git a/dragonfly-client-util/examples/preheat.rs b/dragonfly-client-util/examples/preheat/src/main.rs similarity index 94% rename from dragonfly-client-util/examples/preheat.rs rename to dragonfly-client-util/examples/preheat/src/main.rs index acce155c..e9e27dc8 100644 --- a/dragonfly-client-util/examples/preheat.rs +++ b/dragonfly-client-util/examples/preheat/src/main.rs @@ -34,7 +34,7 @@ use std::time::Duration; /// /// ```shell /// export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" -/// cargo run -p dragonfly-client-util --example preheat +/// cargo run -p preheat /// ``` #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -78,10 +78,7 @@ async fn main() -> anyhow::Result<()> { .await .map_err(|err| anyhow::anyhow!("preheat failed: {}", err))?; - println!( - "Successfully preheated image: {}", - "docker.io/dragonflyoss/scheduler:v2.4.3" - ); + println!("Successfully preheated image: docker.io/dragonflyoss/scheduler:v2.4.3"); Ok(()) } From 3d08e3eb8adc67e26f1b4b55fd869bf82dcfc06e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Apr 2026 05:53:58 +0000 Subject: [PATCH 09/12] Combine get and preheat examples into a single request package Agent-Logs-Url: https://github.com/dragonflyoss/client/sessions/bd783370-64d6-4cdc-a4c8-73a0be8c77c9 Co-authored-by: gaius-qi <15955374+gaius-qi@users.noreply.github.com> --- Cargo.lock | 27 ++++++------------ Cargo.toml | 3 +- dragonfly-client-util/examples/get/Cargo.toml | 16 ----------- dragonfly-client-util/examples/get/README.md | 15 ---------- .../examples/preheat/README.md | 17 ----------- .../examples/{preheat => request}/Cargo.toml | 12 ++++++-- .../examples/request/README.md | 28 +++++++++++++++++++ .../src/main.rs => request/src/bin/get.rs} | 0 .../main.rs => request/src/bin/preheat.rs} | 0 9 files changed, 48 insertions(+), 70 deletions(-) delete mode 100644 dragonfly-client-util/examples/get/Cargo.toml delete mode 100644 dragonfly-client-util/examples/get/README.md delete mode 100644 dragonfly-client-util/examples/preheat/README.md rename dragonfly-client-util/examples/{preheat => request}/Cargo.toml (58%) create mode 100644 dragonfly-client-util/examples/request/README.md rename dragonfly-client-util/examples/{get/src/main.rs => request/src/bin/get.rs} (100%) rename dragonfly-client-util/examples/{preheat/src/main.rs => request/src/bin/preheat.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index d277ea6e..f10d425b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1852,15 +1852,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "get" -version = "1.3.2" -dependencies = [ - "anyhow", - "dragonfly-client-util", - "tokio", -] - [[package]] name = "getrandom" version = "0.2.12" @@ -4064,15 +4055,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "preheat" -version = "1.3.2" -dependencies = [ - "anyhow", - "dragonfly-client-util", - "tokio", -] - [[package]] name = "prettyplease" version = "0.2.17" @@ -4643,6 +4625,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "request" +version = "1.3.2" +dependencies = [ + "anyhow", + "dragonfly-client-util", + "tokio", +] + [[package]] name = "reqwest" version = "0.12.28" diff --git a/Cargo.toml b/Cargo.toml index 1fffccb5..7beefc3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,8 +9,7 @@ members = [ "dragonfly-client-storage", "dragonfly-client-util", "dragonfly-client-backend/examples/plugin", - "dragonfly-client-util/examples/get", - "dragonfly-client-util/examples/preheat", + "dragonfly-client-util/examples/request", "dragonfly-client-metric", ] diff --git a/dragonfly-client-util/examples/get/Cargo.toml b/dragonfly-client-util/examples/get/Cargo.toml deleted file mode 100644 index 3beac43b..00000000 --- a/dragonfly-client-util/examples/get/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "get" -description = "An example of using the Dragonfly client to download a file via the P2P network" -version.workspace = true -authors.workspace = true -homepage.workspace = true -repository.workspace = true -keywords.workspace = true -license.workspace = true -edition.workspace = true -publish = false - -[dependencies] -dragonfly-client-util.workspace = true -tokio.workspace = true -anyhow.workspace = true diff --git a/dragonfly-client-util/examples/get/README.md b/dragonfly-client-util/examples/get/README.md deleted file mode 100644 index 41a5bb12..00000000 --- a/dragonfly-client-util/examples/get/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Example of GET Request - -An example of downloading a file via the Dragonfly P2P network using the `get` method. - -## Prerequisites - -1. A running Dragonfly scheduler service. -2. At least one seed peer registered with the scheduler. - -## Run Example - -```shell -export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" -cargo run -p get -``` diff --git a/dragonfly-client-util/examples/preheat/README.md b/dragonfly-client-util/examples/preheat/README.md deleted file mode 100644 index a3c58a8c..00000000 --- a/dragonfly-client-util/examples/preheat/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Example of Preheat Request - -An example of preheating an OCI image (`dragonflyoss/scheduler:v2.4.3`) via the Dragonfly P2P network -using the `preheat` method. All blobs (config and layers) are downloaded through the Dragonfly -seed peer proxy and cached in the P2P network. - -## Prerequisites - -1. A running Dragonfly scheduler service. -2. At least one seed peer registered with the scheduler. - -## Run Example - -```shell -export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" -cargo run -p preheat -``` diff --git a/dragonfly-client-util/examples/preheat/Cargo.toml b/dragonfly-client-util/examples/request/Cargo.toml similarity index 58% rename from dragonfly-client-util/examples/preheat/Cargo.toml rename to dragonfly-client-util/examples/request/Cargo.toml index bfb07a09..3ecb762f 100644 --- a/dragonfly-client-util/examples/preheat/Cargo.toml +++ b/dragonfly-client-util/examples/request/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "preheat" -description = "An example of using the Dragonfly client to preheat an OCI image via the P2P network" +name = "request" +description = "Examples of using the Dragonfly client request module to download files and preheat OCI images via the P2P network" version.workspace = true authors.workspace = true homepage.workspace = true @@ -10,6 +10,14 @@ license.workspace = true edition.workspace = true publish = false +[[bin]] +name = "get" +path = "src/bin/get.rs" + +[[bin]] +name = "preheat" +path = "src/bin/preheat.rs" + [dependencies] dragonfly-client-util.workspace = true tokio.workspace = true diff --git a/dragonfly-client-util/examples/request/README.md b/dragonfly-client-util/examples/request/README.md new file mode 100644 index 00000000..ad19b7c1 --- /dev/null +++ b/dragonfly-client-util/examples/request/README.md @@ -0,0 +1,28 @@ +# Examples of Request + +Examples of downloading a file and preheating an OCI image via the Dragonfly P2P network using the `request` module. + +## Prerequisites + +1. A running Dragonfly scheduler service. +2. At least one seed peer registered with the scheduler. + +## Run GET Example + +Downloads a file via the Dragonfly P2P network using the `get` method. + +```shell +export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" +cargo run -p request --bin get +``` + +## Run Preheat Example + +Preheats an OCI image (`dragonflyoss/scheduler:v2.4.3`) via the Dragonfly P2P network +using the `preheat` method. All blobs (config and layers) are downloaded through the +Dragonfly seed peer proxy and cached in the P2P network. + +```shell +export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" +cargo run -p request --bin preheat +``` diff --git a/dragonfly-client-util/examples/get/src/main.rs b/dragonfly-client-util/examples/request/src/bin/get.rs similarity index 100% rename from dragonfly-client-util/examples/get/src/main.rs rename to dragonfly-client-util/examples/request/src/bin/get.rs diff --git a/dragonfly-client-util/examples/preheat/src/main.rs b/dragonfly-client-util/examples/request/src/bin/preheat.rs similarity index 100% rename from dragonfly-client-util/examples/preheat/src/main.rs rename to dragonfly-client-util/examples/request/src/bin/preheat.rs From 6784cee497f89a2af0883ad0b15bd5c3d64d93e9 Mon Sep 17 00:00:00 2001 From: Gaius Date: Fri, 10 Apr 2026 14:58:58 +0800 Subject: [PATCH 10/12] refactor(request): change `get` and `get_into` to accept `&GetRequest Signed-off-by: Gaius --- .../examples/plugin/src/lib.rs | 2 +- .../examples/request/Cargo.toml | 2 +- .../examples/request/README.md | 7 +---- .../examples/request/src/bin/get.rs | 31 +++---------------- .../examples/request/src/bin/preheat.rs | 31 ++----------------- dragonfly-client-util/src/request/mod.rs | 8 ++--- 6 files changed, 14 insertions(+), 67 deletions(-) diff --git a/dragonfly-client-backend/examples/plugin/src/lib.rs b/dragonfly-client-backend/examples/plugin/src/lib.rs index ef1ee97b..7ee4c672 100644 --- a/dragonfly-client-backend/examples/plugin/src/lib.rs +++ b/dragonfly-client-backend/examples/plugin/src/lib.rs @@ -64,7 +64,7 @@ impl Backend for Hdfs { } } -/// register_plugin is a function that returns a Box. +/// Register plugin is a function that returns a Box. /// This function is used to register the HDFS plugin to the Backend. #[no_mangle] pub fn register_plugin() -> Box { diff --git a/dragonfly-client-util/examples/request/Cargo.toml b/dragonfly-client-util/examples/request/Cargo.toml index 3ecb762f..c87d78d2 100644 --- a/dragonfly-client-util/examples/request/Cargo.toml +++ b/dragonfly-client-util/examples/request/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "request" -description = "Examples of using the Dragonfly client request module to download files and preheat OCI images via the P2P network" +description = "An example of request library for the Dragonfly client" version.workspace = true authors.workspace = true homepage.workspace = true diff --git a/dragonfly-client-util/examples/request/README.md b/dragonfly-client-util/examples/request/README.md index ad19b7c1..bdda5a61 100644 --- a/dragonfly-client-util/examples/request/README.md +++ b/dragonfly-client-util/examples/request/README.md @@ -1,11 +1,6 @@ # Examples of Request -Examples of downloading a file and preheating an OCI image via the Dragonfly P2P network using the `request` module. - -## Prerequisites - -1. A running Dragonfly scheduler service. -2. At least one seed peer registered with the scheduler. +An example of using the `request` module to download a file and preheat an OCI image via the Dragonfly P2P network. ## Run GET Example diff --git a/dragonfly-client-util/examples/request/src/bin/get.rs b/dragonfly-client-util/examples/request/src/bin/get.rs index 4d2a039f..b1c00541 100644 --- a/dragonfly-client-util/examples/request/src/bin/get.rs +++ b/dragonfly-client-util/examples/request/src/bin/get.rs @@ -1,5 +1,5 @@ /* - * Copyright 2025 The Dragonfly Authors + * Copyright 2026 The Dragonfly Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,28 +18,13 @@ use dragonfly_client_util::request::{GetRequest, Proxy, Request}; use std::time::Duration; use tokio::io::AsyncReadExt; -/// This example demonstrates how to use the `get` method of the Dragonfly request module +/// This example demonstrates how to use the get method of the Dragonfly request module /// to download a file via the Dragonfly P2P network. -/// -/// Prerequisites: -/// 1. A running Dragonfly scheduler service. -/// 2. At least one seed peer registered with the scheduler. -/// -/// Usage: -/// Set the `DRAGONFLY_SCHEDULER_ENDPOINT` environment variable to the scheduler's gRPC endpoint, -/// then run the example: -/// -/// ```shell -/// export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" -/// cargo run -p get -/// ``` #[tokio::main] async fn main() -> anyhow::Result<()> { - // Read the scheduler endpoint from an environment variable, falling back to a default. let scheduler_endpoint = std::env::var("DRAGONFLY_SCHEDULER_ENDPOINT") .unwrap_or_else(|_| "http://127.0.0.1:8002".to_string()); - // Build a Proxy client that communicates with the Dragonfly scheduler and seed peers. let proxy = Proxy::builder() .scheduler_endpoint(scheduler_endpoint) .scheduler_request_timeout(Duration::from_secs(5)) @@ -49,8 +34,6 @@ async fn main() -> anyhow::Result<()> { .await .map_err(|err| anyhow::anyhow!("failed to build proxy: {}", err))?; - // Create a GET request for the target URL. The request is routed through the Dragonfly - // seed peer proxy so that the content is cached in the P2P network. let request = GetRequest { url: "https://example.com/path/to/file".to_string(), header: None, @@ -65,21 +48,15 @@ async fn main() -> anyhow::Result<()> { client_cert: None, }; - // Send the GET request and receive a streaming response. let response = proxy - .get(request) + .get(&request) .await .map_err(|err| anyhow::anyhow!("get request failed: {}", err))?; - println!("Response success: {}", response.success); - println!("Response status: {:?}", response.status_code); - println!("Response headers: {:?}", response.header); - - // Read the response body from the streaming reader. if let Some(mut reader) = response.reader { let mut body = Vec::new(); reader.read_to_end(&mut body).await?; - println!("Downloaded {} bytes", body.len()); + println!("{} downloaded, size: {} bytes", &request.url, body.len()); } Ok(()) diff --git a/dragonfly-client-util/examples/request/src/bin/preheat.rs b/dragonfly-client-util/examples/request/src/bin/preheat.rs index e9e27dc8..186f5f09 100644 --- a/dragonfly-client-util/examples/request/src/bin/preheat.rs +++ b/dragonfly-client-util/examples/request/src/bin/preheat.rs @@ -1,5 +1,5 @@ /* - * Copyright 2025 The Dragonfly Authors + * Copyright 2026 The Dragonfly Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,32 +17,13 @@ use dragonfly_client_util::request::{PreheatRequest, Proxy, Request}; use std::time::Duration; -/// This example demonstrates how to use the `preheat` method of the Dragonfly request module +/// This example demonstrates how to use the preheat method of the Dragonfly request module /// to pre-cache an OCI image via the Dragonfly P2P network. -/// -/// The example preheats the `dragonflyoss/scheduler:v2.4.3` image for the `linux/amd64` platform. -/// All blobs (config and layers) are downloaded through the Dragonfly seed peer proxy and cached -/// in the P2P network, ensuring fast access for subsequent pulls across the cluster. -/// -/// Prerequisites: -/// 1. A running Dragonfly scheduler service. -/// 2. At least one seed peer registered with the scheduler. -/// -/// Usage: -/// Set the `DRAGONFLY_SCHEDULER_ENDPOINT` environment variable to the scheduler's gRPC endpoint, -/// then run the example: -/// -/// ```shell -/// export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" -/// cargo run -p preheat -/// ``` #[tokio::main] async fn main() -> anyhow::Result<()> { - // Read the scheduler endpoint from an environment variable, falling back to a default. let scheduler_endpoint = std::env::var("DRAGONFLY_SCHEDULER_ENDPOINT") .unwrap_or_else(|_| "http://127.0.0.1:8002".to_string()); - // Build a Proxy client that communicates with the Dragonfly scheduler and seed peers. let proxy = Proxy::builder() .scheduler_endpoint(scheduler_endpoint) .scheduler_request_timeout(Duration::from_secs(5)) @@ -52,9 +33,6 @@ async fn main() -> anyhow::Result<()> { .await .map_err(|err| anyhow::anyhow!("failed to build proxy: {}", err))?; - // Create a preheat request for the dragonflyoss/scheduler:v2.4.3 image. - // Anonymous authentication is used here since the image is public. For private - // registries, set `username` and `password` to the appropriate credentials. let request = PreheatRequest { image: "docker.io/dragonflyoss/scheduler:v2.4.3".to_string(), username: None, @@ -71,14 +49,11 @@ async fn main() -> anyhow::Result<()> { client_cert: None, }; - // Preheat the image. This downloads all blobs (config + layers) through the Dragonfly - // seed peer proxy, caching them in the P2P network. proxy .preheat(&request) .await .map_err(|err| anyhow::anyhow!("preheat failed: {}", err))?; - println!("Successfully preheated image: docker.io/dragonflyoss/scheduler:v2.4.3"); - + println!("{} has been preheated", request.image); Ok(()) } diff --git a/dragonfly-client-util/src/request/mod.rs b/dragonfly-client-util/src/request/mod.rs index 5ef3177d..20714a34 100644 --- a/dragonfly-client-util/src/request/mod.rs +++ b/dragonfly-client-util/src/request/mod.rs @@ -91,7 +91,7 @@ pub trait Request { /// This method is designed for scenarios where the response body is expected to be processed as a /// stream, allowing efficient handling of large or continuous data. The response includes metadata /// such as status codes and headers, along with a streaming `Body` for accessing the response content. - async fn get(&self, request: GetRequest) -> Result>; + async fn get(&self, request: &GetRequest) -> Result>; /// Sends an GET request to a remote server via the Dragonfly and writes the response /// body directly into the provided buffer. @@ -100,7 +100,7 @@ pub trait Request { /// memory, avoiding the overhead of streaming for smaller or fixed-size responses. The provided /// `BytesMut` buffer is used to store the response content, and the response metadata (e.g., status /// and headers) is returned separately. - async fn get_into(&self, request: GetRequest, buf: &mut BytesMut) -> Result; + async fn get_into(&self, request: &GetRequest, buf: &mut BytesMut) -> Result; /// Preheats an OCI image by downloading all its blobs via the Dragonfly. /// @@ -439,7 +439,7 @@ impl Request for Proxy { /// This method is designed for scenarios where the response body is expected to be processed as a /// stream, allowing efficient handling of large or continuous data. The response includes metadata /// such as status codes and headers, along with a streaming `Body` for accessing the response content. - async fn get(&self, request: GetRequest) -> Result { + async fn get(&self, request: &GetRequest) -> Result { let response = self.try_send(&request).await?; let header = response.headers().clone(); let status_code = response.status(); @@ -464,7 +464,7 @@ impl Request for Proxy { /// memory, avoiding the overhead of streaming for smaller or fixed-size responses. The provided /// `BytesMut` buffer is used to store the response content, and the response metadata (e.g., status /// and headers) is returned separately. - async fn get_into(&self, request: GetRequest, buf: &mut BytesMut) -> Result { + async fn get_into(&self, request: &GetRequest, buf: &mut BytesMut) -> Result { let get_into = async { let response = self.try_send(&request).await?; let status = response.status(); From 6db30ae09ef380cba51133a0f9bfb473cefa411a Mon Sep 17 00:00:00 2001 From: Gaius Date: Fri, 10 Apr 2026 15:47:11 +0800 Subject: [PATCH 11/12] refactor: add Default impl for GetRequest and PreheatRequest structs Signed-off-by: Gaius --- .../examples/request/src/bin/get.rs | 11 +--- .../examples/request/src/bin/preheat.rs | 12 +--- dragonfly-client-util/src/request/mod.rs | 65 +++++++++++++++---- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/dragonfly-client-util/examples/request/src/bin/get.rs b/dragonfly-client-util/examples/request/src/bin/get.rs index b1c00541..fb0db5b1 100644 --- a/dragonfly-client-util/examples/request/src/bin/get.rs +++ b/dragonfly-client-util/examples/request/src/bin/get.rs @@ -36,16 +36,7 @@ async fn main() -> anyhow::Result<()> { let request = GetRequest { url: "https://example.com/path/to/file".to_string(), - header: None, - piece_length: None, - tag: None, - application: None, - filtered_query_params: Vec::new(), - content_for_calculating_task_id: None, - enable_task_id_based_blob_digest: false, - priority: None, - timeout: Duration::from_secs(300), - client_cert: None, + ..Default::default() }; let response = proxy diff --git a/dragonfly-client-util/examples/request/src/bin/preheat.rs b/dragonfly-client-util/examples/request/src/bin/preheat.rs index 186f5f09..a1c8132e 100644 --- a/dragonfly-client-util/examples/request/src/bin/preheat.rs +++ b/dragonfly-client-util/examples/request/src/bin/preheat.rs @@ -35,18 +35,8 @@ async fn main() -> anyhow::Result<()> { let request = PreheatRequest { image: "docker.io/dragonflyoss/scheduler:v2.4.3".to_string(), - username: None, - password: None, platform: Some("linux/amd64".to_string()), - piece_length: None, - tag: None, - application: None, - filtered_query_params: Vec::new(), - content_for_calculating_task_id: None, - enable_task_id_based_blob_digest: false, - priority: None, - timeout: Duration::from_secs(600), - client_cert: None, + ..Default::default() }; proxy diff --git a/dragonfly-client-util/src/request/mod.rs b/dragonfly-client-util/src/request/mod.rs index 20714a34..7bddc5d4 100644 --- a/dragonfly-client-util/src/request/mod.rs +++ b/dragonfly-client-util/src/request/mod.rs @@ -145,19 +145,39 @@ pub struct GetRequest { /// when downloading from OCI registries. When enabled for OCI blob URLs (e.g., /v2//blobs/sha256:), /// the task ID is derived from the blob digest rather than the full URL. This enables deduplication across /// registries - the same blob from different registries shares one task ID, eliminating redundant downloads - /// and storage. + /// and storage, default is true. pub enable_task_id_based_blob_digest: bool, /// Refer to https://github.com/dragonflyoss/api/blob/main/proto/common.proto#L67 pub priority: Option, - /// timeout is the timeout of the request. + /// timeout is the timeout of the request, default is 300s. pub timeout: Duration, /// Client cert is the client certificates for the request. pub client_cert: Option>>, } +/// Default implementation for GetRequest. +impl Default for GetRequest { + /// Default returns a default GetRequest with empty url and default values for other fields. + fn default() -> Self { + Self { + url: String::new(), + header: None, + piece_length: None, + tag: None, + application: None, + filtered_query_params: default_proxy_rule_filtered_query_params(), + content_for_calculating_task_id: None, + enable_task_id_based_blob_digest: true, + priority: None, + timeout: Duration::from_secs(300), + client_cert: None, + } + } +} + /// GetResponse represents a GET response received via the Dragonfly. pub struct GetResponse where @@ -220,19 +240,42 @@ pub struct PreheatRequest { /// when downloading from OCI registries. When enabled for OCI blob URLs (e.g., /v2//blobs/sha256:), /// the task ID is derived from the blob digest rather than the full URL. This enables deduplication across /// registries - the same blob from different registries shares one task ID, eliminating redundant downloads - /// and storage. + /// and storage, default is true. pub enable_task_id_based_blob_digest: bool, /// Refer to https://github.com/dragonflyoss/api/blob/main/proto/common.proto#L67 pub priority: Option, - /// Timeout is the timeout for each blob download request. + /// Timeout is the timeout for each blob download request, default is 300s. pub timeout: Duration, /// Client cert is the optional client certificates for the request. pub client_cert: Option>>, } +/// Default implementation for PreheatRequest. +impl Default for PreheatRequest { + /// Default returns a default PreheatRequest with empty image and default values for other + /// fields. + fn default() -> Self { + Self { + image: String::new(), + username: None, + password: None, + platform: None, + piece_length: None, + tag: None, + application: None, + filtered_query_params: default_proxy_rule_filtered_query_params(), + content_for_calculating_task_id: None, + enable_task_id_based_blob_digest: true, + priority: None, + timeout: Duration::from_secs(300), + client_cert: None, + } + } +} + /// Factory for creating HTTPClient instances. #[derive(Debug, Clone, Default)] struct HTTPClientFactory {} @@ -440,7 +483,7 @@ impl Request for Proxy { /// stream, allowing efficient handling of large or continuous data. The response includes metadata /// such as status codes and headers, along with a streaming `Body` for accessing the response content. async fn get(&self, request: &GetRequest) -> Result { - let response = self.try_send(&request).await?; + let response = self.try_send(request).await?; let header = response.headers().clone(); let status_code = response.status(); let reader = Box::new(StreamReader::new( @@ -466,7 +509,7 @@ impl Request for Proxy { /// and headers) is returned separately. async fn get_into(&self, request: &GetRequest, buf: &mut BytesMut) -> Result { let get_into = async { - let response = self.try_send(&request).await?; + let response = self.try_send(request).await?; let status = response.status(); let headers = response.headers().clone(); @@ -568,7 +611,7 @@ impl Request for Proxy { client_cert: request.client_cert.clone(), }; - let response = self.get(get_request).await?; + let response = self.get(&get_request).await?; match response.reader { Some(mut reader) => { tokio::io::copy(&mut reader, &mut tokio::io::sink()) @@ -600,12 +643,6 @@ impl Proxy { &self, request: &GetRequest, ) -> Result>> { - let filtered_query_params = if request.filtered_query_params.is_empty() { - default_proxy_rule_filtered_query_params() - } else { - request.filtered_query_params.clone() - }; - // Generate task id for selecting seed peer. let task_id = self .id_generator @@ -620,7 +657,7 @@ impl Proxy { piece_length: request.piece_length, tag: request.tag.clone(), application: request.application.clone(), - filtered_query_params, + filtered_query_params: request.filtered_query_params.clone(), revision: None, } }, From 8b9c34f1e9d5ffc25b25af159f86cf9e46bd1e21 Mon Sep 17 00:00:00 2001 From: Gaius Date: Fri, 10 Apr 2026 15:55:13 +0800 Subject: [PATCH 12/12] chore: bump workspace version from 1.3.2 to 1.3.3 Signed-off-by: Gaius --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 18 +++++++++--------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f10d425b..9727e6e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1202,7 +1202,7 @@ dependencies = [ [[package]] name = "dragonfly-client" -version = "1.3.2" +version = "1.3.3" dependencies = [ "anyhow", "async-trait", @@ -1284,7 +1284,7 @@ dependencies = [ [[package]] name = "dragonfly-client-backend" -version = "1.3.2" +version = "1.3.3" dependencies = [ "async-trait", "dashmap", @@ -1323,7 +1323,7 @@ dependencies = [ [[package]] name = "dragonfly-client-config" -version = "1.3.2" +version = "1.3.3" dependencies = [ "bytesize", "bytesize-serde", @@ -1354,7 +1354,7 @@ dependencies = [ [[package]] name = "dragonfly-client-core" -version = "1.3.2" +version = "1.3.3" dependencies = [ "cgroups-rs", "headers 0.4.1", @@ -1375,7 +1375,7 @@ dependencies = [ [[package]] name = "dragonfly-client-init" -version = "1.3.2" +version = "1.3.3" dependencies = [ "anyhow", "clap", @@ -1393,7 +1393,7 @@ dependencies = [ [[package]] name = "dragonfly-client-metric" -version = "1.3.2" +version = "1.3.3" dependencies = [ "dragonfly-api", "dragonfly-client-config", @@ -1408,7 +1408,7 @@ dependencies = [ [[package]] name = "dragonfly-client-storage" -version = "1.3.2" +version = "1.3.3" dependencies = [ "bincode", "bytes", @@ -1442,7 +1442,7 @@ dependencies = [ [[package]] name = "dragonfly-client-util" -version = "1.3.2" +version = "1.3.3" dependencies = [ "async-trait", "base64 0.22.1", @@ -2000,7 +2000,7 @@ dependencies = [ [[package]] name = "hdfs" -version = "1.3.2" +version = "1.3.3" dependencies = [ "async-trait", "dragonfly-client-backend", @@ -4627,7 +4627,7 @@ dependencies = [ [[package]] name = "request" -version = "1.3.2" +version = "1.3.3" dependencies = [ "anyhow", "dragonfly-client-util", diff --git a/Cargo.toml b/Cargo.toml index 7beefc3a..d077456e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ ] [workspace.package] -version = "1.3.2" +version = "1.3.3" authors = ["The Dragonfly Developers"] homepage = "https://d7y.io/" repository = "https://github.com/dragonflyoss/client.git" @@ -24,14 +24,14 @@ readme = "README.md" edition = "2021" [workspace.dependencies] -dragonfly-client = { path = "dragonfly-client", version = "1.3.2" } -dragonfly-client-core = { path = "dragonfly-client-core", version = "1.3.2" } -dragonfly-client-config = { path = "dragonfly-client-config", version = "1.3.2" } -dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.3.2" } -dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.3.2" } -dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.3.2" } -dragonfly-client-util = { path = "dragonfly-client-util", version = "1.3.2" } -dragonfly-client-init = { path = "dragonfly-client-init", version = "1.3.2" } +dragonfly-client = { path = "dragonfly-client", version = "1.3.3" } +dragonfly-client-core = { path = "dragonfly-client-core", version = "1.3.3" } +dragonfly-client-config = { path = "dragonfly-client-config", version = "1.3.3" } +dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.3.3" } +dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.3.3" } +dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.3.3" } +dragonfly-client-util = { path = "dragonfly-client-util", version = "1.3.3" } +dragonfly-client-init = { path = "dragonfly-client-init", version = "1.3.3" } dragonfly-api = "=2.2.28" thiserror = "2.0" futures = "0.3.32"