diff --git a/Cargo.lock b/Cargo.lock index d1eabed4..9727e6e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,9 +574,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.41" +version = "1.2.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7" +checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" dependencies = [ "find-msvc-tools", "jobserver", @@ -831,6 +831,26 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "const_format" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -985,6 +1005,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -1066,6 +1121,37 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", +] + [[package]] name = "digest" version = "0.10.7" @@ -1116,7 +1202,7 @@ dependencies = [ [[package]] name = "dragonfly-client" -version = "1.3.2" +version = "1.3.3" dependencies = [ "anyhow", "async-trait", @@ -1164,7 +1250,7 @@ dependencies = [ "rand 0.9.2", "rcgen", "regex", - "reqwest", + "reqwest 0.12.28", "rolling-file", "rustls", "rustls-pki-types", @@ -1198,7 +1284,7 @@ dependencies = [ [[package]] name = "dragonfly-client-backend" -version = "1.3.2" +version = "1.3.3" dependencies = [ "async-trait", "dashmap", @@ -1216,7 +1302,7 @@ dependencies = [ "opendal", "percent-encoding", "rcgen", - "reqwest", + "reqwest 0.12.28", "reqwest-middleware", "reqwest-retry", "reqwest-tracing", @@ -1237,7 +1323,7 @@ dependencies = [ [[package]] name = "dragonfly-client-config" -version = "1.3.2" +version = "1.3.3" dependencies = [ "bytesize", "bytesize-serde", @@ -1253,7 +1339,7 @@ dependencies = [ "local-ip-address", "rcgen", "regex", - "reqwest", + "reqwest 0.12.28", "rustls-pki-types", "serde", "serde_json", @@ -1268,7 +1354,7 @@ dependencies = [ [[package]] name = "dragonfly-client-core" -version = "1.3.2" +version = "1.3.3" dependencies = [ "cgroups-rs", "headers 0.4.1", @@ -1276,7 +1362,7 @@ dependencies = [ "hyper 1.9.0", "hyper-util", "opendal", - "reqwest", + "reqwest 0.12.28", "reqwest-middleware", "thiserror 2.0.18", "tokio", @@ -1289,7 +1375,7 @@ dependencies = [ [[package]] name = "dragonfly-client-init" -version = "1.3.2" +version = "1.3.3" dependencies = [ "anyhow", "clap", @@ -1307,7 +1393,7 @@ dependencies = [ [[package]] name = "dragonfly-client-metric" -version = "1.3.2" +version = "1.3.3" dependencies = [ "dragonfly-api", "dragonfly-client-config", @@ -1322,7 +1408,7 @@ dependencies = [ [[package]] name = "dragonfly-client-storage" -version = "1.3.2" +version = "1.3.3" dependencies = [ "bincode", "bytes", @@ -1340,7 +1426,7 @@ dependencies = [ "num_cpus", "prost-wkt-types", "quinn", - "reqwest", + "reqwest 0.12.28", "rocksdb", "rustls", "rustls-pki-types", @@ -1356,7 +1442,7 @@ dependencies = [ [[package]] name = "dragonfly-client-util" -version = "1.3.2" +version = "1.3.3" dependencies = [ "async-trait", "base64 0.22.1", @@ -1381,13 +1467,15 @@ dependencies = [ "lru", "mocktail", "num_cpus", + "oci-client", + "oci-spec", "openssl", "parking_lot", "pnet", "protobuf 3.7.2", "rcgen", "regex", - "reqwest", + "reqwest 0.12.28", "reqwest-middleware", "reqwest-tracing", "ringbuf", @@ -1555,9 +1643,9 @@ checksum = "a043dc74da1e37d6afe657061213aa6f425f855399a11d3463c6ecccc4dfda1f" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "findshlibs" @@ -1791,6 +1879,18 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "getset" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "gimli" version = "0.28.1" @@ -1900,7 +2000,7 @@ dependencies = [ [[package]] name = "hdfs" -version = "1.3.2" +version = "1.3.3" dependencies = [ "async-trait", "dragonfly-client-backend", @@ -2089,6 +2189,15 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-auth" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" +dependencies = [ + "memchr", +] + [[package]] name = "http-body" version = "0.4.6" @@ -2426,6 +2535,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.4.0" @@ -2683,10 +2798,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.81" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -2706,6 +2823,20 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "jsonwebtoken" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" +dependencies = [ + "base64 0.22.1", + "getrandom 0.2.12", + "js-sys", + "serde", + "serde_json", + "signature", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -3261,6 +3392,49 @@ dependencies = [ "memchr", ] +[[package]] +name = "oci-client" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b7f8deaffcd3b0e3baf93dddcab3d18b91d46dc37d38a8b170089b234de5bb3" +dependencies = [ + "bytes", + "chrono", + "futures-util", + "http 1.4.0", + "http-auth", + "jsonwebtoken 10.3.0", + "lazy_static", + "oci-spec", + "olpc-cjson", + "regex", + "reqwest 0.13.2", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.18", + "tokio", + "tracing", + "unicase", +] + +[[package]] +name = "oci-spec" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8445a2631507cec628a15fdd6154b54a3ab3f20ed4fe9d73a3b8b7a4e1ba03a" +dependencies = [ + "const_format", + "derive_builder", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror 2.0.18", +] + [[package]] name = "oid-registry" version = "0.6.1" @@ -3270,6 +3444,17 @@ dependencies = [ "asn1-rs", ] +[[package]] +name = "olpc-cjson" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083" +dependencies = [ + "serde", + "serde_json", + "unicode-normalization", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -3313,7 +3498,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.38.3", "reqsign", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "sha2", @@ -3399,7 +3584,7 @@ dependencies = [ "bytes", "http 1.4.0", "opentelemetry", - "reqwest", + "reqwest 0.12.28", ] [[package]] @@ -3414,7 +3599,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost 0.14.3", - "reqwest", + "reqwest 0.12.28", "thiserror 2.0.18", "tokio", "tonic", @@ -4424,13 +4609,13 @@ dependencies = [ "hmac", "home", "http 1.4.0", - "jsonwebtoken", + "jsonwebtoken 9.3.0", "log", "once_cell", "percent-encoding", "quick-xml 0.37.5", "rand 0.8.5", - "reqwest", + "reqwest 0.12.28", "rsa", "rust-ini", "serde", @@ -4440,6 +4625,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "request" +version = "1.3.3" +dependencies = [ + "anyhow", + "dragonfly-client-util", + "tokio", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -4485,11 +4679,50 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.0", "web-sys", "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.9.0", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + [[package]] name = "reqwest-middleware" version = "0.4.2" @@ -4499,7 +4732,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.4.0", - "reqwest", + "reqwest 0.12.28", "serde", "thiserror 1.0.69", "tower-service", @@ -4517,7 +4750,7 @@ dependencies = [ "getrandom 0.2.12", "http 1.4.0", "hyper 1.9.0", - "reqwest", + "reqwest 0.12.28", "reqwest-middleware", "retry-policies", "thiserror 2.0.18", @@ -4537,7 +4770,7 @@ dependencies = [ "getrandom 0.2.12", "http 1.4.0", "matchit", - "reqwest", + "reqwest 0.12.28", "reqwest-middleware", "tracing", ] @@ -5163,6 +5396,24 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "subtle" version = "2.5.0" @@ -5967,12 +6218,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.7.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" [[package]] name = "unicode-bidi" @@ -6218,9 +6466,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.104" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" dependencies = [ "cfg-if", "once_cell", @@ -6229,37 +6477,21 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.117", - "wasm-bindgen-shared", -] - [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" dependencies = [ - "cfg-if", "js-sys", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.104" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6267,22 +6499,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.104" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" dependencies = [ + "bumpalo", "proc-macro2", "quote", "syn 2.0.117", - "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.104" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" dependencies = [ "unicode-ident", ] @@ -6300,6 +6532,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmtimer" version = "0.4.3" @@ -6316,9 +6561,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 2777ab33..d077456e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,11 +9,12 @@ members = [ "dragonfly-client-storage", "dragonfly-client-util", "dragonfly-client-backend/examples/plugin", + "dragonfly-client-util/examples/request", "dragonfly-client-metric", ] [workspace.package] -version = "1.3.2" +version = "1.3.3" authors = ["The Dragonfly Developers"] homepage = "https://d7y.io/" repository = "https://github.com/dragonflyoss/client.git" @@ -23,14 +24,14 @@ readme = "README.md" edition = "2021" [workspace.dependencies] -dragonfly-client = { path = "dragonfly-client", version = "1.3.2" } -dragonfly-client-core = { path = "dragonfly-client-core", version = "1.3.2" } -dragonfly-client-config = { path = "dragonfly-client-config", version = "1.3.2" } -dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.3.2" } -dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.3.2" } -dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.3.2" } -dragonfly-client-util = { path = "dragonfly-client-util", version = "1.3.2" } -dragonfly-client-init = { path = "dragonfly-client-init", version = "1.3.2" } +dragonfly-client = { path = "dragonfly-client", version = "1.3.3" } +dragonfly-client-core = { path = "dragonfly-client-core", version = "1.3.3" } +dragonfly-client-config = { path = "dragonfly-client-config", version = "1.3.3" } +dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.3.3" } +dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.3.3" } +dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.3.3" } +dragonfly-client-util = { path = "dragonfly-client-util", version = "1.3.3" } +dragonfly-client-init = { path = "dragonfly-client-init", version = "1.3.3" } dragonfly-api = "=2.2.28" thiserror = "2.0" futures = "0.3.32" @@ -120,6 +121,8 @@ cgroups-rs = "0.5" num_cpus = "1.17" async-trait = "0.1" humantime-serde = "1.1.1" +oci-client = { version = "0.16.1", default-features = false, features = ["native-tls"] } +oci-spec = "0.9" # TODO(Gaius): Remove the git dependency after the next release of mocktail, refer to https://github.com/IBM/mocktail/issues/65. mocktail = { version = "0.3.0", git = "https://github.com/IBM/mocktail", rev = "860e75e171a8eb818083813dbeed4401d1a40b3b" } diff --git a/dragonfly-client-backend/examples/plugin/src/lib.rs b/dragonfly-client-backend/examples/plugin/src/lib.rs index ef1ee97b..7ee4c672 100644 --- a/dragonfly-client-backend/examples/plugin/src/lib.rs +++ b/dragonfly-client-backend/examples/plugin/src/lib.rs @@ -64,7 +64,7 @@ impl Backend for Hdfs { } } -/// register_plugin is a function that returns a Box. +/// Register plugin is a function that returns a Box. /// This function is used to register the HDFS plugin to the Backend. #[no_mangle] pub fn register_plugin() -> Box { diff --git a/dragonfly-client-util/Cargo.toml b/dragonfly-client-util/Cargo.toml index ddfb47d4..d5edd098 100644 --- a/dragonfly-client-util/Cargo.toml +++ b/dragonfly-client-util/Cargo.toml @@ -49,6 +49,8 @@ reqwest-tracing.workspace = true reqwest-middleware.workspace = true num_cpus.workspace = true humantime-serde.workspace = true +oci-client.workspace = true +oci-spec.workspace = true rustix = { version = "1.1.3", features = ["fs"] } base64 = "0.22.1" pnet = "0.35.0" diff --git a/dragonfly-client-util/examples/request/Cargo.toml b/dragonfly-client-util/examples/request/Cargo.toml new file mode 100644 index 00000000..c87d78d2 --- /dev/null +++ b/dragonfly-client-util/examples/request/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "request" +description = "An example of request library for the Dragonfly client" +version.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +keywords.workspace = true +license.workspace = true +edition.workspace = true +publish = false + +[[bin]] +name = "get" +path = "src/bin/get.rs" + +[[bin]] +name = "preheat" +path = "src/bin/preheat.rs" + +[dependencies] +dragonfly-client-util.workspace = true +tokio.workspace = true +anyhow.workspace = true diff --git a/dragonfly-client-util/examples/request/README.md b/dragonfly-client-util/examples/request/README.md new file mode 100644 index 00000000..bdda5a61 --- /dev/null +++ b/dragonfly-client-util/examples/request/README.md @@ -0,0 +1,23 @@ +# Examples of Request + +An example of using the `request` module to download a file and preheat an OCI image via the Dragonfly P2P network. + +## Run GET Example + +Downloads a file via the Dragonfly P2P network using the `get` method. + +```shell +export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" +cargo run -p request --bin get +``` + +## Run Preheat Example + +Preheats an OCI image (`dragonflyoss/scheduler:v2.4.3`) via the Dragonfly P2P network +using the `preheat` method. All blobs (config and layers) are downloaded through the +Dragonfly seed peer proxy and cached in the P2P network. + +```shell +export DRAGONFLY_SCHEDULER_ENDPOINT="http://127.0.0.1:8002" +cargo run -p request --bin preheat +``` diff --git a/dragonfly-client-util/examples/request/src/bin/get.rs b/dragonfly-client-util/examples/request/src/bin/get.rs new file mode 100644 index 00000000..fb0db5b1 --- /dev/null +++ b/dragonfly-client-util/examples/request/src/bin/get.rs @@ -0,0 +1,54 @@ +/* + * Copyright 2026 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use dragonfly_client_util::request::{GetRequest, Proxy, Request}; +use std::time::Duration; +use tokio::io::AsyncReadExt; + +/// This example demonstrates how to use the get method of the Dragonfly request module +/// to download a file via the Dragonfly P2P network. +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let scheduler_endpoint = std::env::var("DRAGONFLY_SCHEDULER_ENDPOINT") + .unwrap_or_else(|_| "http://127.0.0.1:8002".to_string()); + + let proxy = Proxy::builder() + .scheduler_endpoint(scheduler_endpoint) + .scheduler_request_timeout(Duration::from_secs(5)) + .health_check_interval(Duration::from_secs(60)) + .max_retries(3) + .build() + .await + .map_err(|err| anyhow::anyhow!("failed to build proxy: {}", err))?; + + let request = GetRequest { + url: "https://example.com/path/to/file".to_string(), + ..Default::default() + }; + + let response = proxy + .get(&request) + .await + .map_err(|err| anyhow::anyhow!("get request failed: {}", err))?; + + if let Some(mut reader) = response.reader { + let mut body = Vec::new(); + reader.read_to_end(&mut body).await?; + println!("{} downloaded, size: {} bytes", &request.url, body.len()); + } + + Ok(()) +} diff --git a/dragonfly-client-util/examples/request/src/bin/preheat.rs b/dragonfly-client-util/examples/request/src/bin/preheat.rs new file mode 100644 index 00000000..a1c8132e --- /dev/null +++ b/dragonfly-client-util/examples/request/src/bin/preheat.rs @@ -0,0 +1,49 @@ +/* + * Copyright 2026 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use dragonfly_client_util::request::{PreheatRequest, Proxy, Request}; +use std::time::Duration; + +/// This example demonstrates how to use the preheat method of the Dragonfly request module +/// to pre-cache an OCI image via the Dragonfly P2P network. +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let scheduler_endpoint = std::env::var("DRAGONFLY_SCHEDULER_ENDPOINT") + .unwrap_or_else(|_| "http://127.0.0.1:8002".to_string()); + + let proxy = Proxy::builder() + .scheduler_endpoint(scheduler_endpoint) + .scheduler_request_timeout(Duration::from_secs(5)) + .health_check_interval(Duration::from_secs(60)) + .max_retries(3) + .build() + .await + .map_err(|err| anyhow::anyhow!("failed to build proxy: {}", err))?; + + let request = PreheatRequest { + image: "docker.io/dragonflyoss/scheduler:v2.4.3".to_string(), + platform: Some("linux/amd64".to_string()), + ..Default::default() + }; + + proxy + .preheat(&request) + .await + .map_err(|err| anyhow::anyhow!("preheat failed: {}", err))?; + + println!("{} has been preheated", request.image); + Ok(()) +} diff --git a/dragonfly-client-util/src/request/mod.rs b/dragonfly-client-util/src/request/mod.rs index 1fe57435..7bddc5d4 100644 --- a/dragonfly-client-util/src/request/mod.rs +++ b/dragonfly-client-util/src/request/mod.rs @@ -14,9 +14,6 @@ * limitations under the License. */ -pub mod errors; -mod selector; - use crate::digest::is_blob_url; use crate::http::{headermap_to_hashmap, query_params::default_proxy_rule_filtered_query_params}; use crate::id_generator::{IDGenerator, TaskIDParameter}; @@ -29,7 +26,15 @@ use dragonfly_api::scheduler::v2::scheduler_client::SchedulerClient; use errors::{BackendError, DfdaemonError, Error, ProxyError}; use futures::TryStreamExt; use hostname; -use reqwest::{header::HeaderMap, header::HeaderValue, Client}; +use oci_client::client::ClientConfig; +use oci_client::manifest::ImageIndexEntry; +use oci_client::secrets::RegistryAuth; +use oci_client::{Client as OciClient, Reference, RegistryOperation}; +use oci_spec::image::{Arch, Os}; +use reqwest::{ + header::{HeaderMap, HeaderValue, AUTHORIZATION}, + Client, +}; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_tracing::TracingMiddleware; use rustix::path::Arg; @@ -43,7 +48,10 @@ use std::time::Duration; use tokio::io::AsyncRead; use tokio_util::io::StreamReader; use tonic::transport::Endpoint; -use tracing::{debug, error}; +use tracing::{debug, error, warn}; + +pub mod errors; +mod selector; /// POOL_MAX_IDLE_PER_HOST is the max idle connections per host. const POOL_MAX_IDLE_PER_HOST: usize = 1024; @@ -83,7 +91,7 @@ pub trait Request { /// This method is designed for scenarios where the response body is expected to be processed as a /// stream, allowing efficient handling of large or continuous data. The response includes metadata /// such as status codes and headers, along with a streaming `Body` for accessing the response content. - async fn get(&self, request: GetRequest) -> Result>; + async fn get(&self, request: &GetRequest) -> Result>; /// Sends an GET request to a remote server via the Dragonfly and writes the response /// body directly into the provided buffer. @@ -92,7 +100,16 @@ pub trait Request { /// memory, avoiding the overhead of streaming for smaller or fixed-size responses. The provided /// `BytesMut` buffer is used to store the response content, and the response metadata (e.g., status /// and headers) is returned separately. - async fn get_into(&self, request: GetRequest, buf: &mut BytesMut) -> Result; + async fn get_into(&self, request: &GetRequest, buf: &mut BytesMut) -> Result; + + /// Preheats an OCI image by downloading all its blobs via the Dragonfly. + /// + /// This method is designed for scenarios where OCI image content needs to be pre-cached in + /// the seed client before actual consumption, ensuring faster subsequent access across the + /// cluster. It parses the image reference, authenticates with the OCI registry, resolves + /// the image manifest (including multi-platform image indexes), and downloads each blob + /// (config and layers) through the seed client. + async fn preheat(&self, request: &PreheatRequest) -> Result<()>; } /// GetRequest represents a GET request to be sent via the Dragonfly. @@ -128,19 +145,39 @@ pub struct GetRequest { /// when downloading from OCI registries. When enabled for OCI blob URLs (e.g., /v2//blobs/sha256:), /// the task ID is derived from the blob digest rather than the full URL. This enables deduplication across /// registries - the same blob from different registries shares one task ID, eliminating redundant downloads - /// and storage. + /// and storage, default is true. pub enable_task_id_based_blob_digest: bool, /// Refer to https://github.com/dragonflyoss/api/blob/main/proto/common.proto#L67 pub priority: Option, - /// timeout is the timeout of the request. + /// timeout is the timeout of the request, default is 300s. pub timeout: Duration, /// Client cert is the client certificates for the request. pub client_cert: Option>>, } +/// Default implementation for GetRequest. +impl Default for GetRequest { + /// Default returns a default GetRequest with empty url and default values for other fields. + fn default() -> Self { + Self { + url: String::new(), + header: None, + piece_length: None, + tag: None, + application: None, + filtered_query_params: default_proxy_rule_filtered_query_params(), + content_for_calculating_task_id: None, + enable_task_id_based_blob_digest: true, + priority: None, + timeout: Duration::from_secs(300), + client_cert: None, + } + } +} + /// GetResponse represents a GET response received via the Dragonfly. pub struct GetResponse where @@ -159,6 +196,86 @@ where pub reader: Option, } +/// PreheatRequest represents a request to preheat an OCI image through the +/// Dragonfly seed client. The preheat downloads all blobs (config and layers) +/// of the specified image via the Dragonfly proxy, effectively caching them +/// in the P2P network for faster downloading. +pub struct PreheatRequest { + /// Image is the OCI image reference (e.g., "docker.io/library/nginx:latest"). + pub image: String, + + /// Username for registry authentication. If not provided, anonymous access is used. + pub username: Option, + + /// Password for registry authentication. If not provided, anonymous access is used. + pub password: Option, + + /// Platform specifies the target platform in the format "os/arch" + /// (e.g., "linux/amd64", "linux/arm64"). This is used to select the correct + /// manifest from a multi-platform image index, default is current platform. + pub platform: Option, + + /// Piece length is the optional piece length for the Dragonfly task. + pub piece_length: Option, + + /// Tag identifies different tasks for the same URL. + pub tag: Option, + + /// Application identifies different tasks for the same URL. + pub application: Option, + + /// Filtered query params to generate the task id. + /// When filter is ["Signature", "Expires", "ns"], for example: + /// http://example.com/xyz?Expires=e1&Signature=s1&ns=docker.io and http://example.com/xyz?Expires=e2&Signature=s2&ns=docker.io + /// will generate the same task id. + /// Default value includes the filtered query params of s3, gcs, oss, obs, cos. + pub filtered_query_params: Vec, + + /// Content for calculating task id. This is used when the task ID cannot be calculated based + /// on URL and other parameters, such as when the URL contains dynamic query parameters that + /// cannot be filtered out. + pub content_for_calculating_task_id: Option, + + /// Enable task id based blob digest. It indicates whether to use the blob digest for task ID calculation + /// when downloading from OCI registries. When enabled for OCI blob URLs (e.g., /v2//blobs/sha256:), + /// the task ID is derived from the blob digest rather than the full URL. This enables deduplication across + /// registries - the same blob from different registries shares one task ID, eliminating redundant downloads + /// and storage, default is true. + pub enable_task_id_based_blob_digest: bool, + + /// Refer to https://github.com/dragonflyoss/api/blob/main/proto/common.proto#L67 + pub priority: Option, + + /// Timeout is the timeout for each blob download request, default is 300s. + pub timeout: Duration, + + /// Client cert is the optional client certificates for the request. + pub client_cert: Option>>, +} + +/// Default implementation for PreheatRequest. +impl Default for PreheatRequest { + /// Default returns a default PreheatRequest with empty image and default values for other + /// fields. + fn default() -> Self { + Self { + image: String::new(), + username: None, + password: None, + platform: None, + piece_length: None, + tag: None, + application: None, + filtered_query_params: default_proxy_rule_filtered_query_params(), + content_for_calculating_task_id: None, + enable_task_id_based_blob_digest: true, + priority: None, + timeout: Duration::from_secs(300), + client_cert: None, + } + } +} + /// Factory for creating HTTPClient instances. #[derive(Debug, Clone, Default)] struct HTTPClientFactory {} @@ -365,8 +482,8 @@ impl Request for Proxy { /// This method is designed for scenarios where the response body is expected to be processed as a /// stream, allowing efficient handling of large or continuous data. The response includes metadata /// such as status codes and headers, along with a streaming `Body` for accessing the response content. - async fn get(&self, request: GetRequest) -> Result { - let response = self.try_send(&request).await?; + async fn get(&self, request: &GetRequest) -> Result { + let response = self.try_send(request).await?; let header = response.headers().clone(); let status_code = response.status(); let reader = Box::new(StreamReader::new( @@ -390,9 +507,9 @@ impl Request for Proxy { /// memory, avoiding the overhead of streaming for smaller or fixed-size responses. The provided /// `BytesMut` buffer is used to store the response content, and the response metadata (e.g., status /// and headers) is returned separately. - async fn get_into(&self, request: GetRequest, buf: &mut BytesMut) -> Result { + async fn get_into(&self, request: &GetRequest, buf: &mut BytesMut) -> Result { let get_into = async { - let response = self.try_send(&request).await?; + let response = self.try_send(request).await?; let status = response.status(); let headers = response.headers().clone(); @@ -417,6 +534,106 @@ impl Request for Proxy { .await .map_err(|err| Error::RequestTimeout(err.to_string()))? } + + /// Preheats an OCI image by downloading all its blobs via the Dragonfly. + /// + /// This method is designed for scenarios where OCI image content needs to be pre-cached in + /// the seed client before actual consumption, ensuring faster subsequent access across the + /// cluster. It parses the image reference, authenticates with the OCI registry, resolves + /// the image manifest (including multi-platform image indexes), and downloads each blob + /// (config and layers) through the seed client. + async fn preheat(&self, request: &PreheatRequest) -> Result<()> { + let oci_client = Self::oci_client(request.platform.clone())?; + + // Parse image reference. + let reference: Reference = request + .image + .parse() + .map_err(|err| Error::InvalidArgument(format!("invalid image reference: {}", err)))?; + + // Create registry authentication. + let auth = match (&request.username, &request.password) { + (Some(username), Some(password)) => { + RegistryAuth::Basic(username.clone(), password.clone()) + } + _ => RegistryAuth::Anonymous, + }; + + // Pull image manifest. This handles multi-platform image index manifests + // by selecting the platform-specific manifest using our resolver. + let (manifest, digest) = oci_client + .pull_image_manifest(&reference, &auth) + .await + .map_err(|err| Error::Internal(format!("failed to pull image manifest: {}", err)))?; + debug!( + "pulled manifest for image {} with digest {}, layers: {}", + request.image, + digest, + manifest.layers.len() + ); + + // Authenticate with the registry and get a bearer token if available. + let token = oci_client + .auth(&reference, &auth, RegistryOperation::Pull) + .await + .map_err(|err| { + Error::Internal(format!("failed to authenticate with registry: {}", err)) + })? + .ok_or_else(|| { + Error::Internal("registry did not return authentication token".to_string()) + })?; + + // Build authorization header for blob downloads through the Dragonfly. + let mut header = HeaderMap::new(); + header.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", token)) + .map_err(|err| Error::Internal(format!("invalid auth token: {}", err)))?, + ); + + let registry = reference.resolve_registry(); + let repository = reference.repository(); + for digest in std::iter::once(&manifest.config.digest) + .chain(manifest.layers.iter().map(|layer| &layer.digest)) + { + let url = Self::build_blob_url(registry, repository, digest); + let get_request = GetRequest { + url: url.clone(), + header: Some(header.clone()), + piece_length: request.piece_length, + tag: request.tag.clone(), + application: request.application.clone(), + filtered_query_params: request.filtered_query_params.clone(), + content_for_calculating_task_id: request.content_for_calculating_task_id.clone(), + enable_task_id_based_blob_digest: request.enable_task_id_based_blob_digest, + priority: request.priority, + timeout: request.timeout, + client_cert: request.client_cert.clone(), + }; + + let response = self.get(&get_request).await?; + match response.reader { + Some(mut reader) => { + tokio::io::copy(&mut reader, &mut tokio::io::sink()) + .await + .map_err(|err| { + Error::Internal(format!("failed to read blob {}: {}", digest, err)) + })?; + } + None => { + warn!( + "no response body for blob {}, download may not have completed", + digest + ); + } + } + + debug!("preheated blob: {}", url); + } + + debug!("preheat completed for image: {}", request.image); + Ok(()) + } } /// Proxy implements proxy request logic. @@ -426,12 +643,6 @@ impl Proxy { &self, request: &GetRequest, ) -> Result>> { - let filtered_query_params = if request.filtered_query_params.is_empty() { - default_proxy_rule_filtered_query_params() - } else { - request.filtered_query_params.clone() - }; - // Generate task id for selecting seed peer. let task_id = self .id_generator @@ -446,7 +657,7 @@ impl Proxy { piece_length: request.piece_length, tag: request.tag.clone(), application: request.application.clone(), - filtered_query_params, + filtered_query_params: request.filtered_query_params.clone(), revision: None, } }, @@ -567,7 +778,7 @@ impl Proxy { } } - /// make_request_headers applies p2p related headers to the request headers. + /// Make request headers applies p2p related headers to the request headers. fn make_request_headers(&self, request: &GetRequest) -> Result { let mut headers = request.header.clone().unwrap_or_default(); @@ -652,6 +863,41 @@ impl Proxy { headers.insert("X-Dragonfly-Use-P2P", HeaderValue::from_static("true")); Ok(headers) } + + /// Helper function to check if a URL is an OCI blob URL (e.g., /v2//blobs/sha256: + /// ). + fn build_blob_url(registry: &str, repository: &str, digest: &str) -> String { + format!("https://{}/v2/{}/blobs/{}", registry, repository, digest) + } + + /// Builds an OCI client with a platform resolver that matches the requested os/arch. + fn oci_client(platform: Option) -> Result { + let mut oci_config = ClientConfig::default(); + if let Some(platform) = platform { + let (os, arch) = platform + .split_once('/') + .map(|(os, arch)| (Os::from(os), Arch::from(arch))) + .ok_or_else(|| { + Error::InvalidArgument(format!( + "invalid platform format '{}', expected 'os/arch' (e.g., 'linux/amd64')", + platform + )) + })?; + + oci_config.platform_resolver = Some(Box::new(move |manifests: &[ImageIndexEntry]| { + manifests + .iter() + .find(|entry| { + entry.platform.as_ref().is_some_and(|platform| { + platform.os == os && platform.architecture == arch + }) + }) + .map(|entry| entry.digest.clone()) + })) + }; + + Ok(OciClient::new(oci_config)) + } } #[cfg(test)]