From b0ac9d58af39a69b081ff549f7df20c1daf46f0f Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Tue, 21 Oct 2025 16:15:06 -0500 Subject: [PATCH] Import minireq as bitreq A while back I forked `minreq` and re-named it to `minireq`. I emailed the original author and got his blessing. Since then the name has been taken on crates.io Import the crate here and rename it to `bitreq`. We grabbed the name already on crates.io --- Cargo-minimal.lock | 471 +++++++++++++++++- Cargo-recent.lock | 477 +++++++++++++++++- bitreq/.github/workflows/lint.yml | 29 ++ bitreq/.github/workflows/msrv.yml | 24 + bitreq/.github/workflows/unit-tests.yml | 63 +++ bitreq/.gitignore | 5 + bitreq/CHANGELOG.md | 382 ++++++++++++++- bitreq/COPYING.md | 16 + bitreq/Cargo.toml | 58 ++- bitreq/README.md | 84 +++- bitreq/examples/async_hello.rs | 11 + bitreq/examples/hello.rs | 10 + bitreq/examples/iterator.rs | 48 ++ bitreq/examples/no-std.rs | 71 +++ bitreq/examples/wasm_example.rs | 25 + bitreq/src/connection.rs | 381 +++++++++++++++ bitreq/src/connection/rustls_stream.rs | 75 +++ bitreq/src/error.rs | 140 ++++++ bitreq/src/http_url.rs | 202 ++++++++ bitreq/src/lib.rs | 247 ++++++++++ bitreq/src/proxy.rs | 162 +++++++ bitreq/src/request.rs | 618 ++++++++++++++++++++++++ bitreq/src/response.rs | 574 ++++++++++++++++++++++ bitreq/src/wasm.rs | 228 +++++++++ bitreq/tests/main.rs | 209 ++++++++ bitreq/tests/setup.rs | 201 ++++++++ 26 files changed, 4803 insertions(+), 8 deletions(-) create mode 100644 bitreq/.github/workflows/lint.yml create mode 100644 bitreq/.github/workflows/msrv.yml create mode 100644 bitreq/.github/workflows/unit-tests.yml create mode 100644 bitreq/.gitignore create mode 100644 bitreq/COPYING.md create mode 100644 bitreq/examples/async_hello.rs create mode 100644 bitreq/examples/hello.rs create mode 100644 bitreq/examples/iterator.rs create mode 100644 bitreq/examples/no-std.rs create mode 100644 bitreq/examples/wasm_example.rs create mode 100644 bitreq/src/connection.rs create mode 100644 bitreq/src/connection/rustls_stream.rs create mode 100644 bitreq/src/error.rs create mode 100644 bitreq/src/http_url.rs create mode 100644 bitreq/src/lib.rs create mode 100644 bitreq/src/proxy.rs create mode 100644 bitreq/src/request.rs create mode 100644 bitreq/src/response.rs create mode 100644 bitreq/src/wasm.rs create mode 100644 bitreq/tests/main.rs create mode 100644 bitreq/tests/setup.rs diff --git a/Cargo-minimal.lock b/Cargo-minimal.lock index 8c3eaa2e..b6d159d1 100644 --- a/Cargo-minimal.lock +++ b/Cargo-minimal.lock @@ -2,12 +2,30 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.95" @@ -20,6 +38,33 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base58ck" version = "0.1.0" @@ -111,7 +156,27 @@ checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "bitreq" -version = "0.0.0" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "chrono", + "log", + "punycode", + "rustls", + "rustls-native-certs", + "rustls-webpki", + "tiny_http", + "tokio", + "tokio-rustls", + "urlencoding", + "webpki-roots", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "byteorder" @@ -119,6 +184,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "bzip2" version = "0.4.4" @@ -155,6 +226,41 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "corepc-client" version = "0.10.0" @@ -267,6 +373,18 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "hex-conservative" version = "0.2.1" @@ -282,12 +400,52 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "js-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "jsonrpc" version = "0.18.0" @@ -328,6 +486,12 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + [[package]] name = "miniz_oxide" version = "0.8.4" @@ -352,12 +516,54 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi", + "libc", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + [[package]] name = "pkg-config" version = "0.3.31" @@ -373,6 +579,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "punycode" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e1dcb320d6839f6edb64f7a4a59d39b30480d4d1765b56873f7c858538a5fe" + [[package]] name = "quote" version = "1.0.36" @@ -406,6 +618,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + [[package]] name = "rustix" version = "0.38.37" @@ -431,6 +649,27 @@ dependencies = [ "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -441,12 +680,27 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "ryu" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "sct" version = "0.7.1" @@ -477,6 +731,29 @@ dependencies = [ "cc", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.203" @@ -514,6 +791,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socks" version = "0.3.4" @@ -566,6 +853,55 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + +[[package]] +name = "tokio" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2209a14885b74764cce87ffa777ffa1b8ce81a3f3166c6f886b83337fe7e077f" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "unicode-ident" version = "1.0.8" @@ -578,12 +914,77 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +dependencies = [ + "unicode-ident", +] + [[package]] name = "webpki-roots" version = "0.25.4" @@ -621,6 +1022,65 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -639,6 +1099,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" diff --git a/Cargo-recent.lock b/Cargo-recent.lock index 38882902..53897d05 100644 --- a/Cargo-recent.lock +++ b/Cargo-recent.lock @@ -2,12 +2,30 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.95" @@ -20,6 +38,33 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base58ck" version = "0.1.0" @@ -111,7 +156,27 @@ checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "bitreq" -version = "0.0.0" +version = "0.1.0" +dependencies = [ + "base64 0.22.1", + "chrono", + "log", + "punycode", + "rustls", + "rustls-native-certs", + "rustls-webpki", + "tiny_http", + "tokio", + "tokio-rustls", + "urlencoding", + "webpki-roots", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "byteorder" @@ -119,6 +184,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "bzip2" version = "0.4.4" @@ -155,6 +226,41 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "corepc-client" version = "0.10.0" @@ -267,6 +373,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "hex-conservative" version = "0.2.1" @@ -282,12 +394,63 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "io-uring" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "itoa" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "js-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "jsonrpc" version = "0.18.0" @@ -358,12 +521,53 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "mio" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + [[package]] name = "pkg-config" version = "0.3.31" @@ -379,6 +583,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "punycode" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e1dcb320d6839f6edb64f7a4a59d39b30480d4d1765b56873f7c858538a5fe" + [[package]] name = "quote" version = "1.0.38" @@ -412,6 +622,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + [[package]] name = "rustix" version = "0.38.44" @@ -437,6 +653,27 @@ dependencies = [ "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -447,12 +684,27 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "ryu" version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "sct" version = "0.7.1" @@ -483,6 +735,29 @@ dependencies = [ "cc", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.217" @@ -521,6 +796,22 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socks" version = "0.3.4" @@ -573,6 +864,57 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + +[[package]] +name = "tokio" +version = "1.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "pin-project-lite", + "slab", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "unicode-ident" version = "1.0.16" @@ -585,12 +927,77 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +dependencies = [ + "unicode-ident", +] + [[package]] name = "webpki-roots" version = "0.25.4" @@ -628,6 +1035,65 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -646,6 +1112,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" diff --git a/bitreq/.github/workflows/lint.yml b/bitreq/.github/workflows/lint.yml new file mode 100644 index 00000000..9d7048f3 --- /dev/null +++ b/bitreq/.github/workflows/lint.yml @@ -0,0 +1,29 @@ +name: lint + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +env: + CARGO_TERM_COLOR: always + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout Crate + uses: actions/checkout@v3 + - name: Set Toolchain + # https://github.com/dtolnay/rust-toolchain + uses: dtolnay/rust-toolchain@stable + - name: Run rustfmt + # rustfmt defaults to edition 2015 it seems. + run: rustfmt --check --edition=2018 src/lib.rs + - name: Run cargo doc + run: cargo doc --features "punycode proxy https" + - name: Run clippy + run: | + cargo clippy --all-targets --features "punycode proxy https-rustls" -- --no-deps -D warnings + cargo clippy --all-targets --features "punycode proxy https-rustls-probe" -- --no-deps -D warnings diff --git a/bitreq/.github/workflows/msrv.yml b/bitreq/.github/workflows/msrv.yml new file mode 100644 index 00000000..c85713e8 --- /dev/null +++ b/bitreq/.github/workflows/msrv.yml @@ -0,0 +1,24 @@ +name: msrv + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + schedule: + - cron: "47 5 * * 6" + +env: + CARGO_TERM_COLOR: always + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout Crate + uses: actions/checkout@v3 + - name: Checkout Toolchain + uses: dtolnay/rust-toolchain@1.63 + - name: Running test script + run: | + cargo test diff --git a/bitreq/.github/workflows/unit-tests.yml b/bitreq/.github/workflows/unit-tests.yml new file mode 100644 index 00000000..d91899be --- /dev/null +++ b/bitreq/.github/workflows/unit-tests.yml @@ -0,0 +1,63 @@ +name: unit-tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +env: + CARGO_TERM_COLOR: always + +jobs: + test-linux: + runs-on: ubuntu-latest + steps: + - name: Checkout Crate + uses: actions/checkout@v3 + - name: Set Toolchain + # https://github.com/dtolnay/rust-toolchain + uses: dtolnay/rust-toolchain@stable + - name: Build + run: cargo build + - name: Test + run: | + cargo test + cargo test --features punycode + cargo test --features proxy + cargo test --features urlencoding + cargo test --features https + test-windows: + runs-on: windows-latest + steps: + - name: Checkout Crate + uses: actions/checkout@v3 + - name: Set Toolchain + uses: dtolnay/rust-toolchain@stable + - name: Build + run: cargo build + - name: Test + run: | + cargo test + cargo test --features punycode + cargo test --features proxy + cargo test --features urlencoding + cargo test --features https + cargo test --features "punycode proxy urlencoding https" + test-macos: + runs-on: macos-latest + steps: + - name: Checkout Crate + uses: actions/checkout@v3 + - name: Set Toolchain + uses: dtolnay/rust-toolchain@stable + - name: Build + run: cargo build + - name: Test + run: | + cargo test + cargo test --features punycode + cargo test --features proxy + cargo test --features urlencoding + cargo test --features https + cargo test --features "punycode proxy urlencoding https" diff --git a/bitreq/.gitignore b/bitreq/.gitignore new file mode 100644 index 00000000..7d6a69bf --- /dev/null +++ b/bitreq/.gitignore @@ -0,0 +1,5 @@ + +/target/ +**/*.rs.bk +Cargo.lock +.claude diff --git a/bitreq/CHANGELOG.md b/bitreq/CHANGELOG.md index 14a4fbc9..4c01c877 100644 --- a/bitreq/CHANGELOG.md +++ b/bitreq/CHANGELOG.md @@ -1,3 +1,381 @@ -# 2025-10-21 - 0.0.0 +# Changelog +All notable changes to this project will be documented in this file. -Grabbing the name on crates.io. \ No newline at end of file +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +### Changed +- `https-bundled`, `https-bundled-probe`: Removed almost all of the bundled + native-tls code (~1k LoC), only keeping the relevant part (~30 LoC). There + should be no change to the actual code that ends up being ran, but if you're + using these features, make sure to test that everything works as you expect, + something might have slipped. + +### Fixed +- `https-*`: Refactored the TLS handling code a bit. This should have no visible + effect downstream, `src/connection.rs` is just a little bit more readable now. +- Removed `build.rs`, which turned out to be dead code. This should have no + effect, but if it does, it should also only affect the `https-bundled` and + `https-bundled-probe` features. + +## [2.13.4] - 2025-04-11 +### Fixed +- Updated the base64 dependency, only used by the `proxy` feature, to its newest + minor version. Thanks for the report, @Jackhr-arch! + ([#119](https://github.com/neonmoe/minreq/issues/119)) + +## [2.13.3] - 2025-03-11 +### Fixed +- Removed the `once_cell` dependency by making use of the new + `std::sync::OnceLock` type. This change only affects the rustls-based https + features. Thanks for the PR, @LyonSyonII! + ([#115](https://github.com/neonmoe/minreq/pull/115)) +- MSRV builds that got broken due to a `rustix` update. Now `tempfile` is pinned + as well. + +## [2.13.2] - 2025-01-29 +### Fixed +- Reverted a part of 2.13.1, accidentally removed some code that wasn't actually + dead code. + +## [2.13.1] - 2025-01-29 +### Fixed +- Usage of an openssl-probe function that's deprecated due to safety issues. See + [rustsec/advisory-db#2209](https://github.com/rustsec/advisory-db/pull/2209) + for further info. + +## [2.13.0] - 2024-12-04 +### Changed +- The `https-rustls-probe` feature no longer brings in the `webpki-roots` and + `rustls-webpki` crates. Thanks for the report, @polarathene! + ([#111](https://github.com/neonmoe/minreq/issues/111)) + +### Fixed +- Cleaned up an unnecessary `format!()` in `Connection::connect`. Thanks for the + PR, @melotic! ([#112](https://github.com/neonmoe/minreq/pull/112)) +- Fixed some msrv and lint issues introduced by libc and clippy updates + respectively. + +## [2.12.0] - 2024-07-16 +### Added +- Request::with_headers, to allow passing in many headers at a + time. Thanks for the idea and PR, @rawhuul! + ([#110](https://github.com/neonmoe/minreq/pull/110)) + +## [2.11.2] - 2024-04-26 +### Fixed +- The dev dependency tiny_http's version up to 0.12. Thanks for the + PR, @davide125! ([#107](https://github.com/neonmoe/minreq/pull/107)) + +## [2.11.1] - 2024-02-04 +### Fixed +- Unnecessary buffering causing performance problems. Thanks for the + PRs, @mrkline! ([#102](https://github.com/neonmoe/minreq/pull/102), + [#103](https://github.com/neonmoe/minreq/pull/103)) +- Connections failing if the first resolved address fails to connect + (even if there's more to try). Thanks for the PR, @darosior! + ([#106](https://github.com/neonmoe/minreq/pull/106)) + +## [2.11.0] - 2023-10-17 +### Changed +- Removed upper bounds on the `serde_json`, `log` and `chrono` + dependencies (dev-dependency in the case of `chrono`). If you were + depending on minreq compiling with the MSRV compiler without any + issues, check out the MSRV section in the readme, it's been updated + with additional instructions. Thanks for the report, @RCasatta! + ([#99](https://github.com/neonmoe/minreq/issues/99)) + +## [2.10.0] - 2023-09-05 +### Fixed +- Fragment handling, once again. Turns out you're not supposed to include + fragments in the request. This may break usage with servers that are written + with the wrong assumptions. Thanks for the report, @rawhuul! + ([#100](https://github.com/neonmoe/minreq/issues/100)) + +### Added +- `Response::url` and `ResponseLazy::url` fields, to contain the final URL after + redirects and fragment replacement semantics. + +## [2.9.1] - 2023-08-28 +### Changed +- Loosened the rustls version requirement from 0.21.6 to 0.21.1. + +## [2.9.0] - 2023-08-24 +### Changed +- From webpki to rustls-webpki. Thanks for the heads-up about webpki not + being maintained, @RCasatta! + ([#98](https://github.com/neonmoe/minreq/issues/98)) +- Updated rustls and webpki-roots to their most recent versions. +- Maximum versions for the following dependencies to keep minreq compiling on + Rust 1.48: + - serde_json (`>=1.0.0, <1.0.101`) + - log (`>=0.4.0, <0.4.19`) + - chrono (dev-dependency, `>=0.4.0, <0.4.24`) + +### Fixed +- Errors when using an IP address as the host with HTTPS (tested with + ). ([#34](https://github.com/neonmoe/minreq/issues/34)) + +## [2.8.1] - 2023-05-20 +### Fixed +- Proxy strings with the protocol included not working. Thanks for the report, + @tkkcc! ([#95](https://github.com/neonmoe/minreq/issues/95)) + +## [2.8.0] - 2023-05-13 +### Added +- Default proxy from environment variables when the `proxy` feature is + enabled, based on what curl does. Thanks for the PR, @krypt0nn! + ([#94](https://github.com/neonmoe/minreq/pull/94)) + +## [2.7.0] - 2023-03-19 +### Changed +- From lazy_static to once_cell for library internals. Thanks for the PR, + @alpha-tango-kilo! ([#80](https://github.com/neonmoe/minreq/pull/80)) + +### Added +- A Read impl for ResponseLazy. Thanks for the PR, @Luro02! + ([#81](https://github.com/neonmoe/minreq/pull/81)) +- Building with `--all-features`, with the `send_https` function defaulting to + the rustls-based implementation. Thanks for the PR, @tcharding! + ([#89](https://github.com/neonmoe/minreq/pull/89)) +- An explicit minimum supported rust version policy. The MSRV for versions 2.x + is 1.48. Thanks for the suggestion and PR, @tcharding! + ([#90](https://github.com/neonmoe/minreq/pull/90)) +- Performance improvements, test fixes, CI updates. + +## [2.6.0] - 2022-02-23 +### Changed +- The error returned when the request url does not start with + `https://` or `http://` now is now a slightly different IoError, + with a clearer message. This will be changed to a proper + minreq-specific error in 3.0, but for now it's an IoError to avoid + breaking the Error type. + +### Added +- The `urlencoding` feature for automatically percent-encoding + urls. Thanks for the idea and PR, @alpha-tango-kilo! + ([#67](https://github.com/neonmoe/minreq/issues/67), + [#68](https://github.com/neonmoe/minreq/pull/68)) + +## [2.5.1] - 2022-01-07 +### Fixed +- GitHub API requests without User-Agent returning an IoError. Thanks + for the report, @tech-ticks! + ([#66](https://github.com/neonmoe/minreq/issues/66)) + +## [2.5.0] - 2022-01-06 +### Fixed +- Returning the wrong status code when the response was missing a + status phrase. Thanks for the PR, @richarddd! + ([#64](https://github.com/neonmoe/minreq/issues/64)) +- Non-lazy requests crashing if the request had a very big + Content-Length header. Thanks for the report, @Shnatsel! + ([#63](https://github.com/neonmoe/minreq/issues/63)) + +## [2.4.2] - 2021-06-11 +### Fixed +- A regression in 2.4.1 where the port is no longer included in the + `Host`, even if it's a non-standard port. Now the port is always + included if it's in the request URL, and omitted if the port is + implied. Thanks for the report, @ollpu! + ([#61](https://github.com/neonmoe/minreq/issues/61)) + +## [2.4.1] - 2021-06-05 +### Fixed +- The port is no longer included in the `Host` header when sending + requests, and port handling was cleaned up overall. This fixes + issues with infinite redirections and https handshakes for some + websites. Thanks to @Shnatsel for reporting the issues, and @joeried + for debugging and figuring out the root cause of these problems! + ([#48](https://github.com/neonmoe/minreq/issues/48), + [#49](https://github.com/neonmoe/minreq/issues/49)) + +## [2.4.0] - 2021-05-27 +### Added +- `Request::with_param` for more ergonomic query parameter + usage. Thanks for the PR, @sjvignesh! + ([#54](https://github.com/neonmoe/minreq/pull/54)) +- `Request::with_max_headers_size` and + `Request::with_max_status_line_length` for avoiding DoS when the + server sends large headers or status lines. Thanks for the report, + @Shnatsel! ([#55](https://github.com/neonmoe/minreq/issues/55)) +- Support for the `rustls-native-certs` crate via a new + `https-rustls-probe` feature. Thanks for the PR, @joeried! + ([#59](https://github.com/neonmoe/minreq/pull/59)) + +### Fixed +- Chunk length handling for some servers with slightly off-spec chunk + lengths. Thanks for the report, @Shnatsel! + ([#50](https://github.com/neonmoe/minreq/issues/50)) +- Timeouts not always being properly enforced. Thanks for the report, + @Shnatsel! ([#52](https://github.com/neonmoe/minreq/issues/52)) + +## [2.3.1] - 2021-02-10 +### Fixed +- Removed some leftover printlns from the redirection update in 2.3.0 + and ensured there's no printlns in the library anymore. Thanks for + reporting the issue @Shnatsel! + [#45](https://github.com/neonmoe/minreq/issues/45) +- Fixed the timeout not being respected during the initial TCP + connect. Thanks for the report and fix @KarthikNedunchezhiyan! + [#46](https://github.com/neonmoe/minreq/issues/46), + [#47](https://github.com/neonmoe/minreq/pull/47) + +## [2.3.0] - 2021-01-04 +### Changed +- **Breaking (sort of):** the redirection code was improved to match + [RFC 7231 section + 7.1.2](https://tools.ietf.org/html/rfc7231#section-7.1.2), which + could subtly break some programs relying on very specific redirects, + which is why this should be investigated if you come across weird + behaviour after updating. No API changes though, so only a minor + version bump. The following two points are now fixed when + redirecting: + - Fragments, the bit after a #-character in the url. If the + redirecting url has a fragment, and the one in `Location` does + not, the original fragment should be included in the new url. If + `Location` does have a fragment, it should override the one in the + redirecting url. + - Relative urls. Minreq now properly redirects when `Location` is + relative, e.g. `/Foo.html` instead of + `https://example.com/Foo.html`. Thanks, @fjt523! + +### Fixed +- The `Content-Length: 0` header is now inserted into requests that + should have it. Thanks, @KarthikNedunchezhiyan! +- Status line parsing is now fixed, so "400 Bad Request" is not parsed + as "400 Bad". Thanks, @KarthikNedunchezhiyan! + +### Added +- M1 Mac support by bumping the ring dependency. Thanks, @ryanmcgrath! + +## [2.2.1] - 2020-08-22 +### Fixed +- Some documentation which has been long due for an update. I just + always forget when writing an actual update. No code changes! + +## [2.2.0] - 2020-06-18 +### Added +- Support for `native-tls` and `openssl-sys` via new features, in + addition to `rustls`. Thanks to @dubiousjim! + +## [2.1.1] - 2020-05-01 +### Fixed +- Handling of status codes 204 and 304. Thanks to @Mubelotix! + +## [2.1.0] - 2020-03-14 +### Added +- Proxy support via the `proxy` feature. Thanks to @rustysec! + +## [2.0.3] - 2020-01-15 +### Fixed +- Fixed regression in header parsing caused by 2.0.2, which was yanked. + +## [2.0.2] - 2020-01-15 +### Fixed +- Fixed a panic when sending a request to an invalid domain + via https. +- Fixed a panic when parsing headers that have >1 byte + unicode characters right after the ":" in the response. + +## [2.0.1] - 2020-01-11 +### Fixed +- Made timeouts work as described in the documentation. + Fixed issue #22. + +## [2.0.0] - 2019-11-23 +### Added +- API for loading the HTTP response body through an iterator, allowing + for processing of the data during the download. + - See the `ResponseLazy` documentation for more information. +- Error type for all the errors that this crate can run into for + easier `?` usage and better debuggability. +- Punycode support for non-ascii hostnames via the `punycode` feature. +- Trailer header support. +- Examples [`hello`](examples/hello.rs), + [`iterator`](examples/iterator.rs), and [`json`](examples/json.rs). + +### Changed +- **Breaking, will cause problems not detectable by the compiler:** + Response headers' field names are now in lowercase, as they are + case-insensitive and this makes getting header values easier. The + values are unaffected. So if your code has + `response.headers.get("Content-Type")`, you have to change it to + `response.headers.get("content-type")`, or it will not return what + you want. +- **Breaking**: Restructure the `Response` struct: + - Removed `bytes` and `body_bytes`. + - Added `as_bytes()`, `into_bytes()`, and `as_str()` in their place. +- **Breaking**: Changed the `with_body` parameter type to + `Into>` from `Into`. + - `String`s implement `Into>`, so this shouldn't cause any + problems, unless you're using some interesting types that + implement `Into` but not `Into>`. +- Clean up the crate internals overall. **Note**: This might cause + instability, if you're very concerned about stability, please hold + off upgrading for a while. +- Remove `panic!` when trying to make an `https://` request without + the `https` feature. The request will now return an error + instead. The library should not panic anymore. +- Audit the remaining `unwrap()`s from library code, none of them + should actually ever cause a panic now. + +### Removed +- `create_request` in favor of just using `Response::new`. + +## [1.4.1] - 2019-10-13 +### Changed +- Updated dependencies. + +### Fixed +- Tests on Windows by changing the ip in tests from `0.0.0.0` to + `localhost`. +- Reuse `rustls::ClientConfig` between requests. +- `Content-Length` and `Transfer-Encoding` detection failing because + of case-sensitiveness. + +## [1.4.0] - 2019-07-13 +### Added +- `json-using-serde` feature. + +## [1.3.0] - 2019-06-04 +### Added +- The `body_bytes` field to Response, containing the body in raw + bytes. + +### Fixed +- Some clippy warnings. +- Panic when getting a non-UTF-8 response, instead setting the `body` + string to an empty string, for now. + +## [1.2.1] - 2019-05-24 +### Fixed +- HTTP response body handling. + +## [1.2.0] - 2019-05-23 +### Added +- Support for the HTTP status codes 301, 302, 303, and 307. + +### Fixed +- Less .clones()s. + +## [1.1.2] - 2019-04-14 +### Fixed +- Fix response handling when `Transfer-Encoding` is `chunked`. + +## [1.1.1] - 2019-03-28 +### Changed +- Moved to 2018 edition. + +### Fixed +- HEAD requests and ones that receive a 1xx, 204, or 304 status code + as a response. + +## [1.1.0] - 2019-03-24 +### Changed +- Timeout made optional. +- Updated dependencies. + +### Fixed +- Improved performance for HTTP (not HTTPS) requests. diff --git a/bitreq/COPYING.md b/bitreq/COPYING.md new file mode 100644 index 00000000..c2cedfc9 --- /dev/null +++ b/bitreq/COPYING.md @@ -0,0 +1,16 @@ +ISC License + +Copyright (c) 2018, Jens Pitkanen +Copyright (c) 2025, Tobin C. Harding + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/bitreq/Cargo.toml b/bitreq/Cargo.toml index e6e76d67..bce0dfe6 100644 --- a/bitreq/Cargo.toml +++ b/bitreq/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "bitreq" -version = "0.0.0" -authors = ["Tobin C. Harding "] +version = "0.1.0" +authors = ["Jens Pitkanen ", "Tobin C. Harding "] description = "Simple, minimal-dependency HTTP client" documentation = "https://docs.rs/bitreq" repository = "https://github.com/rust-bitcoin/corepc" @@ -12,4 +12,58 @@ license = "ISC" edition = "2021" rust-version = "1.74.0" +[badges] +maintenance = { status = "experimental" } + [dependencies] +# For the urlencoding feature: +urlencoding = { version = "2.1.0", optional = true } +# For the punycode feature: +punycode = { version = "0.4.1", optional = true } +# For the proxy feature: +base64 = { version = "0.22", optional = true } +# For the https features: +rustls = { version = "0.21.1", optional = true } +rustls-native-certs = { version = "0.6.1", optional = true } +webpki-roots = { version = "0.25.2", optional = true } +rustls-webpki = { version = "0.101.0", optional = true } +log = { version = "0.4.0", optional = true } +# For the async feature: +tokio = { version = "1.0", features = ["net", "time", "io-util", "rt", "rt-multi-thread", "macros"], optional = true } +tokio-rustls = { version = "0.24", optional = true } + +[dev-dependencies] +tiny_http = "0.12" +chrono = "0.4.0" + +[package.metadata.docs.rs] +features = ["proxy", "https", "punycode"] + +[features] +default = ["std"] +std = [] +log = ["dep:log"] +https = ["https-rustls"] +https-rustls = ["rustls", "webpki-roots", "rustls-webpki"] +https-rustls-probe = ["rustls", "rustls-native-certs"] +proxy = ["base64"] +async = ["tokio", "std"] +async-https = ["async", "https-rustls", "tokio-rustls"] + +[[example]] +name = "hello" +required-features = ["std"] + +[[example]] +name = "iterator" +required-features = ["std"] + +[[example]] +name = "async_hello" +required-features = ["async"] + +[lints.clippy] +# Allow `format!("{}", x)`instead of enforcing `format!("{x}")` +uninlined_format_args = "allow" + +# vim: ft=conf diff --git a/bitreq/README.md b/bitreq/README.md index 777598a7..1815cd7d 100644 --- a/bitreq/README.md +++ b/bitreq/README.md @@ -1,3 +1,83 @@ -# bitreq +# bitreq - forked from minreq +[![Crates.io](https://img.shields.io/crates/d/bitreq.svg)](https://crates.io/crates/bitreq) +[![Documentation](https://docs.rs/bitreq/badge.svg)](https://docs.rs/bitreq) +![Unit tests](https://github.com/tcharding/bitreq/actions/workflows/unit-tests.yml/badge.svg) +![MSRV](https://github.com/tcharding/bitreq/actions/workflows/msrv.yml/badge.svg) -Minimal dependency HTTP crate. +This crate is a fork for the very nice +[minreq](https://github.com/neonmoe/minreq). I chose to fork and +rename it because I wanted to totally gut it and provide a crate with +different goals. Many thanks to the original author. + +Simple, minimal-dependency HTTP client. Optional features for +unicode domains (`punycode`), http proxies (`proxy`), async support +(`async`, `async-https`), and https with various TLS implementations +(`https-rustls`, `https-rustls-probe`, and `https` which is an alias +for `https-rustls`). + +Without any optional features, my casual testing indicates about 100 +KB additional executable size for stripped release builds using this +crate. Compiled with rustc 1.45.2, `println!("Hello, World!");` is 239 +KB on my machine, where the [hello](examples/hello.rs) example is 347 +KB. Both are pure Rust, so aside from `libc`, everything is statically +linked. + +Note: some of the dependencies of this crate (especially the various +`https` libraries) are a lot more complicated than this library, and +their impact on executable size reflects that. + +## Documentation + +Build your own with `cargo doc --all-features`, or browse the online +documentation at [docs.rs/bitreq](https://docs.rs/bitreq). + +## Minimum Supported Rust Version (MSRV) + +If you don't care about the MSRV, you can ignore this section +entirely, including the commands instructed. + +We use an MSRV per major release, i.e., with a new major release we +reserve the right to change the MSRV. + +The current major version of this library should always compile with +default features (i.e., `std`) on **Rust 1.63**. Other features may +require a higher MSRV. + +## License +This crate is distributed under the terms of the [ISC license](COPYING.md). + +## Planned for 3.0.0 + +This is a list of features I'll implement once it gets long enough, or +a severe enough issue is found that there's good reason to make a +major version bump. + +- Change the response/request structs to allow multiple headers with + the same name. +- Set sane defaults for maximum header size and status line + length. The ability to add maximums was added in response to + [#55](https://github.com/neonmoe/minreq/issues/55), but defaults for + the limits is a breaking change. +- Clearer error when making a request to an url that does not start + with `http://` or `https://`. +- Change default proxy port to 1080 (from 8080). Curl uses 1080, so it's a sane + default. +- Bump MSRV enough to compile the latest versions of all dependencies, and add + the `rust-version` (at least 1.56) and `edition` (at least 2021) fields to + Cargo.toml. + +### Potential ideas + +Just thinking out loud, might not end up doing some or all of these. + +- Non-exhaustive error type, to be able to add new errors in minor + versions. +- Refactor applicable parts to `#![no_std]`, maybe even exposing a + less convenient API for `#![no_std]` usage. Keep the current API as + in any case (at the very least, as a default feature). + - Maybe something along the lines of ["The case for + sans-io"](https://fasterthanli.me/articles/the-case-for-sans-io)? + Adding the much-requested async support as a feature could be + pretty clean if built around this idea. +- Would be good if the crate got smaller with 3.0, not bigger. Maybe + there's something to cut, something to optimize? diff --git a/bitreq/examples/async_hello.rs b/bitreq/examples/async_hello.rs new file mode 100644 index 00000000..705f0843 --- /dev/null +++ b/bitreq/examples/async_hello.rs @@ -0,0 +1,11 @@ +//! This example demonstrates the `async` feature. + +#[tokio::main] +async fn main() -> Result<(), bitreq::Error> { + let response = bitreq::get("http://httpbin.org/get").send_async().await?; + + println!("Status: {}", response.status_code); + println!("Body: {}", response.as_str()?); + + Ok(()) +} diff --git a/bitreq/examples/hello.rs b/bitreq/examples/hello.rs new file mode 100644 index 00000000..1a3cf647 --- /dev/null +++ b/bitreq/examples/hello.rs @@ -0,0 +1,10 @@ +//! This is a simple example to demonstrate the usage of this library. + +#![cfg(feature = "std")] + +fn main() -> Result<(), bitreq::Error> { + let response = bitreq::get("http://example.com").send()?; + let html = response.as_str()?; + println!("{}", html); + Ok(()) +} diff --git a/bitreq/examples/iterator.rs b/bitreq/examples/iterator.rs new file mode 100644 index 00000000..a39dc4c1 --- /dev/null +++ b/bitreq/examples/iterator.rs @@ -0,0 +1,48 @@ +//! This example demonstrates probably the most complicated part of +//! `bitreq`. Useful when making loading bars, for example. + +#![cfg(feature = "std")] + +fn main() -> Result<(), bitreq::Error> { + let mut buffer = Vec::new(); + for byte in bitreq::get("http://example.com").send_lazy()? { + // The connection could have a problem at any point during the + // download, so each byte needs to be unwrapped. + let (byte, len) = byte?; + + // The `byte` is the current u8 of data we're iterating + // through. + print!("{}", byte as char); + + // The `len` is the expected amount of incoming bytes + // including the current one: this will be the rest of the + // body if the server provided a Content-Length header, or + // just the size of the remaining chunk in chunked transfers. + buffer.reserve(len); + buffer.push(byte); + + // Flush the printed text so each char appears on your + // terminal right away. + flush(); + + // Wait for 50ms so the data doesn't appear instantly fast + // internet connections, to demonstrate that the body is being + // printed char-by-char. + sleep(); + } + Ok(()) +} + +// Helper functions + +fn flush() { + use std::io::{stdout, Write}; + stdout().lock().flush().ok(); +} + +fn sleep() { + use std::thread::sleep; + use std::time::Duration; + + sleep(Duration::from_millis(2)); +} diff --git a/bitreq/examples/no-std.rs b/bitreq/examples/no-std.rs new file mode 100644 index 00000000..1ebffc54 --- /dev/null +++ b/bitreq/examples/no-std.rs @@ -0,0 +1,71 @@ +//! This is a simple example to demonstrate the usage of this library. + +#![cfg(feature = "std")] + +const _RESPONSE: &str = r#" + + + Example Domain + + + + + + + + +
+

Example Domain

+

This domain is for use in illustrative examples in documents. You may use this + domain in literature without prior coordination or asking for permission.

+

More information...

+
+ +"#; + +fn main() -> Result<(), bitreq::Error> { + // TODO: For this request object to be useful we probably need to + // either make the `Request` fields all public or make the + // `ParsedRequest` and associated types public. + // + // Either option is a reasonably invasive change in design so + // needs thorough consideration. + + // FIXME: Found to be broken while importing from minireq. + + // let request = bitreq::get("http://example.com"); + + // // Do what you need to do to send the request to the server. + // let response = Response::from_bytes(RESPONSE); + + // let body = response.as_str()?; + // println!("{}", body); + + Ok(()) +} diff --git a/bitreq/examples/wasm_example.rs b/bitreq/examples/wasm_example.rs new file mode 100644 index 00000000..b2e5edc3 --- /dev/null +++ b/bitreq/examples/wasm_example.rs @@ -0,0 +1,25 @@ +//! WASM example demonstrating extern C function integration +//! +//! This example shows how to use bitreq with the `wasm` feature. +//! The actual HTTP requests are performed by JavaScript through extern C functions. + +fn main() { + // Example usage in WASM environment + let _request = + bitreq::get("https://httpbin.org/get").with_header("User-Agent", "bitreq-wasm/0.1.0"); + + // In a real WASM environment, this would call out to JavaScript + // The JavaScript implementation would need to provide: + // - bitreq_wasm_http_request + // - bitreq_wasm_get_status_code + // - bitreq_wasm_get_response_headers + + println!("Request prepared for WASM execution:"); + println!("URL: https://httpbin.org/get"); + println!("Method: GET"); + println!("Headers: User-Agent: bitreq-wasm/0.1.0"); + + // Note: This won't actually execute in non-WASM environments + // as the extern C functions aren't implemented + println!("To run this, compile to WASM and provide JavaScript implementations"); +} diff --git a/bitreq/src/connection.rs b/bitreq/src/connection.rs new file mode 100644 index 00000000..0a72066d --- /dev/null +++ b/bitreq/src/connection.rs @@ -0,0 +1,381 @@ +use core::time::Duration; +use std::env; +use std::io::{self, Read, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::time::Instant; + +use crate::request::ParsedRequest; +use crate::{Error, Method, ResponseLazy}; + +type UnsecuredStream = TcpStream; + +#[cfg(feature = "rustls")] +mod rustls_stream; +#[cfg(feature = "rustls")] +type SecuredStream = rustls_stream::SecuredStream; + +pub(crate) enum HttpStream { + Unsecured(UnsecuredStream, Option), + #[cfg(feature = "rustls")] + Secured(Box, Option), +} + +impl HttpStream { + fn create_unsecured(reader: UnsecuredStream, timeout_at: Option) -> HttpStream { + HttpStream::Unsecured(reader, timeout_at) + } + + #[cfg(feature = "rustls")] + fn create_secured(reader: SecuredStream, timeout_at: Option) -> HttpStream { + HttpStream::Secured(Box::new(reader), timeout_at) + } +} + +fn timeout_err() -> io::Error { + io::Error::new(io::ErrorKind::TimedOut, "the timeout of the request was reached") +} + +fn timeout_at_to_duration(timeout_at: Option) -> Result, io::Error> { + if let Some(timeout_at) = timeout_at { + if let Some(duration) = timeout_at.checked_duration_since(Instant::now()) { + Ok(Some(duration)) + } else { + Err(timeout_err()) + } + } else { + Ok(None) + } +} + +impl Read for HttpStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let timeout = |tcp: &TcpStream, timeout_at: Option| -> io::Result<()> { + let _ = tcp.set_read_timeout(timeout_at_to_duration(timeout_at)?); + Ok(()) + }; + + let result = match self { + HttpStream::Unsecured(inner, timeout_at) => { + timeout(inner, *timeout_at)?; + inner.read(buf) + } + #[cfg(feature = "rustls")] + HttpStream::Secured(inner, timeout_at) => { + timeout(inner.get_ref(), *timeout_at)?; + inner.read(buf) + } + }; + match result { + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + // We're a blocking socket, so EWOULDBLOCK indicates a timeout + Err(timeout_err()) + } + r => r, + } + } +} + +/// An async connection to the server for sending +/// [`Request`](struct.Request.html)s. +#[cfg(feature = "async")] +pub struct AsyncConnection { + request: ParsedRequest, + timeout_at: Option, +} + +#[cfg(feature = "async")] +impl AsyncConnection { + /// Creates a new `AsyncConnection`. + pub(crate) fn new(request: ParsedRequest) -> AsyncConnection { + let timeout = request.config.timeout.or_else(|| match env::var("MINREQ_TIMEOUT") { + Ok(t) => t.parse::().ok(), + Err(_) => None, + }); + let timeout_at = timeout.map(|t| Instant::now() + Duration::from_secs(t)); + AsyncConnection { request, timeout_at } + } + + /// Sends the [`Request`](struct.Request.html) asynchronously using HTTPS. + #[cfg(feature = "async-https")] + pub(crate) async fn send_https(self) -> Result { + // Use spawn_blocking to run the sync HTTPS code in a thread pool + let sync_conn = Connection { request: self.request, timeout_at: self.timeout_at }; + + tokio::task::spawn_blocking(move || sync_conn.send_https()) + .await + .map_err(|e| Error::IoError(io::Error::new(io::ErrorKind::Other, e)))? + } + + /// Sends the [`Request`](struct.Request.html) asynchronously using HTTP. + pub(crate) async fn send(self) -> Result { + // Use spawn_blocking to run the sync HTTP code in a thread pool + let sync_conn = Connection { request: self.request, timeout_at: self.timeout_at }; + + tokio::task::spawn_blocking(move || sync_conn.send()) + .await + .map_err(|e| Error::IoError(io::Error::new(io::ErrorKind::Other, e)))? + } +} + +/// A connection to the server for sending +/// [`Request`](struct.Request.html)s. +pub struct Connection { + request: ParsedRequest, + timeout_at: Option, +} + +impl Connection { + /// Creates a new `Connection`. See [Request] and [ParsedRequest] + /// for specifics about *what* is being sent. + pub(crate) fn new(request: ParsedRequest) -> Connection { + let timeout = request.config.timeout.or_else(|| match env::var("MINREQ_TIMEOUT") { + Ok(t) => t.parse::().ok(), + Err(_) => None, + }); + let timeout_at = timeout.map(|t| Instant::now() + Duration::from_secs(t)); + Connection { request, timeout_at } + } + + /// Returns the timeout duration for operations that should end at + /// timeout and are starting "now". + /// + /// The Result will be Err if the timeout has already passed. + fn timeout(&self) -> Result, io::Error> { + let timeout = timeout_at_to_duration(self.timeout_at); + #[cfg(feature = "log")] + log::trace!("Timeout requested, it is currently: {:?}", timeout); + timeout + } + + /// Sends the [`Request`](struct.Request.html), consumes this + /// connection, and returns a [`Response`](struct.Response.html). + #[cfg(feature = "rustls")] + pub(crate) fn send_https(mut self) -> Result { + enforce_timeout(self.timeout_at, move || { + self.request.url.host = ensure_ascii_host(self.request.url.host)?; + + let secured_stream = rustls_stream::create_secured_stream(&self)?; + + #[cfg(feature = "log")] + log::trace!("Reading HTTPS response from {}.", self.request.url.host); + let response = ResponseLazy::from_stream( + secured_stream, + self.request.config.max_headers_size, + self.request.config.max_status_line_len, + )?; + + handle_redirects(self, response) + }) + } + + /// Sends the [`Request`](struct.Request.html), consumes this + /// connection, and returns a [`Response`](struct.Response.html). + pub(crate) fn send(mut self) -> Result { + enforce_timeout(self.timeout_at, move || { + self.request.url.host = ensure_ascii_host(self.request.url.host)?; + let bytes = self.request.as_bytes(); + + #[cfg(feature = "log")] + log::trace!("Establishing TCP connection to {}.", self.request.url.host); + let mut tcp = self.connect()?; + + // Send request + #[cfg(feature = "log")] + log::trace!("Writing HTTP request."); + let _ = tcp.set_write_timeout(self.timeout()?); + tcp.write_all(&bytes)?; + + // Receive response + #[cfg(feature = "log")] + log::trace!("Reading HTTP response."); + let stream = HttpStream::create_unsecured(tcp, self.timeout_at); + let response = ResponseLazy::from_stream( + stream, + self.request.config.max_headers_size, + self.request.config.max_status_line_len, + )?; + handle_redirects(self, response) + }) + } + + fn connect(&self) -> Result { + let tcp_connect = |host: &str, port: u32| -> Result { + let addrs = (host, port as u16).to_socket_addrs().map_err(Error::IoError)?; + let addrs_count = addrs.len(); + + // Try all resolved addresses. Return the first one to which we could connect. If all + // failed return the last error encountered. + for (i, addr) in addrs.enumerate() { + let stream = if let Some(timeout) = self.timeout()? { + TcpStream::connect_timeout(&addr, timeout) + } else { + TcpStream::connect(addr) + }; + if stream.is_ok() || i == addrs_count - 1 { + return stream.map_err(Error::from); + } + } + + Err(Error::AddressNotFound) + }; + + #[cfg(feature = "proxy")] + match self.request.config.proxy { + Some(ref proxy) => { + // do proxy things + let mut tcp = tcp_connect(&proxy.server, proxy.port)?; + + write!(tcp, "{}", proxy.connect(&self.request)).unwrap(); + tcp.flush()?; + + let mut proxy_response = Vec::new(); + + loop { + let mut buf = vec![0; 256]; + let total = tcp.read(&mut buf)?; + proxy_response.append(&mut buf); + if total < 256 { + break; + } + } + + crate::Proxy::verify_response(&proxy_response)?; + + Ok(tcp) + } + None => tcp_connect(&self.request.url.host, self.request.url.port.port()), + } + + #[cfg(not(feature = "proxy"))] + tcp_connect(&self.request.url.host, self.request.url.port.port()) + } +} + +fn handle_redirects( + connection: Connection, + mut response: ResponseLazy, +) -> Result { + let status_code = response.status_code; + let url = response.headers.get("location"); + match get_redirect(connection, status_code, url) { + NextHop::Redirect(connection) => { + let connection = connection?; + if connection.request.url.https { + #[cfg(not(feature = "rustls"))] + return Err(Error::HttpsFeatureNotEnabled); + #[cfg(feature = "rustls")] + return connection.send_https(); + } else { + connection.send() + } + } + NextHop::Destination(connection) => { + let dst_url = connection.request.url; + dst_url.write_base_url_to(&mut response.url).unwrap(); + dst_url.write_resource_to(&mut response.url).unwrap(); + Ok(response) + } + } +} + +enum NextHop { + Redirect(Result), + Destination(Connection), +} + +fn get_redirect(mut connection: Connection, status_code: i32, url: Option<&String>) -> NextHop { + match status_code { + 301 | 302 | 303 | 307 => { + let url = match url { + Some(url) => url, + None => return NextHop::Redirect(Err(Error::RedirectLocationMissing)), + }; + #[cfg(feature = "log")] + log::debug!("Redirecting ({}) to: {}", status_code, url); + + match connection.request.redirect_to(url.as_str()) { + Ok(()) => { + if status_code == 303 { + match connection.request.config.method { + Method::Post | Method::Put | Method::Delete => { + connection.request.config.method = Method::Get; + } + _ => {} + } + } + + NextHop::Redirect(Ok(connection)) + } + Err(err) => NextHop::Redirect(Err(err)), + } + } + _ => NextHop::Destination(connection), + } +} + +fn ensure_ascii_host(host: String) -> Result { + if host.is_ascii() { + Ok(host) + } else { + #[cfg(not(feature = "punycode"))] + { + Err(Error::PunycodeFeatureNotEnabled) + } + + #[cfg(feature = "punycode")] + { + let mut result = String::with_capacity(host.len() * 2); + for s in host.split('.') { + if s.is_ascii() { + result += s; + } else { + match punycode::encode(s) { + Ok(s) => result = result + "xn--" + &s, + Err(_) => return Err(Error::PunycodeConversionFailed), + } + } + result += "."; + } + result.truncate(result.len() - 1); // Remove the trailing dot + Ok(result) + } + } +} + +/// Enforce the timeout by running the function in a new thread and +/// parking the current one with a timeout. +/// +/// While bitreq does use timeouts (somewhat) properly, some +/// interfaces such as [ToSocketAddrs] don't allow for specifying the +/// timeout. Hence this. +fn enforce_timeout(timeout_at: Option, f: F) -> Result +where + F: 'static + Send + FnOnce() -> Result, + R: 'static + Send, +{ + use std::sync::mpsc::{channel, RecvTimeoutError}; + + match timeout_at { + Some(deadline) => { + let (sender, receiver) = channel(); + let thread = std::thread::spawn(move || { + let result = f(); + let _ = sender.send(()); + result + }); + if let Some(timeout_duration) = deadline.checked_duration_since(Instant::now()) { + match receiver.recv_timeout(timeout_duration) { + Ok(()) => thread.join().unwrap(), + Err(err) => match err { + RecvTimeoutError::Timeout => Err(Error::IoError(timeout_err())), + RecvTimeoutError::Disconnected => + Err(Error::Other("request connection paniced")), + }, + } + } else { + Err(Error::IoError(timeout_err())) + } + } + None => f(), + } +} diff --git a/bitreq/src/connection/rustls_stream.rs b/bitreq/src/connection/rustls_stream.rs new file mode 100644 index 00000000..f0fe7ba6 --- /dev/null +++ b/bitreq/src/connection/rustls_stream.rs @@ -0,0 +1,75 @@ +//! TLS connection handling functionality when using the `rustls` crate for +//! handling TLS. + +use alloc::sync::Arc; +use core::convert::TryFrom; +use std::io::{self, Write}; +use std::net::TcpStream; + +use rustls::{self, ClientConfig, ClientConnection, RootCertStore, ServerName, StreamOwned}; +#[cfg(feature = "rustls-webpki")] +use webpki_roots::TLS_SERVER_ROOTS; + +use super::{Connection, HttpStream}; +use crate::Error; + +pub type SecuredStream = StreamOwned; + +#[allow(clippy::incompatible_msrv)] // We only guarantee MSRV for a subset of features. +static CONFIG: std::sync::LazyLock> = std::sync::LazyLock::new(|| { + let mut root_certificates = RootCertStore::empty(); + + // Try to load native certs + #[cfg(feature = "https-rustls-probe")] + if let Ok(os_roots) = rustls_native_certs::load_native_certs() { + for root_cert in os_roots { + // Ignore erroneous OS certificates, there's nothing + // to do differently in that situation anyways. + let _ = root_certificates.add(&rustls::Certificate(root_cert.0)); + } + } + + #[cfg(feature = "rustls-webpki")] + #[allow(deprecated)] // Need to use add_server_trust_anchors to compile with rustls 0.21.1 + root_certificates.add_server_trust_anchors(TLS_SERVER_ROOTS.iter().map(|ta| { + rustls::OwnedTrustAnchor::from_subject_spki_name_constraints( + ta.subject, + ta.spki, + ta.name_constraints, + ) + })); + + let config = ClientConfig::builder() + .with_safe_defaults() + .with_root_certificates(root_certificates) + .with_no_client_auth(); + Arc::new(config) +}); + +pub fn create_secured_stream(conn: &Connection) -> Result { + // Rustls setup + #[cfg(feature = "log")] + log::trace!("Setting up TLS parameters for {}.", conn.request.url.host); + let dns_name = match ServerName::try_from(&*conn.request.url.host) { + Ok(result) => result, + Err(err) => return Err(Error::IoError(io::Error::new(io::ErrorKind::Other, err))), + }; + let sess = + ClientConnection::new(CONFIG.clone(), dns_name).map_err(Error::RustlsCreateConnection)?; + + // Connect + #[cfg(feature = "log")] + log::trace!("Establishing TCP connection to {}.", conn.request.url.host); + let tcp = conn.connect()?; + + // Send request + #[cfg(feature = "log")] + log::trace!("Establishing TLS session to {}.", conn.request.url.host); + let mut tls = StreamOwned::new(sess, tcp); // I don't think this actually does any communication. + #[cfg(feature = "log")] + log::trace!("Writing HTTPS request to {}.", conn.request.url.host); + let _ = tls.get_ref().set_write_timeout(conn.timeout()?); + tls.write_all(&conn.request.as_bytes())?; + + Ok(HttpStream::create_secured(tls, conn.timeout_at)) +} diff --git a/bitreq/src/error.rs b/bitreq/src/error.rs new file mode 100644 index 00000000..0487fca7 --- /dev/null +++ b/bitreq/src/error.rs @@ -0,0 +1,140 @@ +use core::{fmt, str}; +#[cfg(feature = "std")] +use std::{error, io}; + +/// Represents an error while sending, receiving, or parsing an HTTP response. +#[derive(Debug)] +// TODO: Make non-exhaustive for 3.0? +// TODO: Maybe make a few inner error types containing groups of these, based on +// what the user might want to handle? This error doesn't really invite graceful +// handling. +pub enum Error { + /// The response body contains invalid UTF-8, so the `as_str()` + /// conversion failed. + InvalidUtf8InBody(str::Utf8Error), + + #[cfg(feature = "rustls")] + /// Ran into a rustls error while creating the connection. + RustlsCreateConnection(rustls::Error), + /// Ran into an IO problem while loading the response. + #[cfg(feature = "std")] + IoError(io::Error), + /// Couldn't parse the incoming chunk's length while receiving a + /// response with the header `Transfer-Encoding: chunked`. + MalformedChunkLength, + /// The chunk did not end after reading the previously read amount + /// of bytes. + MalformedChunkEnd, + /// Couldn't parse the `Content-Length` header's value as an + /// `usize`. + MalformedContentLength, + /// The response contains headers whose total size surpasses + /// [Request::with_max_headers_size](crate::request::Request::with_max_headers_size). + HeadersOverflow, + /// The response's status line length surpasses + /// [Request::with_max_status_line_size](crate::request::Request::with_max_status_line_length). + StatusLineOverflow, + /// [ToSocketAddrs](std::net::ToSocketAddrs) did not resolve to an + /// address. + AddressNotFound, + /// The response was a redirection, but the `Location` header is + /// missing. + RedirectLocationMissing, + /// The response redirections caused an infinite redirection loop. + InfiniteRedirectionLoop, + /// Followed + /// [`max_redirections`](struct.Request.html#method.with_max_redirections) + /// redirections, won't follow any more. + TooManyRedirections, + /// The response contained invalid UTF-8 where it should be valid + /// (eg. headers), so the response cannot interpreted correctly. + InvalidUtf8InResponse, + /// The provided url contained a domain that has non-ASCII + /// characters, and could not be converted into punycode. It is + /// probably not an actual domain. + PunycodeConversionFailed, + /// Tried to send a secure request (ie. the url started with + /// `https://`), but the crate's `https` feature was not enabled, + /// and as such, a connection cannot be made. + HttpsFeatureNotEnabled, + /// The provided url contained a domain that has non-ASCII + /// characters, but it could not be converted into punycode + /// because the `punycode` feature was not enabled. + PunycodeFeatureNotEnabled, + /// The provided proxy information was not properly formatted. See + /// [Proxy::new](crate::Proxy::new) for the valid format. + BadProxy, + /// The provided credentials were rejected by the proxy server. + BadProxyCreds, + /// The provided proxy credentials were malformed. + ProxyConnect, + /// The provided credentials were rejected by the proxy server. + InvalidProxyCreds, + // TODO: Uncomment these two for 3.0 + // /// The URL does not start with http:// or https://. + // InvalidProtocol, + // /// The URL ended up redirecting to an URL that does not start + // /// with http:// or https://. + // InvalidProtocolInRedirect, + /// This is a special error case, one that should never be + /// returned! Think of this as a cleaner alternative to calling + /// `unreachable!()` inside the library. If you come across this, + /// please open an issue, and include the string inside this + /// error, as it can be used to locate the problem. + Other(&'static str), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use Error::*; + match self { + #[cfg(feature = "std")] + IoError(err) => write!(f, "{}", err), + InvalidUtf8InBody(err) => write!(f, "{}", err), + + #[cfg(feature = "rustls")] + RustlsCreateConnection(err) => write!(f, "error creating rustls connection: {}", err), + MalformedChunkLength => write!(f, "non-usize chunk length with transfer-encoding: chunked"), + MalformedChunkEnd => write!(f, "chunk did not end after reading the expected amount of bytes"), + MalformedContentLength => write!(f, "non-usize content length"), + HeadersOverflow => write!(f, "the headers' total size surpassed max_headers_size"), + StatusLineOverflow => write!(f, "the status line length surpassed max_status_line_length"), + AddressNotFound => write!(f, "could not resolve host to a socket address"), + RedirectLocationMissing => write!(f, "redirection location header missing"), + InfiniteRedirectionLoop => write!(f, "infinite redirection loop detected"), + TooManyRedirections => write!(f, "too many redirections (over the max)"), + InvalidUtf8InResponse => write!(f, "response contained invalid utf-8 where valid utf-8 was expected"), + HttpsFeatureNotEnabled => write!(f, "request url contains https:// but the https feature is not enabled"), + PunycodeFeatureNotEnabled => write!(f, "non-ascii urls needs to be converted into punycode, and the feature is missing"), + PunycodeConversionFailed => write!(f, "non-ascii url conversion to punycode failed"), + BadProxy => write!(f, "the provided proxy information is malformed"), + BadProxyCreds => write!(f, "the provided proxy credentials are malformed"), + ProxyConnect => write!(f, "could not connect to the proxy server"), + InvalidProxyCreds => write!(f, "the provided proxy credentials are invalid"), + // TODO: Uncomment these two for 3.0 + // InvalidProtocol => write!(f, "the url does not start with http:// or https://"), + // InvalidProtocolInRedirect => write!(f, "got redirected to an absolute url which does not start with http:// or https://"), + Other(msg) => write!(f, "error in bitreq: please open an issue in the bitreq repo, include the following: '{}'", msg), + } + } +} + +#[cfg(feature = "std")] +impl error::Error for Error { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + use Error::*; + match self { + #[cfg(feature = "std")] + IoError(err) => Some(err), + InvalidUtf8InBody(err) => Some(err), + #[cfg(feature = "rustls")] + RustlsCreateConnection(err) => Some(err), + _ => None, + } + } +} + +#[cfg(feature = "std")] +impl From for Error { + fn from(other: io::Error) -> Error { Error::IoError(other) } +} diff --git a/bitreq/src/http_url.rs b/bitreq/src/http_url.rs new file mode 100644 index 00000000..7f5a3950 --- /dev/null +++ b/bitreq/src/http_url.rs @@ -0,0 +1,202 @@ +use core::fmt::{self, Write}; + +use crate::Error; + +#[derive(Clone, Copy, PartialEq)] +pub(crate) enum Port { + ImplicitHttp, + ImplicitHttps, + Explicit(u32), +} + +impl Port { + pub(crate) fn port(self) -> u32 { + match self { + Port::ImplicitHttp => 80, + Port::ImplicitHttps => 443, + Port::Explicit(port) => port, + } + } +} + +/// URL split into its parts. See [RFC 3986 section +/// 3](https://datatracker.ietf.org/doc/html/rfc3986#section-3). Note that the +/// userinfo component is not allowed since [RFC +/// 7230](https://datatracker.ietf.org/doc/html/rfc7230#section-2.7.1). +/// +/// ```text +/// scheme "://" host [ ":" port ] path [ "?" query ] [ "#" fragment ] +/// ``` +#[derive(Clone, PartialEq)] +pub(crate) struct HttpUrl { + /// If scheme is "https", true, if "http", false. + pub(crate) https: bool, + /// `host` + pub(crate) host: String, + /// `[":" port]` + pub(crate) port: Port, + /// `path ["?" query]` including the `?`. + pub(crate) path_and_query: String, + /// `["#" fragment]` without the `#`. + pub(crate) fragment: Option, +} + +impl HttpUrl { + pub(crate) fn parse(url: &str, redirected_from: Option<&HttpUrl>) -> Result { + enum UrlParseStatus { + Host, + Port, + PathAndQuery, + Fragment, + } + + let (url, https) = if let Some(after_protocol) = url.strip_prefix("http://") { + (after_protocol, false) + } else if let Some(after_protocol) = url.strip_prefix("https://") { + (after_protocol, true) + } else { + // TODO: Uncomment this for 3.0 + // return Err(Error::InvalidProtocol); + #[cfg(feature = "std")] + return Err(Error::IoError(std::io::Error::new( + std::io::ErrorKind::Other, + "was redirected to an absolute url with an invalid protocol", + ))); + #[cfg(not(feature = "std"))] + return Err(Error::Other("invalid protocol in url")); + }; + + let mut host = String::new(); + let mut port = String::new(); + let mut resource = String::new(); // At first this is the path and query, after # this becomes fragment. + let mut path_and_query = None; + let mut status = UrlParseStatus::Host; + for c in url.chars() { + match status { + UrlParseStatus::Host => { + match c { + '/' | '?' => { + // Tolerate typos like: www.example.com?some=params + status = UrlParseStatus::PathAndQuery; + resource.push(c); + } + ':' => status = UrlParseStatus::Port, + _ => host.push(c), + } + } + UrlParseStatus::Port => match c { + '/' | '?' => { + status = UrlParseStatus::PathAndQuery; + resource.push(c); + } + _ => port.push(c), + }, + UrlParseStatus::PathAndQuery if c == '#' => { + status = UrlParseStatus::Fragment; + path_and_query = Some(resource); + resource = String::new(); + } + #[cfg(not(feature = "urlencoding"))] + UrlParseStatus::PathAndQuery | UrlParseStatus::Fragment => resource.push(c), + #[cfg(feature = "urlencoding")] + UrlParseStatus::PathAndQuery | UrlParseStatus::Fragment => match c { + // All URL-'safe' characters, plus URL 'special + // characters' like &, #, =, / ,? + '0'..='9' + | 'A'..='Z' + | 'a'..='z' + | '-' + | '.' + | '_' + | '~' + | '&' + | '#' + | '=' + | '/' + | '?' => { + resource.push(c); + } + // There is probably a simpler way to do this, but this + // method avoids any heap allocations (except extending + // `resource`) + _ => { + // Any UTF-8 character can fit in 4 bytes + let mut utf8_buf = [0u8; 4]; + // Bytes fill buffer from the front + c.encode_utf8(&mut utf8_buf); + // Slice disregards the unused portion of the buffer + utf8_buf[..c.len_utf8()].iter().for_each(|byte| { + // Convert byte to URL escape, e.g. %21 for b'!' + let rem = *byte % 16; + let right_char = to_hex_digit(rem); + let left_char = to_hex_digit((*byte - rem) >> 4); + resource.push('%'); + resource.push(left_char); + resource.push(right_char); + }); + } + }, + } + } + let (mut path_and_query, mut fragment) = if let Some(path_and_query) = path_and_query { + (path_and_query, Some(resource)) + } else { + (resource, None) + }; + + // If a redirected resource does not have a fragment, but the original + // URL did, the fragment should be preserved over redirections. See RFC + // 7231 section 7.1.2. + if fragment.is_none() { + if let Some(old_fragment) = redirected_from.and_then(|url| url.fragment.clone()) { + fragment = Some(old_fragment); + } + } + + // Ensure the resource is *something* + if path_and_query.is_empty() { + path_and_query.push('/'); + } + + // Set appropriate port + let port = port.parse::().map(Port::Explicit).unwrap_or_else(|_| { + if https { + Port::ImplicitHttps + } else { + Port::ImplicitHttp + } + }); + + Ok(HttpUrl { https, host, port, path_and_query, fragment }) + } + + /// Writes the `scheme "://" host [ ":" port ]` part to the destination. + pub(crate) fn write_base_url_to(&self, dst: &mut W) -> fmt::Result { + write!(dst, "http{s}://{host}", s = if self.https { "s" } else { "" }, host = &self.host,)?; + if let Port::Explicit(port) = self.port { + write!(dst, ":{}", port)?; + } + Ok(()) + } + + /// Writes the `path [ "?" query ] [ "#" fragment ]` part to the destination. + pub(crate) fn write_resource_to(&self, dst: &mut W) -> fmt::Result { + write!( + dst, + "{path_and_query}{maybe_hash}{maybe_fragment}", + path_and_query = &self.path_and_query, + maybe_hash = if self.fragment.is_some() { "#" } else { "" }, + maybe_fragment = self.fragment.as_deref().unwrap_or(""), + ) + } +} + +// https://github.com/kornelski/rust_urlencoding/blob/a4df8027ab34a86a63f1be727965cf101556403f/src/enc.rs#L130-L136 +// Converts a UTF-8 byte to a single hexadecimal character +#[cfg(feature = "urlencoding")] +fn to_hex_digit(digit: u8) -> char { + match digit { + 0..=9 => (b'0' + digit) as char, + 10..=255 => (b'A' - 10 + digit) as char, + } +} diff --git a/bitreq/src/lib.rs b/bitreq/src/lib.rs new file mode 100644 index 00000000..eda3bbe7 --- /dev/null +++ b/bitreq/src/lib.rs @@ -0,0 +1,247 @@ +//! # Minreq +//! +//! Simple, minimal-dependency HTTP client. The library has a very +//! minimal API, so you'll probably know everything you need to after +//! reading a few examples. +//! +//! Note: as a minimal library, bitreq has been written with the +//! assumption that servers are well-behaved. This means that there is +//! little error-correction for incoming data, which may cause some +//! requests to fail unexpectedly. If you're writing an application or +//! library that connects to servers you can't test beforehand, +//! consider using a more robust library, such as +//! [curl](https://crates.io/crates/curl). +//! +//! # Additional features +//! +//! Since the crate is supposed to be minimal in terms of +//! dependencies, there are no default features, and optional +//! functionality can be enabled by specifying features for `bitreq` +//! dependency in `Cargo.toml`: +//! +//! ```toml +//! [dependencies] +//! bitreq = { version = "2.13.5-alpha", features = ["punycode"] } +//! ``` +//! +//! Below is the list of all available features. +//! +//! ## `https` or `https-rustls` +//! +//! This feature uses the (very good) +//! [`rustls`](https://crates.io/crates/rustls) crate to secure the +//! connection when needed. Note that if this feature is not enabled +//! (and it is not by default), requests to urls that start with +//! `https://` will fail and return a +//! [`HttpsFeatureNotEnabled`](enum.Error.html#variant.HttpsFeatureNotEnabled) +//! error. `https` was the name of this feature until the other https +//! feature variants were added, and is now an alias for +//! `https-rustls`. +//! +//! ## `https-rustls-probe` +//! +//! Like `https-rustls`, but also includes the +//! [`rustls-native-certs`](https://crates.io/crates/rustls-native-certs) +//! crate to auto-detect root certificates installed in common +//! locations. +//! +//! ## `punycode` +//! +//! This feature enables requests to non-ascii domains: the +//! [`punycode`](https://crates.io/crates/punycode) crate is used to +//! convert the non-ascii parts into their punycode representations +//! before making the request. If you try to make a request to 㯙㯜㯙 +//! 㯟.net or i❤.ws for example, with this feature disabled (as it is +//! by default), your request will fail with a +//! [`PunycodeFeatureNotEnabled`](enum.Error.html#variant.PunycodeFeatureNotEnabled) +//! error. +//! +//! ## `async` +//! +//! This feature enables asynchronous HTTP requests using tokio. It provides +//! [`send_async()`](struct.Request.html#method.send_async) and +//! [`send_lazy_async()`](struct.Request.html#method.send_lazy_async) methods +//! that return futures for non-blocking operation. +//! +//! ## `async-https` +//! +//! Like `async`, but also enables asynchronous HTTPS support using tokio-rustls. +//! This feature depends on both `async` and `https-rustls` features. +//! +//! ## `proxy` +//! +//! This feature enables HTTP proxy support. See [Proxy]. +//! +//! ## `urlencoding` +//! +//! This feature enables percent-encoding for the URL resource when +//! creating a request and any subsequently added parameters from +//! [`Request::with_param`]. +//! +//! # Examples +//! +//! ## Get +//! +//! This is a simple example of sending a GET request and printing out +//! the response's body, status code, and reason phrase. The `?` are +//! needed because the server could return invalid UTF-8 in the body, +//! or something could go wrong during the download. +//! +//! ``` +//! # #[cfg(feature = "std")] +//! # fn main() -> Result<(), Box> { +//! let response = bitreq::get("http://example.com").send()?; +//! assert!(response.as_str()?.contains("")); +//! assert_eq!(200, response.status_code); +//! assert_eq!("OK", response.reason_phrase); +//! # Ok(()) } +//! # #[cfg(not(feature = "std"))] +//! # fn main() -> Result<(), Box> { Ok(()) } +//! ``` +//! +//! Note: you could change the `get` function to `head` or `put` or +//! any other HTTP request method: the api is the same for all of +//! them, it just changes what is sent to the server. +//! +//! ## Body (sending) +//! +//! To include a body, add `with_body("")` before +//! `send()`. +//! +//! ``` +//! # #[cfg(feature = "std")] +//! # fn main() -> Result<(), Box> { +//! let response = bitreq::post("http://example.com") +//! .with_body("Foobar") +//! .send()?; +//! # Ok(()) } +//! # #[cfg(not(feature = "std"))] +//! # fn main() -> Result<(), Box> { Ok(()) } +//! ``` +//! +//! ## Headers (sending) +//! +//! To add a header, add `with_header("Key", "Value")` before +//! `send()`. +//! +//! ``` +//! # #[cfg(feature = "std")] +//! # fn main() -> Result<(), Box> { +//! let response = bitreq::get("http://example.com") +//! .with_header("Accept", "text/html") +//! .send()?; +//! # Ok(()) } +//! # #[cfg(not(feature = "std"))] +//! # fn main() -> Result<(), Box> { Ok(()) } +//! ``` +//! +//! ## Headers (receiving) +//! +//! Reading the headers sent by the servers is done via the +//! [`headers`](struct.Response.html#structfield.headers) field of the +//! [`Response`](struct.Response.html). Note: the header field names +//! (that is, the *keys* of the `HashMap`) are all lowercase: this is +//! because the names are case-insensitive according to the spec, and +//! this unifies the casings for easier `get()`ing. +//! +//! ``` +//! # #[cfg(feature = "std")] +//! # fn main() -> Result<(), Box> { +//! let response = bitreq::get("http://example.com").send()?; +//! assert!(response.headers.get("content-type").unwrap().starts_with("text/html")); +//! # Ok(()) } +//! # #[cfg(not(feature = "std"))] +//! # fn main() -> Result<(), Box> { Ok(()) } +//! ``` +//! +//! ## Timeouts +//! +//! To avoid timing out, or limit the request's response time, use +//! `with_timeout(n)` before `send()`. The given value is in seconds. +//! +//! NOTE: There is no timeout by default. +//! +//! ```no_run +//! # #[cfg(feature = "std")] +//! # fn main() -> Result<(), Box> { +//! let response = bitreq::post("http://example.com") +//! .with_timeout(10) +//! .send()?; +//! # Ok(()) } +//! # #[cfg(not(feature = "std"))] +//! # fn main() -> Result<(), Box> { Ok(()) } +//! ``` +//! +//! ## Proxy +//! +//! To use a proxy server, simply create a `Proxy` instance and use +//! `.with_proxy()` on your request. +//! +//! Supported proxy formats are `host:port` and +//! `user:password@proxy:host`. Only HTTP CONNECT proxies are +//! supported at this time. +//! +//! ```no_run +//! # #[cfg(feature = "std")] +//! # fn main() -> Result<(), Box> { +//! #[cfg(feature = "proxy")] +//! { +//! let proxy = bitreq::Proxy::new("localhost:8080")?; +//! let response = bitreq::post("http://example.com") +//! .with_proxy(proxy) +//! .send()?; +//! println!("{}", response.as_str()?); +//! } +//! # Ok(()) } +//! # #[cfg(not(feature = "std"))] +//! # fn main() -> Result<(), Box> { Ok(()) } +//! ``` +//! +//! # Timeouts +//! +//! By default, a request has no timeout. You can change this in two +//! ways: +//! +//! - Use [`with_timeout`](struct.Request.html#method.with_timeout) on +//! your request to set the timeout per-request like so: +//! ```text,ignore +//! bitreq::get("/").with_timeout(8).send(); +//! ``` +//! - Set the environment variable `MINREQ_TIMEOUT` to the desired +//! amount of seconds until timeout. Ie. if you have a program called +//! `foo` that uses bitreq, and you want all the requests made by that +//! program to timeout in 8 seconds, you launch the program like so: +//! ```text,ignore +//! $ MINREQ_TIMEOUT=8 ./foo +//! ``` +//! Or add the following somewhere before the requests in the code. +//! ``` +//! std::env::set_var("MINREQ_TIMEOUT", "8"); +//! ``` +//! If the timeout is set with `with_timeout`, the environment +//! variable will be ignored. + +#![deny(missing_docs)] +// std::io::Error::other was added in 1.74, so occurrences of this lint can't be +// fixed before our MSRV gets that high. +#![allow(clippy::io_other_error)] + +extern crate alloc; + +#[cfg(feature = "std")] +mod connection; +mod error; +#[cfg(feature = "std")] +mod http_url; +#[cfg(feature = "proxy")] +mod proxy; +mod request; +mod response; + +pub use error::*; +#[cfg(feature = "proxy")] +pub use proxy::*; +pub use request::*; +pub use response::Response; +#[cfg(feature = "std")] +pub use response::ResponseLazy; diff --git a/bitreq/src/proxy.rs b/bitreq/src/proxy.rs new file mode 100644 index 00000000..9e11bde8 --- /dev/null +++ b/bitreq/src/proxy.rs @@ -0,0 +1,162 @@ +use base64::engine::general_purpose::STANDARD; +use base64::engine::Engine; + +use crate::error::Error; +use crate::ParsedRequest; + +/// Kind of proxy connection (Basic, Digest, etc) +#[derive(Clone, PartialEq, Eq, Debug)] +pub(crate) enum ProxyKind { + Basic, +} + +/// Proxy configuration. Only HTTP CONNECT proxies are supported (no SOCKS or +/// HTTPS). +/// +/// When credentials are provided, the Basic authentication type is used for +/// Proxy-Authorization. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct Proxy { + pub(crate) server: String, + pub(crate) port: u32, + pub(crate) user: Option, + pub(crate) password: Option, + pub(crate) kind: ProxyKind, +} + +impl Proxy { + fn parse_creds(creds: &str) -> (Option, Option) { + if let Some((user, pass)) = split_once(creds, ":") { + (Some(user.to_string()), Some(pass.to_string())) + } else { + (Some(creds.to_string()), None) + } + } + + fn parse_address(host: &str) -> Result<(String, Option), Error> { + if let Some((host, port)) = split_once(host, ":") { + let port = port.parse::().map_err(|_| Error::BadProxy)?; + Ok((host.to_string(), Some(port))) + } else { + Ok((host.to_string(), None)) + } + } + + /// Creates a new Proxy configuration. + /// + /// Supported proxy format is: + /// + /// ```plaintext + /// [http://][user[:password]@]host[:port] + /// ``` + /// + /// The default port is 8080, to be changed to 1080 in bitreq 3.0. + /// + /// # Example + /// + /// ``` + /// let proxy = bitreq::Proxy::new("user:password@localhost:1080").unwrap(); + /// let request = bitreq::post("http://example.com").with_proxy(proxy); + /// ``` + /// + pub fn new>(proxy: S) -> Result { + let proxy = proxy.as_ref(); + let authority = if let Some((proto, auth)) = split_once(proxy, "://") { + if proto != "http" { + return Err(Error::BadProxy); + } + auth + } else { + proxy + }; + + let ((user, password), host) = if let Some((userinfo, host)) = rsplit_once(authority, "@") { + (Proxy::parse_creds(userinfo), host) + } else { + ((None, None), authority) + }; + + let (host, port) = Proxy::parse_address(host)?; + + Ok(Self { + server: host, + user, + password, + port: port.unwrap_or(8080), + kind: ProxyKind::Basic, + }) + } + + pub(crate) fn connect(&self, proxied_req: &ParsedRequest) -> String { + let authorization = if let Some(user) = &self.user { + match self.kind { + ProxyKind::Basic => { + let creds = if let Some(password) = &self.password { + STANDARD.encode(format!("{}:{}", user, password)) + } else { + STANDARD.encode(user) + }; + format!("Proxy-Authorization: Basic {}\r\n", creds) + } + } + } else { + String::new() + }; + let host = &proxied_req.url.host; + let port = proxied_req.url.port.port(); + format!("CONNECT {}:{} HTTP/1.1\r\n{}\r\n", host, port, authorization) + } + + pub(crate) fn verify_response(response: &[u8]) -> Result<(), Error> { + let response_string = String::from_utf8_lossy(response); + let top_line = response_string.lines().next().ok_or(Error::ProxyConnect)?; + let status_code = top_line.split_whitespace().nth(1).ok_or(Error::BadProxy)?; + + match status_code { + "200" => Ok(()), + "401" | "407" => Err(Error::InvalidProxyCreds), + _ => Err(Error::BadProxy), + } + } +} + +#[allow(clippy::manual_split_once)] +/// Replacement for str::split_once until MSRV is at least 1.52.0. +fn split_once<'a>(string: &'a str, pattern: &str) -> Option<(&'a str, &'a str)> { + let mut parts = string.splitn(2, pattern); + let first = parts.next()?; + let second = parts.next()?; + Some((first, second)) +} + +#[allow(clippy::manual_split_once)] +/// Replacement for str::rsplit_once until MSRV is at least 1.52.0. +fn rsplit_once<'a>(string: &'a str, pattern: &str) -> Option<(&'a str, &'a str)> { + let mut parts = string.rsplitn(2, pattern); + let second = parts.next()?; + let first = parts.next()?; + Some((first, second)) +} + +#[cfg(test)] +mod tests { + use super::Proxy; + + #[test] + fn parse_proxy() { + let proxy = Proxy::new("user:p@ssw0rd@localhost:9999").unwrap(); + assert_eq!(proxy.user, Some(String::from("user"))); + assert_eq!(proxy.password, Some(String::from("p@ssw0rd"))); + assert_eq!(proxy.server, String::from("localhost")); + assert_eq!(proxy.port, 9999); + } + + #[test] + fn parse_regular_proxy_with_protocol() { + let proxy = Proxy::new("http://localhost:1080").unwrap(); + assert_eq!(proxy.user, None); + assert_eq!(proxy.password, None); + assert_eq!(proxy.server, String::from("localhost")); + assert_eq!(proxy.port, 1080); + } +} diff --git a/bitreq/src/request.rs b/bitreq/src/request.rs new file mode 100644 index 00000000..6c55ac09 --- /dev/null +++ b/bitreq/src/request.rs @@ -0,0 +1,618 @@ +use alloc::collections::BTreeMap; +use core::fmt; +#[cfg(feature = "std")] +use core::fmt::Write; + +#[cfg(feature = "async")] +use crate::connection::AsyncConnection; +#[cfg(feature = "std")] +use crate::connection::Connection; +#[cfg(feature = "std")] +use crate::http_url::{HttpUrl, Port}; +#[cfg(feature = "proxy")] +use crate::proxy::Proxy; +#[cfg(feature = "std")] +use crate::{Error, Response, ResponseLazy}; + +/// A URL type for requests. +pub type URL = String; + +/// An HTTP request method. +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum Method { + /// The GET method + Get, + /// The HEAD method + Head, + /// The POST method + Post, + /// The PUT method + Put, + /// The DELETE method + Delete, + /// The CONNECT method + Connect, + /// The OPTIONS method + Options, + /// The TRACE method + Trace, + /// The PATCH method + Patch, + /// A custom method, use with care: the string will be embedded in + /// your request as-is. + Custom(String), +} + +impl fmt::Display for Method { + /// Formats the Method to the form in the HTTP request, + /// ie. Method::Get -> "GET", Method::Post -> "POST", etc. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + Method::Get => write!(f, "GET"), + Method::Head => write!(f, "HEAD"), + Method::Post => write!(f, "POST"), + Method::Put => write!(f, "PUT"), + Method::Delete => write!(f, "DELETE"), + Method::Connect => write!(f, "CONNECT"), + Method::Options => write!(f, "OPTIONS"), + Method::Trace => write!(f, "TRACE"), + Method::Patch => write!(f, "PATCH"), + Method::Custom(ref s) => write!(f, "{}", s), + } + } +} + +/// An HTTP request. +/// +/// Generally created by the [`bitreq::get`](fn.get.html)-style +/// functions, corresponding to the HTTP method we want to use. +/// +/// # Example +/// +/// ``` +/// let request = bitreq::post("http://example.com"); +/// ``` +/// +/// After creating the request, you would generally call +/// [`send`](struct.Request.html#method.send) or +/// [`send_lazy`](struct.Request.html#method.send_lazy) on it, as it +/// doesn't do much on its own. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct Request { + pub(crate) method: Method, + url: URL, + params: String, + headers: BTreeMap, + body: Option>, + pub(crate) timeout: Option, + pub(crate) max_headers_size: Option, + pub(crate) max_status_line_len: Option, + max_redirects: usize, + #[cfg(feature = "proxy")] + pub(crate) proxy: Option, +} + +impl Request { + /// Creates a new HTTP `Request`. + /// + /// This is only the request's data, it is not sent yet. For + /// sending the request, see [`send`](struct.Request.html#method.send). + /// + /// If `urlencoding` is not enabled, it is the responsibility of the + /// user to ensure there are no illegal characters in the URL. + /// + /// If `urlencoding` is enabled, the resource part of the URL will be + /// encoded. Any URL special characters (e.g. &, #, =) are not encoded + /// as they are assumed to be meaningful parameters etc. + pub fn new>(method: Method, url: T) -> Request { + Request { + method, + url: url.into(), + params: String::new(), + headers: BTreeMap::new(), + body: None, + timeout: None, + max_headers_size: None, + max_status_line_len: None, + max_redirects: 100, + #[cfg(feature = "proxy")] + proxy: None, + } + } + + /// Add headers to the request this is called on. Use this + /// function to add headers to your requests. + pub fn with_headers(mut self, headers: T) -> Request + where + T: IntoIterator, + K: Into, + V: Into, + { + let headers = headers.into_iter().map(|(k, v)| (k.into(), v.into())); + self.headers.extend(headers); + self + } + + /// Adds a header to the request this is called on. Use this + /// function to add headers to your requests. + pub fn with_header, U: Into>(mut self, key: T, value: U) -> Request { + self.headers.insert(key.into(), value.into()); + self + } + + /// Sets the request body. + pub fn with_body>>(mut self, body: T) -> Request { + let body = body.into(); + let body_length = body.len(); + self.body = Some(body); + self.with_header("Content-Length", format!("{}", body_length)) + } + + /// Adds given key and value as query parameter to request url + /// (resource). + /// + /// If `urlencoding` is not enabled, it is the responsibility + /// of the user to ensure there are no illegal characters in the + /// key or value. + /// + /// If `urlencoding` is enabled, the key and value are both encoded. + pub fn with_param, U: Into>(mut self, key: T, value: U) -> Request { + let key = key.into(); + #[cfg(feature = "urlencoding")] + let key = urlencoding::encode(&key); + let value = value.into(); + #[cfg(feature = "urlencoding")] + let value = urlencoding::encode(&value); + + if !self.params.is_empty() { + self.params.push('&'); + } + self.params.push_str(&key); + self.params.push('='); + self.params.push_str(&value); + self + } + + /// Sets the request timeout in seconds. + pub fn with_timeout(mut self, timeout: u64) -> Request { + self.timeout = Some(timeout); + self + } + + /// Sets the max redirects we follow until giving up. 100 by + /// default. + /// + /// Warning: setting this to a very high number, such as 1000, may + /// cause a stack overflow if that many redirects are followed. If + /// you have a use for so many redirects that the stack overflow + /// becomes a problem, please open an issue. + pub fn with_max_redirects(mut self, max_redirects: usize) -> Request { + self.max_redirects = max_redirects; + self + } + + /// Sets the maximum size of all the headers this request will + /// accept. + /// + /// If this limit is passed, the request will close the connection + /// and return an [Error::HeadersOverflow] error. + /// + /// The maximum length is counted in bytes, including line-endings + /// and other whitespace. Both normal and trailing headers count + /// towards this cap. + /// + /// `None` disables the cap, and may cause the program to use any + /// amount of memory if the server responds with a lot of headers + /// (or an infinite amount). The default is None, so setting this + /// manually is recommended when talking to untrusted servers. + pub fn with_max_headers_size>>(mut self, max_headers_size: S) -> Request { + self.max_headers_size = max_headers_size.into(); + self + } + + /// Sets the maximum length of the status line this request will + /// accept. + /// + /// If this limit is passed, the request will close the connection + /// and return an [Error::StatusLineOverflow] error. + /// + /// The maximum length is counted in bytes, including the + /// line-ending `\r\n`. + /// + /// `None` disables the cap, and may cause the program to use any + /// amount of memory if the server responds with a long (or + /// infinite) status line. The default is None, so setting this + /// manually is recommended when talking to untrusted servers. + pub fn with_max_status_line_length>>( + mut self, + max_status_line_len: S, + ) -> Request { + self.max_status_line_len = max_status_line_len.into(); + self + } + + /// Sets the proxy to use. + #[cfg(feature = "proxy")] + pub fn with_proxy(mut self, proxy: Proxy) -> Request { + self.proxy = Some(proxy); + self + } + + /// Sends this request to the host. + /// + /// # Errors + /// + /// Returns `Err` if we run into an error while sending the + /// request, or receiving/parsing the response. The specific error + /// is described in the `Err`, and it can be any + /// [`bitreq::Error`](enum.Error.html) except + /// [`InvalidUtf8InBody`](enum.Error.html#variant.InvalidUtf8InBody). + #[cfg(feature = "std")] + pub fn send(self) -> Result { + let parsed_request = ParsedRequest::new(self)?; + if parsed_request.url.https { + #[cfg(feature = "rustls")] + { + let is_head = parsed_request.config.method == Method::Head; + let response = Connection::new(parsed_request).send_https()?; + Response::create(response, is_head) + } + #[cfg(not(feature = "rustls"))] + { + Err(Error::HttpsFeatureNotEnabled) + } + } else { + let is_head = parsed_request.config.method == Method::Head; + let response = Connection::new(parsed_request).send()?; + Response::create(response, is_head) + } + } + + /// Sends this request to the host, loaded lazily. + /// + /// # Errors + /// + /// See [`send`](struct.Request.html#method.send). + #[cfg(feature = "std")] + pub fn send_lazy(self) -> Result { + let parsed_request = ParsedRequest::new(self)?; + if parsed_request.url.https { + #[cfg(feature = "rustls")] + { + Connection::new(parsed_request).send_https() + } + #[cfg(not(feature = "rustls"))] + { + Err(Error::HttpsFeatureNotEnabled) + } + } else { + Connection::new(parsed_request).send() + } + } + + /// Sends this request to the host asynchronously. + /// + /// # Errors + /// + /// Returns `Err` if we run into an error while sending the + /// request, or receiving/parsing the response. The specific error + /// is described in the `Err`, and it can be any + /// [`minreq::Error`](enum.Error.html) except + /// [`InvalidUtf8InBody`](enum.Error.html#variant.InvalidUtf8InBody). + #[cfg(feature = "async")] + pub async fn send_async(self) -> Result { + let parsed_request = ParsedRequest::new(self)?; + if parsed_request.url.https { + #[cfg(feature = "async-https")] + { + let is_head = parsed_request.config.method == Method::Head; + let response = AsyncConnection::new(parsed_request).send_https().await?; + Response::create(response, is_head) + } + #[cfg(not(feature = "async-https"))] + { + Err(Error::HttpsFeatureNotEnabled) + } + } else { + let is_head = parsed_request.config.method == Method::Head; + let response = AsyncConnection::new(parsed_request).send().await?; + Response::create(response, is_head) + } + } + + /// Sends this request to the host asynchronously, loaded lazily. + /// + /// # Errors + /// + /// See [`send_async`](struct.Request.html#method.send_async). + #[cfg(feature = "async")] + pub async fn send_lazy_async(self) -> Result { + let parsed_request = ParsedRequest::new(self)?; + if parsed_request.url.https { + #[cfg(feature = "async-https")] + { + AsyncConnection::new(parsed_request).send_https().await + } + #[cfg(not(feature = "async-https"))] + { + Err(Error::HttpsFeatureNotEnabled) + } + } else { + AsyncConnection::new(parsed_request).send().await + } + } +} + +#[cfg(feature = "std")] +pub(crate) struct ParsedRequest { + pub(crate) url: HttpUrl, + pub(crate) redirects: Vec, + pub(crate) config: Request, +} + +#[cfg(feature = "std")] +impl ParsedRequest { + #[allow(unused_mut)] + fn new(mut config: Request) -> Result { + let mut url = HttpUrl::parse(&config.url, None)?; + + if !config.params.is_empty() { + if url.path_and_query.contains('?') { + url.path_and_query.push('&'); + } else { + url.path_and_query.push('?'); + } + url.path_and_query.push_str(&config.params); + } + + #[cfg(all(feature = "proxy", feature = "std"))] + // Set default proxy from environment variables + // + // Curl documentation: https://everything.curl.dev/usingcurl/proxies/env + // + // Accepted variables are `http_proxy`, `https_proxy`, `HTTPS_PROXY`, `ALL_PROXY` + // + // Note: https://everything.curl.dev/usingcurl/proxies/env#http_proxy-in-lower-case-only + if config.proxy.is_none() { + // Set HTTP proxies if request's protocol is HTTPS and they're given + if url.https { + if let Ok(proxy) = + std::env::var("https_proxy").map_err(|_| std::env::var("HTTPS_PROXY")) + { + if let Ok(proxy) = Proxy::new(proxy) { + config.proxy = Some(proxy); + } + } + } + // Set HTTP proxies if request's protocol is HTTP and they're given + else if let Ok(proxy) = std::env::var("http_proxy") { + if let Ok(proxy) = Proxy::new(proxy) { + config.proxy = Some(proxy); + } + } + // Set any given proxies if neither of HTTP/HTTPS were given + else if let Ok(proxy) = + std::env::var("all_proxy").map_err(|_| std::env::var("ALL_PROXY")) + { + if let Ok(proxy) = Proxy::new(proxy) { + config.proxy = Some(proxy); + } + } + } + + Ok(ParsedRequest { url, redirects: Vec::new(), config }) + } + + fn get_http_head(&self) -> String { + let mut http = String::with_capacity(32); + + // NOTE: As of 2.10.0, the fragment is intentionally left out of the request, based on: + // - [RFC 3986 section 3.5](https://datatracker.ietf.org/doc/html/rfc3986#section-3.5): + // "...the fragment identifier is not used in the scheme-specific + // processing of a URI; instead, the fragment identifier is separated + // from the rest of the URI prior to a dereference..." + // - [RFC 7231 section 9.5](https://datatracker.ietf.org/doc/html/rfc7231#section-9.5): + // "Although fragment identifiers used within URI references are not + // sent in requests..." + + // Add the request line and the "Host" header + write!( + http, + "{} {} HTTP/1.1\r\nHost: {}", + self.config.method, self.url.path_and_query, self.url.host + ) + .unwrap(); + if let Port::Explicit(port) = self.url.port { + write!(http, ":{}", port).unwrap(); + } + http += "\r\n"; + + // Add other headers + for (k, v) in &self.config.headers { + write!(http, "{}: {}\r\n", k, v).unwrap(); + } + + if self.config.method == Method::Post + || self.config.method == Method::Put + || self.config.method == Method::Patch + { + let not_length = |key: &String| { + let key = key.to_lowercase(); + key != "content-length" && key != "transfer-encoding" + }; + if self.config.headers.keys().all(not_length) { + // A user agent SHOULD send a Content-Length in a request message when no Transfer-Encoding + // is sent and the request method defines a meaning for an enclosed payload body. + // refer: https://tools.ietf.org/html/rfc7230#section-3.3.2 + + // A client MUST NOT send a message body in a TRACE request. + // refer: https://tools.ietf.org/html/rfc7231#section-4.3.8 + // similar line found for GET, HEAD, CONNECT and DELETE. + + http += "Content-Length: 0\r\n"; + } + } + + http += "\r\n"; + http + } + + /// Returns the HTTP request as bytes, ready to be sent to + /// the server. + pub(crate) fn as_bytes(&self) -> Vec { + let mut head = self.get_http_head().into_bytes(); + if let Some(body) = &self.config.body { + head.extend(body); + } + head + } + + /// Returns the redirected version of this Request, unless an + /// infinite redirection loop was detected, or the redirection + /// limit was reached. + pub(crate) fn redirect_to(&mut self, url: &str) -> Result<(), Error> { + if url.contains("://") { + let mut url = HttpUrl::parse(url, Some(&self.url)).map_err(|_| { + // TODO: Uncomment this for 3.0 + // Error::InvalidProtocolInRedirect + #[cfg(feature = "std")] + { + Error::IoError(std::io::Error::new( + std::io::ErrorKind::Other, + "was redirected to an absolute url with an invalid protocol", + )) + } + #[cfg(not(feature = "std"))] + { + Error::Other("invalid protocol in redirect") + } + })?; + std::mem::swap(&mut url, &mut self.url); + self.redirects.push(url); + } else { + // The url does not have the protocol part, assuming it's + // a relative resource. + let mut absolute_url = String::new(); + self.url.write_base_url_to(&mut absolute_url).unwrap(); + absolute_url.push_str(url); + let mut url = HttpUrl::parse(&absolute_url, Some(&self.url))?; + std::mem::swap(&mut url, &mut self.url); + self.redirects.push(url); + } + + if self.redirects.len() > self.config.max_redirects { + Err(Error::TooManyRedirections) + } else if self.redirects.iter().any(|redirect_url| redirect_url == &self.url) { + Err(Error::InfiniteRedirectionLoop) + } else { + Ok(()) + } + } +} + +/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to +/// [Method::Get](enum.Method.html). +pub fn get>(url: T) -> Request { Request::new(Method::Get, url) } + +/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to +/// [Method::Head](enum.Method.html). +pub fn head>(url: T) -> Request { Request::new(Method::Head, url) } + +/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to +/// [Method::Post](enum.Method.html). +pub fn post>(url: T) -> Request { Request::new(Method::Post, url) } + +/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to +/// [Method::Put](enum.Method.html). +pub fn put>(url: T) -> Request { Request::new(Method::Put, url) } + +/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to +/// [Method::Delete](enum.Method.html). +pub fn delete>(url: T) -> Request { Request::new(Method::Delete, url) } + +/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to +/// [Method::Connect](enum.Method.html). +pub fn connect>(url: T) -> Request { Request::new(Method::Connect, url) } + +/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to +/// [Method::Options](enum.Method.html). +pub fn options>(url: T) -> Request { Request::new(Method::Options, url) } + +/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to +/// [Method::Trace](enum.Method.html). +pub fn trace>(url: T) -> Request { Request::new(Method::Trace, url) } + +/// Alias for [Request::new](struct.Request.html#method.new) with `method` set to +/// [Method::Patch](enum.Method.html). +pub fn patch>(url: T) -> Request { Request::new(Method::Patch, url) } + +#[cfg(test)] +#[cfg(feature = "std")] +mod parsing_tests { + + use alloc::collections::BTreeMap; + + use super::{get, ParsedRequest}; + + #[test] + fn test_headers() { + let mut headers = BTreeMap::new(); + headers.insert("foo".to_string(), "bar".to_string()); + headers.insert("foo".to_string(), "baz".to_string()); + + let req = get("http://www.example.org/test/res").with_headers(headers.clone()); + + assert_eq!(req.headers, headers); + } + + #[test] + fn test_multiple_params() { + let req = get("http://www.example.org/test/res") + .with_param("foo", "bar") + .with_param("asd", "qwe"); + let req = ParsedRequest::new(req).unwrap(); + assert_eq!(&req.url.path_and_query, "/test/res?foo=bar&asd=qwe"); + } + + #[test] + fn test_domain() { + let req = get("http://www.example.org/test/res").with_param("foo", "bar"); + let req = ParsedRequest::new(req).unwrap(); + assert_eq!(&req.url.host, "www.example.org"); + } + + #[test] + fn test_protocol() { + let req = + ParsedRequest::new(get("http://www.example.org/").with_param("foo", "bar")).unwrap(); + assert!(!req.url.https); + let req = + ParsedRequest::new(get("https://www.example.org/").with_param("foo", "bar")).unwrap(); + assert!(req.url.https); + } +} + +#[cfg(all(test, feature = "urlencoding"))] +mod encoding_tests { + use super::{get, ParsedRequest}; + + #[test] + fn test_with_param() { + let req = get("http://www.example.org").with_param("foo", "bar"); + let req = ParsedRequest::new(req).unwrap(); + assert_eq!(&req.url.path_and_query, "/?foo=bar"); + + let req = get("http://www.example.org").with_param("ówò", "what's this? 👀"); + let req = ParsedRequest::new(req).unwrap(); + assert_eq!(&req.url.path_and_query, "/?%C3%B3w%C3%B2=what%27s%20this%3F%20%F0%9F%91%80"); + } + + #[test] + fn test_on_creation() { + let req = ParsedRequest::new(get("http://www.example.org/?foo=bar#baz")).unwrap(); + assert_eq!(&req.url.path_and_query, "/?foo=bar"); + + let req = ParsedRequest::new(get("http://www.example.org/?ówò=what's this? 👀")).unwrap(); + assert_eq!(&req.url.path_and_query, "/?%C3%B3w%C3%B2=what%27s%20this?%20%F0%9F%91%80"); + } +} diff --git a/bitreq/src/response.rs b/bitreq/src/response.rs new file mode 100644 index 00000000..d9f13e89 --- /dev/null +++ b/bitreq/src/response.rs @@ -0,0 +1,574 @@ +use alloc::collections::BTreeMap; +use core::str; +#[cfg(feature = "std")] +use std::io::{self, BufReader, Bytes, Read}; + +#[cfg(feature = "std")] +use crate::connection::HttpStream; +use crate::Error; + +#[cfg(feature = "std")] +const BACKING_READ_BUFFER_LENGTH: usize = 16 * 1024; +#[cfg(feature = "std")] +const MAX_CONTENT_LENGTH: usize = 16 * 1024; + +/// An HTTP response. +/// +/// Returned by [`Request::send`](struct.Request.html#method.send). +/// +/// # Example +/// +/// ```no_run +/// # #[cfg(feature = "std")] +/// # fn main() -> Result<(), bitreq::Error> { +/// let response = bitreq::get("http://example.com").send()?; +/// println!("{}", response.as_str()?); +/// # Ok(()) } +/// # #[cfg(not(feature = "std"))] +/// # fn main() -> Result<(), Box> { Ok(()) } +/// ``` +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct Response { + /// The status code of the response, eg. 404. + pub status_code: i32, + /// The reason phrase of the response, eg. "Not Found". + pub reason_phrase: String, + /// The headers of the response. The header field names (the + /// keys) are all lowercase. + pub headers: BTreeMap, + /// The URL of the resource returned in this response. May differ from the + /// request URL if it was redirected or typo corrections were applied (e.g. + /// would be corrected to + /// ). + pub url: String, + + body: Vec, +} + +impl Response { + #[cfg(feature = "std")] + pub(crate) fn create(mut parent: ResponseLazy, is_head: bool) -> Result { + let mut body = Vec::new(); + if !is_head && parent.status_code != 204 && parent.status_code != 304 { + for byte in &mut parent { + let (byte, length) = byte?; + body.reserve(length); + body.push(byte); + } + } + + let ResponseLazy { status_code, reason_phrase, headers, url, .. } = parent; + + Ok(Response { status_code, reason_phrase, headers, url, body }) + } + + /// Returns the body as an `&str`. + /// + /// # Errors + /// + /// Returns + /// [`InvalidUtf8InBody`](enum.Error.html#variant.InvalidUtf8InBody) + /// if the body is not UTF-8, with a description as to why the + /// provided slice is not UTF-8. + /// + /// # Example + /// + /// ```no_run + /// # #[cfg(feature = "std")] + /// # fn main() -> Result<(), Box> { + /// # let url = "http://example.org/"; + /// let response = bitreq::get(url).send()?; + /// println!("{}", response.as_str()?); + /// # Ok(()) } + /// # #[cfg(not(feature = "std"))] + /// # fn main() -> Result<(), Box> { Ok(()) } + /// ``` + pub fn as_str(&self) -> Result<&str, Error> { + match str::from_utf8(&self.body) { + Ok(s) => Ok(s), + Err(err) => Err(Error::InvalidUtf8InBody(err)), + } + } + + /// Returns a reference to the contained bytes of the body. If you + /// want the `Vec` itself, use + /// [`into_bytes()`](#method.into_bytes) instead. + /// + /// # Example + /// + /// ```no_run + /// # #[cfg(feature = "std")] + /// # fn main() -> Result<(), Box> { + /// # let url = "http://example.org/"; + /// let response = bitreq::get(url).send()?; + /// println!("{:?}", response.as_bytes()); + /// # Ok(()) } + /// # #[cfg(not(feature = "std"))] + /// # fn main() -> Result<(), Box> { Ok(()) } + /// ``` + pub fn as_bytes(&self) -> &[u8] { &self.body } + + /// Turns the `Response` into the inner `Vec`, the bytes that + /// make up the response's body. If you just need a `&[u8]`, use + /// [`as_bytes()`](#method.as_bytes) instead. + /// + /// # Example + /// + /// ```no_run + /// # #[cfg(feature = "std")] + /// # fn main() -> Result<(), Box> { + /// # let url = "http://example.org/"; + /// let response = bitreq::get(url).send()?; + /// println!("{:?}", response.into_bytes()); + /// // This would error, as into_bytes consumes the Response: + /// // let x = response.status_code; + /// # Ok(()) } + /// # #[cfg(not(feature = "std"))] + /// # fn main() -> Result<(), Box> { Ok(()) } + /// ``` + pub fn into_bytes(self) -> Vec { self.body } +} + +/// An HTTP response, which is loaded lazily. +/// +/// In comparison to [`Response`](struct.Response.html), this is +/// returned from +/// [`send_lazy()`](struct.Request.html#method.send_lazy), where as +/// [`Response`](struct.Response.html) is returned from +/// [`send()`](struct.Request.html#method.send). +/// +/// In practice, "lazy loading" means that the bytes are only loaded +/// as you iterate through them. The bytes are provided in the form of +/// a `Result<(u8, usize), bitreq::Error>`, as the reading operation +/// can fail in various ways. The `u8` is the actual byte that was +/// read, and `usize` is how many bytes we are expecting to read in +/// the future (including this byte). Note, however, that the `usize` +/// can change, particularly when the `Transfer-Encoding` is +/// `chunked`: then it will reflect how many bytes are left of the +/// current chunk. The expected size is capped at 16 KiB to avoid +/// server-side DoS attacks targeted at clients accidentally reserving +/// too much memory. +/// +/// # Example +/// ```no_run +/// // This is how the normal Response works behind the scenes, and +/// // how you might use ResponseLazy. +/// # #[cfg(feature = "std")] +/// # fn main() -> Result<(), bitreq::Error> { +/// let response = bitreq::get("http://example.com").send_lazy()?; +/// let mut vec = Vec::new(); +/// for result in response { +/// let (byte, length) = result?; +/// vec.reserve(length); +/// vec.push(byte); +/// } +/// # Ok(()) +/// # } +/// # #[cfg(not(feature = "std"))] +/// # fn main() -> Result<(), Box> { Ok(()) } +/// ``` +#[cfg(feature = "std")] +pub struct ResponseLazy { + /// The status code of the response, eg. 404. + pub status_code: i32, + /// The reason phrase of the response, eg. "Not Found". + pub reason_phrase: String, + /// The headers of the response. The header field names (the + /// keys) are all lowercase. + pub headers: BTreeMap, + /// The URL of the resource returned in this response. May differ from the + /// request URL if it was redirected or typo corrections were applied (e.g. + /// would be corrected to + /// ). + pub url: String, + + stream: HttpStreamBytes, + state: HttpStreamState, + max_trailing_headers_size: Option, +} + +#[cfg(feature = "std")] +type HttpStreamBytes = Bytes>; + +#[cfg(feature = "std")] +impl ResponseLazy { + pub(crate) fn from_stream( + stream: HttpStream, + max_headers_size: Option, + max_status_line_len: Option, + ) -> Result { + let mut stream = BufReader::with_capacity(BACKING_READ_BUFFER_LENGTH, stream).bytes(); + let ResponseMetadata { + status_code, + reason_phrase, + headers, + state, + max_trailing_headers_size, + } = read_metadata(&mut stream, max_headers_size, max_status_line_len)?; + + Ok(ResponseLazy { + status_code, + reason_phrase, + headers, + url: String::new(), + stream, + state, + max_trailing_headers_size, + }) + } +} + +#[cfg(feature = "std")] +impl Iterator for ResponseLazy { + type Item = Result<(u8, usize), Error>; + + fn next(&mut self) -> Option { + use HttpStreamState::*; + match self.state { + EndOnClose => read_until_closed(&mut self.stream), + ContentLength(ref mut length) => read_with_content_length(&mut self.stream, length), + Chunked(ref mut expecting_chunks, ref mut length, ref mut content_length) => + read_chunked( + &mut self.stream, + &mut self.headers, + expecting_chunks, + length, + content_length, + self.max_trailing_headers_size, + ), + } + } +} + +#[cfg(feature = "std")] +impl Read for ResponseLazy { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let mut index = 0; + for res in self { + // there is no use for the estimated length in the read implementation + // so it is ignored. + let (byte, _) = res.map_err(|e| match e { + Error::IoError(e) => e, + _ => io::Error::new(io::ErrorKind::Other, e), + })?; + + buf[index] = byte; + index += 1; + + // if the buffer is full, it should stop reading + if index >= buf.len() { + break; + } + } + + // index of the next byte is the number of bytes thats have been read + Ok(index) + } +} + +#[cfg(feature = "std")] +fn read_until_closed(bytes: &mut HttpStreamBytes) -> Option<::Item> { + if let Some(byte) = bytes.next() { + match byte { + Ok(byte) => Some(Ok((byte, 1))), + Err(err) => Some(Err(Error::IoError(err))), + } + } else { + None + } +} + +#[cfg(feature = "std")] +fn read_with_content_length( + bytes: &mut HttpStreamBytes, + content_length: &mut usize, +) -> Option<::Item> { + if *content_length > 0 { + *content_length -= 1; + + if let Some(byte) = bytes.next() { + match byte { + // Cap Content-Length to 16KiB, to avoid out-of-memory issues. + Ok(byte) => return Some(Ok((byte, (*content_length).min(MAX_CONTENT_LENGTH) + 1))), + Err(err) => return Some(Err(Error::IoError(err))), + } + } + } + None +} + +#[cfg(feature = "std")] +fn read_trailers( + bytes: &mut HttpStreamBytes, + headers: &mut BTreeMap, + mut max_headers_size: Option, +) -> Result<(), Error> { + loop { + let trailer_line = read_line(bytes, max_headers_size, Error::HeadersOverflow)?; + if let Some(ref mut max_headers_size) = max_headers_size { + *max_headers_size -= trailer_line.len() + 2; + } + if let Some((header, value)) = parse_header(trailer_line) { + headers.insert(header, value); + } else { + break; + } + } + Ok(()) +} + +#[cfg(feature = "std")] +fn read_chunked( + bytes: &mut HttpStreamBytes, + headers: &mut BTreeMap, + expecting_more_chunks: &mut bool, + chunk_length: &mut usize, + content_length: &mut usize, + max_trailing_headers_size: Option, +) -> Option<::Item> { + if !*expecting_more_chunks && *chunk_length == 0 { + return None; + } + + if *chunk_length == 0 { + // Max length of the chunk length line is 1KB: not too long to + // take up much memory, long enough to tolerate some chunk + // extensions (which are ignored). + + // Get the size of the next chunk + let length_line = match read_line(bytes, Some(1024), Error::MalformedChunkLength) { + Ok(line) => line, + Err(err) => return Some(Err(err)), + }; + + // Note: the trim() and check for empty lines shouldn't be + // needed according to the RFC, but we might as well, it's a + // small change and it fixes a few servers. + let incoming_length = if length_line.is_empty() { + 0 + } else { + let length = if let Some(i) = length_line.find(';') { + length_line[..i].trim() + } else { + length_line.trim() + }; + match usize::from_str_radix(length, 16) { + Ok(length) => length, + Err(_) => return Some(Err(Error::MalformedChunkLength)), + } + }; + + if incoming_length == 0 { + if let Err(err) = read_trailers(bytes, headers, max_trailing_headers_size) { + return Some(Err(err)); + } + + *expecting_more_chunks = false; + headers.insert("content-length".to_string(), (*content_length).to_string()); + headers.remove("transfer-encoding"); + return None; + } + *chunk_length = incoming_length; + *content_length += incoming_length; + } + + if *chunk_length > 0 { + *chunk_length -= 1; + if let Some(byte) = bytes.next() { + match byte { + Ok(byte) => { + // If we're at the end of the chunk... + if *chunk_length == 0 { + //...read the trailing \r\n of the chunk, and + // possibly return an error instead. + + // TODO: Maybe this could be written in a way + // that doesn't discard the last ok byte if + // the \r\n reading fails? + if let Err(err) = read_line(bytes, Some(2), Error::MalformedChunkEnd) { + return Some(Err(err)); + } + } + + return Some(Ok((byte, (*chunk_length).min(MAX_CONTENT_LENGTH) + 1))); + } + Err(err) => return Some(Err(Error::IoError(err))), + } + } + } + + None +} + +#[cfg(feature = "std")] +enum HttpStreamState { + // No Content-Length, and Transfer-Encoding != chunked, so we just + // read unti lthe server closes the connection (this should be the + // fallback, if I read the rfc right). + EndOnClose, + // Content-Length was specified, read that amount of bytes + ContentLength(usize), + // Transfer-Encoding == chunked, so we need to save two pieces of + // information: are we expecting more chunks, how much is there + // left of the current chunk, and how much have we read? The last + // number is needed in order to provide an accurate Content-Length + // header after loading all the bytes. + Chunked(bool, usize, usize), +} + +// This struct is just used in the Response and ResponseLazy +// constructors, but not in their structs, for api-cleanliness +// reasons. (Eg. response.status_code is much cleaner than +// response.meta.status_code or similar.) +#[cfg(feature = "std")] +struct ResponseMetadata { + status_code: i32, + reason_phrase: String, + headers: BTreeMap, + state: HttpStreamState, + max_trailing_headers_size: Option, +} + +#[cfg(feature = "std")] +fn read_metadata( + stream: &mut HttpStreamBytes, + mut max_headers_size: Option, + max_status_line_len: Option, +) -> Result { + let line = read_line(stream, max_status_line_len, Error::StatusLineOverflow)?; + let (status_code, reason_phrase) = parse_status_line(&line); + + let mut headers = BTreeMap::new(); + loop { + let line = read_line(stream, max_headers_size, Error::HeadersOverflow)?; + if line.is_empty() { + // Body starts here + break; + } + if let Some(ref mut max_headers_size) = max_headers_size { + *max_headers_size -= line.len() + 2; + } + if let Some(header) = parse_header(line) { + headers.insert(header.0, header.1); + } + } + + let mut chunked = false; + let mut content_length = None; + for (header, value) in &headers { + // Handle the Transfer-Encoding header + if header.to_lowercase().trim() == "transfer-encoding" + && value.to_lowercase().trim() == "chunked" + { + chunked = true; + } + + // Handle the Content-Length header + if header.to_lowercase().trim() == "content-length" { + match str::parse::(value.trim()) { + Ok(length) => content_length = Some(length), + Err(_) => return Err(Error::MalformedContentLength), + } + } + } + + let state = if chunked { + HttpStreamState::Chunked(true, 0, 0) + } else if let Some(length) = content_length { + HttpStreamState::ContentLength(length) + } else { + HttpStreamState::EndOnClose + }; + + Ok(ResponseMetadata { + status_code, + reason_phrase, + headers, + state, + max_trailing_headers_size: max_headers_size, + }) +} + +#[cfg(feature = "std")] +fn read_line( + stream: &mut HttpStreamBytes, + max_len: Option, + overflow_error: Error, +) -> Result { + let mut bytes = Vec::with_capacity(32); + for byte in stream { + match byte { + Ok(byte) => { + if let Some(max_len) = max_len { + if bytes.len() >= max_len { + return Err(overflow_error); + } + } + if byte == b'\n' { + if let Some(b'\r') = bytes.last() { + bytes.pop(); + } + break; + } else { + bytes.push(byte); + } + } + Err(err) => return Err(Error::IoError(err)), + } + } + String::from_utf8(bytes).map_err(|_error| Error::InvalidUtf8InResponse) +} + +#[cfg(feature = "std")] +fn parse_status_line(line: &str) -> (i32, String) { + // sample status line format + // HTTP/1.1 200 OK + let mut status_code = String::with_capacity(3); + let mut reason_phrase = String::with_capacity(2); + + let mut spaces = 0; + + for c in line.chars() { + if spaces >= 2 { + reason_phrase.push(c); + } + + if c == ' ' { + spaces += 1; + } else if spaces == 1 { + status_code.push(c); + } + } + + if let Ok(status_code) = status_code.parse::() { + return (status_code, reason_phrase); + } + + (503, "Server did not provide a status line".to_string()) +} + +#[cfg(feature = "std")] +fn parse_header(mut line: String) -> Option<(String, String)> { + if let Some(location) = line.find(':') { + // Trim the first character of the header if it is a space, + // otherwise return everything after the ':'. This should + // preserve the behavior in versions <=2.0.1 in most cases + // (namely, ones where it was valid), where the first + // character after ':' was always cut off. + let value = if let Some(sp) = line.get(location + 1..location + 2) { + if sp == " " { + line[location + 2..].to_string() + } else { + line[location + 1..].to_string() + } + } else { + line[location + 1..].to_string() + }; + + line.truncate(location); + // Headers should be ascii, I'm pretty sure. If not, please open an issue. + line.make_ascii_lowercase(); + return Some((line, value)); + } + None +} diff --git a/bitreq/src/wasm.rs b/bitreq/src/wasm.rs new file mode 100644 index 00000000..84a5d539 --- /dev/null +++ b/bitreq/src/wasm.rs @@ -0,0 +1,228 @@ +//! WASM-specific HTTP implementation using extern C functions +//! +//! This module provides HTTP functionality for WebAssembly environments +//! by delegating the actual network operations to JavaScript through +//! extern C function calls. + +use crate::{Error, Response}; +use crate::request::ParsedRequest; +use alloc::string::String; +use alloc::vec; + +/// Maximum size for HTTP response bodies in WASM +const MAX_RESPONSE_SIZE: usize = 10 * 1024 * 1024; // 10MB + +// Extern C functions that must be implemented by the JavaScript host environment +extern "C" { + /// Performs an HTTP request and returns the response. + /// + /// # Parameters + /// + /// - method: pointer to null-terminated method string (GET, POST, etc.) + /// - url: pointer to null-terminated URL string + /// - headers: pointer to null-terminated headers string (key:value\nkey:value format) + /// - body: pointer to request body bytes + /// - body_len: length of request body + /// - response_buf: buffer to write response into + /// - response_buf_len: size of response buffer + /// + /// # Returns + /// + /// - Positive value: actual response length written to buffer + /// - Negative value: error code + /// - 0: empty response + fn bitreq_wasm_http_request( + method: *const u8, + url: *const u8, + headers: *const u8, + body: *const u8, + body_len: usize, + response_buf: *mut u8, + response_buf_len: usize, + ) -> i32; + + /// Gets the HTTP status code from the last request. + fn bitreq_wasm_get_status_code() -> i32; + + /// Gets the response headers from the last request + /// + /// # Parameters + /// + /// - headers_buf: buffer to write headers into (key:value\nkey:value format) + /// - headers_buf_len: size of headers buffer + /// + /// # Returns + /// + /// - Positive value: actual headers length written to buffer + /// - Negative value: error code + /// - 0: no headers + fn bitreq_wasm_get_response_headers( + headers_buf: *mut u8, + headers_buf_len: usize, + ) -> i32; +} + +/// Sends an HTTP request using WASM extern functions. +pub(crate) fn send_request(parsed_request: ParsedRequest) -> Result { + // Convert method to C string + let method_str = format!("{}\0", parsed_request.config.method); + let method_ptr = method_str.as_ptr(); + + // Build full URL + let full_url = build_full_url(&parsed_request); + let url_str = format!("{}\0", full_url); + let url_ptr = url_str.as_ptr(); + + // Convert headers to string format + let headers_str = build_headers_string(&parsed_request); + let headers_ptr = headers_str.as_ptr(); + + // Get request body + let body = parsed_request.get_body().map(|b| b.as_slice()).unwrap_or(&[]); + let body_ptr = body.as_ptr(); + let body_len = body.len(); + + // Prepare response buffer + let mut response_buf = vec![0u8; MAX_RESPONSE_SIZE]; + let response_buf_ptr = response_buf.as_mut_ptr(); + let response_buf_len = response_buf.len(); + + // Make the extern C call + let result = unsafe { + bitreq_wasm_http_request( + method_ptr, + url_ptr, + headers_ptr, + body_ptr, + body_len, + response_buf_ptr, + response_buf_len, + ) + }; + + // Handle the result + if result < 0 { + return Err(Error::Other("WASM HTTP request failed")); + } + + let response_len = result as usize; + response_buf.truncate(response_len); + + // Get status code + let status_code = unsafe { bitreq_wasm_get_status_code() }; + + // Get response headers + let response_headers = get_response_headers()?; + + // Build and return Response + Ok(Response::new( + status_code, + get_reason_phrase(status_code), + response_headers, + full_url, + response_buf, + )) +} + +/// Builds the full URL including path and query parameters. +fn build_full_url(parsed_request: &ParsedRequest) -> String { + let mut url = String::new(); + + if parsed_request.url.https { + url.push_str("https://"); + } else { + url.push_str("http://"); + } + + url.push_str(&parsed_request.url.host); + + // Add port if explicit + if let crate::http_url::Port::Explicit(port) = parsed_request.url.port { + url.push(':'); + url.push_str(&port.to_string()); + } + + url.push_str(&parsed_request.url.path_and_query); + + url +} + +/// Builds headers string in `key:value\nkey:value` format. +fn build_headers_string(parsed_request: &ParsedRequest) -> String { + let mut headers_str = String::new(); + + for (key, value) in parsed_request.get_headers() { + if !headers_str.is_empty() { + headers_str.push('\n'); + } + headers_str.push_str(key); + headers_str.push(':'); + headers_str.push_str(value); + } + + headers_str.push('\0'); // Null terminate + headers_str +} + +/// Retrieves response headers from the WASM environment. +fn get_response_headers() -> Result, Error> { + use alloc::collections::BTreeMap; + + let mut headers_buf = vec![0u8; 8192]; // 8KB for headers + let headers_buf_ptr = headers_buf.as_mut_ptr(); + let headers_buf_len = headers_buf.len(); + + let result = unsafe { + bitreq_wasm_get_response_headers(headers_buf_ptr, headers_buf_len) + }; + + if result < 0 { + return Err(Error::Other("Failed to get response headers")); + } + + let headers_len = result as usize; + headers_buf.truncate(headers_len); + + // Parse headers string + let headers_str = String::from_utf8(headers_buf) + .map_err(|_| Error::Other("Invalid UTF-8 in response headers"))?; + + let mut headers = BTreeMap::new(); + + for line in headers_str.lines() { + if let Some(colon_pos) = line.find(':') { + let key = line[..colon_pos].trim().to_lowercase(); + let value = line[colon_pos + 1..].trim().to_string(); + headers.insert(key, value); + } + } + + Ok(headers) +} + +/// Gets a reason phrase for the given status code. +fn get_reason_phrase(status_code: i32) -> String { + match status_code { + 200 => "OK".to_string(), + 201 => "Created".to_string(), + 202 => "Accepted".to_string(), + 204 => "No Content".to_string(), + 301 => "Moved Permanently".to_string(), + 302 => "Found".to_string(), + 303 => "See Other".to_string(), + 304 => "Not Modified".to_string(), + 307 => "Temporary Redirect".to_string(), + 308 => "Permanent Redirect".to_string(), + 400 => "Bad Request".to_string(), + 401 => "Unauthorized".to_string(), + 403 => "Forbidden".to_string(), + 404 => "Not Found".to_string(), + 405 => "Method Not Allowed".to_string(), + 429 => "Too Many Requests".to_string(), + 500 => "Internal Server Error".to_string(), + 502 => "Bad Gateway".to_string(), + 503 => "Service Unavailable".to_string(), + 504 => "Gateway Timeout".to_string(), + _ => "Unknown".to_string(), + } +} diff --git a/bitreq/tests/main.rs b/bitreq/tests/main.rs new file mode 100644 index 00000000..5331e246 --- /dev/null +++ b/bitreq/tests/main.rs @@ -0,0 +1,209 @@ +#![cfg(feature = "std")] + +extern crate bitreq; +mod setup; + +use std::io; + +use self::setup::*; + +#[test] +#[cfg(feature = "rustls")] +fn test_https() { + // TODO: Implement this locally. + assert_eq!(get_status_code(bitreq::get("https://example.com").send()), 200,); +} + +#[test] +fn test_timeout_too_low() { + setup(); + let result = bitreq::get(url("/slow_a")).with_body("Q".to_string()).with_timeout(1).send(); + assert!(result.is_err()); +} + +#[test] +fn test_timeout_high_enough() { + setup(); + let body = + get_body(bitreq::get(url("/slow_a")).with_body("Q".to_string()).with_timeout(3).send()); + assert_eq!(body, "j: Q"); +} + +#[test] +fn test_headers() { + setup(); + let body = get_body(bitreq::get(url("/header_pong")).with_header("Ping", "Qwerty").send()); + assert_eq!("Qwerty", body); +} + +#[test] +fn test_custom_method() { + use bitreq::Method; + setup(); + let body = get_body( + bitreq::Request::new(Method::Custom("GET".to_string()), url("/a")).with_body("Q").send(), + ); + assert_eq!("j: Q", body); +} + +#[test] +fn test_get() { + setup(); + let body = get_body(bitreq::get(url("/a")).with_body("Q").send()); + assert_eq!(body, "j: Q"); +} + +#[test] +fn test_redirect_get() { + setup(); + let body = get_body(bitreq::get(url("/redirect")).with_body("Q").send()); + assert_eq!(body, "j: Q"); +} + +#[test] +fn test_redirect_post() { + setup(); + // POSTing to /redirect should return a 303, which means we should + // make a GET request to the given location. This test relies on + // the fact that the test server only responds to GET requests on + // the /a path. + let body = get_body(bitreq::post(url("/redirect")).with_body("Q").send()); + assert_eq!(body, "j: Q"); +} + +#[test] +fn test_redirect_with_fragment() { + setup(); + let original_url = url("/redirect#foo"); + let res = bitreq::get(original_url).send().unwrap(); + // Fragment should stay the same, otherwise redirected + assert_eq!(res.url.as_str(), url("/a#foo")); +} + +#[test] +fn test_redirect_with_overridden_fragment() { + setup(); + let original_url = url("/redirect-baz#foo"); + let res = bitreq::get(original_url).send().unwrap(); + // This redirect should provide its own fragment, overriding the initial one + assert_eq!(res.url.as_str(), url("/a#baz")); +} + +#[test] +fn test_infinite_redirect() { + setup(); + let body = bitreq::get(url("/infiniteredirect")).send(); + assert!(body.is_err()); +} + +#[test] +fn test_relative_redirect_get() { + setup(); + let body = get_body(bitreq::get(url("/relativeredirect")).with_body("Q").send()); + assert_eq!(body, "j: Q"); +} + +#[test] +fn test_head() { + setup(); + assert_eq!(get_status_code(bitreq::head(url("/b")).send()), 418); +} + +#[test] +fn test_post() { + setup(); + let body = get_body(bitreq::post(url("/c")).with_body("E").send()); + assert_eq!(body, "l: E"); +} + +#[test] +fn test_put() { + setup(); + let body = get_body(bitreq::put(url("/d")).with_body("R").send()); + assert_eq!(body, "m: R"); +} + +#[test] +fn test_delete() { + setup(); + assert_eq!(get_body(bitreq::delete(url("/e")).send()), "n: "); +} + +#[test] +fn test_trace() { + setup(); + assert_eq!(get_body(bitreq::trace(url("/f")).send()), "o: "); +} + +#[test] +fn test_options() { + setup(); + let body = get_body(bitreq::options(url("/g")).with_body("U").send()); + assert_eq!(body, "p: U"); +} + +#[test] +fn test_connect() { + setup(); + let body = get_body(bitreq::connect(url("/h")).with_body("I").send()); + assert_eq!(body, "q: I"); +} + +#[test] +fn test_patch() { + setup(); + let body = get_body(bitreq::patch(url("/i")).with_body("O").send()); + assert_eq!(body, "r: O"); +} + +#[test] +fn tcp_connect_timeout() { + let _listener = std::net::TcpListener::bind("127.0.0.1:32162").unwrap(); + let resp = + bitreq::Request::new(bitreq::Method::Get, "http://127.0.0.1:32162").with_timeout(1).send(); + assert!(resp.is_err()); + if let Some(bitreq::Error::IoError(err)) = resp.err() { + assert_eq!(err.kind(), io::ErrorKind::TimedOut); + } else { + panic!("timeout test request did not return an error"); + } +} + +#[test] +fn test_header_cap() { + setup(); + let body = bitreq::get(url("/long_header")).with_max_headers_size(999).send(); + assert!(body.is_err()); + assert!(matches!(body.err(), Some(bitreq::Error::HeadersOverflow))); + + let body = bitreq::get(url("/long_header")).with_max_headers_size(1500).send(); + assert!(body.is_ok()); +} + +#[test] +fn test_status_line_cap() { + setup(); + let expected_status_line = "HTTP/1.1 203 Non-Authoritative Information"; + + let body = bitreq::get(url("/long_status_line")) + .with_max_status_line_length(expected_status_line.len() + 1) + .send(); + assert!(body.is_err()); + assert!(matches!(body.err(), Some(bitreq::Error::StatusLineOverflow))); + + let body = bitreq::get(url("/long_status_line")) + .with_max_status_line_length(expected_status_line.len() + 2) + .send(); + assert!(body.is_ok()); +} + +#[test] +fn test_massive_content_length() { + setup(); + std::thread::spawn(|| { + // If bitreq trusts Content-Length, this should crash pretty much straight away. + let _ = bitreq::get(url("/massive_content_length")).send(); + }); + std::thread::sleep(std::time::Duration::from_millis(500)); + // If it were to crash, it would have at this point. Pass! +} diff --git a/bitreq/tests/setup.rs b/bitreq/tests/setup.rs new file mode 100644 index 00000000..efde7f97 --- /dev/null +++ b/bitreq/tests/setup.rs @@ -0,0 +1,201 @@ +extern crate bitreq; +extern crate tiny_http; +use std::str::FromStr; +use std::sync::{Arc, Once}; +use std::thread; +use std::time::Duration; + +use self::tiny_http::{Header, Method, Response, Server, StatusCode}; + +static INIT: Once = Once::new(); + +pub fn setup() { + INIT.call_once(|| { + let server = Arc::new(Server::http("localhost:35562").unwrap()); + for _ in 0..4 { + let server = server.clone(); + + thread::spawn(move || loop { + let mut request = { + if let Ok(request) = server.recv() { + request + } else { + continue; // If .recv() fails, just try again. + } + }; + let mut content = String::new(); + request.as_reader().read_to_string(&mut content).ok(); + let headers = Vec::from(request.headers()); + + let url = String::from(request.url().split('#').next().unwrap()); + match request.method() { + Method::Get if url == "/header_pong" => { + for header in headers { + if header.field.as_str() == "Ping" { + let response = Response::from_string(format!("{}", header.value)); + request.respond(response).ok(); + return; + } + } + request.respond(Response::from_string("No header!")).ok(); + } + + Method::Get if url == "/slow_a" => { + thread::sleep(Duration::from_secs(2)); + let response = Response::from_string(format!("j: {}", content)); + request.respond(response).ok(); + } + + Method::Get if url == "/a" => { + let response = Response::from_string(format!("j: {}", content)); + request.respond(response).ok(); + } + Method::Post if url == "/a" => { + let response = Response::from_string("POST to /a is not valid."); + request.respond(response).ok(); + } + + Method::Get if url == "/long_header" => { + let mut long_header = String::with_capacity(1000); + long_header += "Very-Long-Header: "; + for _ in 0..1000 - long_header.len() { + long_header += "."; + } + let long_header = Header::from_str(&long_header).unwrap(); + let response = Response::empty(200).with_header(long_header); + request.respond(response).ok(); + } + Method::Get if url == "/massive_content_length" => { + let status = StatusCode(200); + let body = std::io::empty(); + let length = 1_000_000_000_000_000; + let response = Response::new(status, vec![], body, Some(length), None) + .with_chunked_threshold(2 * length); + request.respond(response).ok(); + } + Method::Get if url == "/long_status_line" => { + request.respond(Response::empty(203)).ok(); + } + + Method::Get if url == "/redirect-baz" => { + let response = Response::empty(301).with_header( + Header::from_str("Location: http://localhost:35562/a#baz").unwrap(), + ); + request.respond(response).ok(); + } + + Method::Get if url == "/redirect" => { + let response = Response::empty(301).with_header( + Header::from_bytes(&b"Location"[..], &b"http://localhost:35562/a"[..]) + .unwrap(), + ); + request.respond(response).ok(); + } + Method::Post if url == "/redirect" => { + let response = Response::empty(303).with_header( + Header::from_bytes(&b"Location"[..], &b"http://localhost:35562/a"[..]) + .unwrap(), + ); + request.respond(response).ok(); + } + + Method::Get if url == "/infiniteredirect" => { + let response = Response::empty(301).with_header( + Header::from_bytes( + &b"Location"[..], + &b"http://localhost:35562/redirectpong"[..], + ) + .unwrap(), + ); + request.respond(response).ok(); + } + Method::Get if url == "/redirectpong" => { + let response = Response::empty(301).with_header( + Header::from_bytes( + &b"Location"[..], + &b"http://localhost:35562/infiniteredirect"[..], + ) + .unwrap(), + ); + request.respond(response).ok(); + } + Method::Get if url == "/relativeredirect" => { + let response = Response::empty(303) + .with_header(Header::from_bytes(&b"Location"[..], &b"/a"[..]).unwrap()); + request.respond(response).ok(); + } + + Method::Post if url == "/echo" => { + request.respond(Response::from_string(content)).ok(); + } + + Method::Head if url == "/b" => { + request.respond(Response::empty(418)).ok(); + } + Method::Post if url == "/c" => { + let response = Response::from_string(format!("l: {}", content)); + request.respond(response).ok(); + } + Method::Put if url == "/d" => { + let response = Response::from_string(format!("m: {}", content)); + request.respond(response).ok(); + } + Method::Delete if url == "/e" => { + let response = Response::from_string(format!("n: {}", content)); + request.respond(response).ok(); + } + Method::Trace if url == "/f" => { + let response = Response::from_string(format!("o: {}", content)); + request.respond(response).ok(); + } + Method::Options if url == "/g" => { + let response = Response::from_string(format!("p: {}", content)); + request.respond(response).ok(); + } + Method::Connect if url == "/h" => { + let response = Response::from_string(format!("q: {}", content)); + request.respond(response).ok(); + } + Method::Patch if url == "/i" => { + let response = Response::from_string(format!("r: {}", content)); + request.respond(response).ok(); + } + + _ => { + request + .respond(Response::from_string("Not Found").with_status_code(404)) + .ok(); + } + } + }); + } + }); +} + +pub fn url(req: &str) -> String { format!("http://localhost:35562{}", req) } + +pub fn get_body(request: Result) -> String { + match request { + Ok(response) => match response.as_str() { + Ok(str) => String::from(str), + Err(err) => { + println!("\n[ERROR]: {}\n", err); + String::new() + } + }, + Err(err) => { + println!("\n[ERROR]: {}\n", err); + String::new() + } + } +} + +pub fn get_status_code(request: Result) -> i32 { + match request { + Ok(response) => response.status_code, + Err(err) => { + println!("\n[ERROR]: {}\n", err); + -1 + } + } +}