diff --git a/facilitator/Cargo.lock b/facilitator/Cargo.lock index 969e85b12..f3f77f9b9 100644 --- a/facilitator/Cargo.lock +++ b/facilitator/Cargo.lock @@ -59,7 +59,7 @@ checksum = "4925647ee64e5056cf231608957ce7c81e12d6d6e316b9ce1404778cc1d35fa7" dependencies = [ "block-cipher", "byteorder", - "opaque-debug", + "opaque-debug 0.2.3", ] [[package]] @@ -69,17 +69,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d050d39b0b7688b3a3254394c3e30a9d66c41dcf9b05b0e2dbdc623f6505d264" dependencies = [ "block-cipher", - "opaque-debug", + "opaque-debug 0.2.3", "stream-cipher", ] +[[package]] +name = "aho-corasick" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" +dependencies = [ + "memchr", +] + [[package]] name = "ansi_term" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -88,6 +97,35 @@ version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b602bfe940d21c130f3895acd65221e8a61270debe89d628b9cb4e3ccb8569b" +[[package]] +name = "arc-swap" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + +[[package]] +name = "async-trait" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b246867b8b3b6ae56035f1eb1ed557c1d8eae97f0d53696138a50fa0e3a3b8c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -96,7 +134,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi", "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -126,6 +164,18 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "base-x" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + [[package]] name = "base64" version = "0.12.3" @@ -138,6 +188,26 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "blake2b_simd" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + [[package]] name = "block-cipher" version = "0.7.1" @@ -159,6 +229,12 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + [[package]] name = "cc" version = "1.0.59" @@ -179,7 +255,8 @@ checksum = "942f72db697d8767c22d46a598e01f2d3b475501ea43d0db4f16d90259182d0b" dependencies = [ "num-integer", "num-traits", - "time", + "serde", + "time 0.1.44", ] [[package]] @@ -197,6 +274,40 @@ dependencies = [ "vec_map", ] +[[package]] +name = "const_fn" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + [[package]] name = "crc32fast" version = "1.2.0" @@ -206,6 +317,45 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if", + "lazy_static", +] + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "ct-logs" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3686f5fa27dbc1d76c751300376e167c5a43387f44bb451fd1c24776e49113" +dependencies = [ + "sct", +] + +[[package]] +name = "ct-logs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c8e13110a84b6315df212c045be706af261fd364791cad863285439ebba672e" +dependencies = [ + "sct", +] + [[package]] name = "ctr" version = "0.4.0" @@ -215,6 +365,17 @@ dependencies = [ "stream-cipher", ] +[[package]] +name = "derivative" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb582b60359da160a9477ee80f15c8d784c477e69c217ef2cdd4169c24ea380f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.9.0" @@ -224,31 +385,182 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dirs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" +dependencies = [ + "cfg-if", + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + [[package]] name = "facilitator" version = "0.1.0" dependencies = [ "anyhow", "avro-rs", - "base64", + "base64 0.12.3", "chrono", "clap", + "derivative", + "hyper", + "hyper-rustls 0.21.0", "prio", "rand 0.7.3", "ring", + "rusoto_core", + "rusoto_mock", + "rusoto_s3", "serde", "tempfile", "thiserror", + "tokio", "uuid", "vergen", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "fuchsia-cprng" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" + +[[package]] +name = "futures-executor" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d6bb888be1153d3abeb9006b11b02cf5e9b209fda28693c31ae1e4e012e314" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" + +[[package]] +name = "futures-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc" + +[[package]] +name = "futures-task" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + [[package]] name = "generic-array" version = "0.14.4" @@ -279,6 +591,31 @@ dependencies = [ "polyval", ] +[[package]] +name = "h2" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993f9e0baeed60001cf565546b0d3dbe6a6ad23f2bd31644a133c641eccf6d53" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + [[package]] name = "heck" version = "0.3.1" @@ -297,6 +634,134 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + +[[package]] +name = "hmac" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" +dependencies = [ + "crypto-mac", + "digest", +] + +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "hyper" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac965ea399ec3a25ac7d13b8affd4b8f39325cca00858ddf5eb29b79e6b14b08" +dependencies = [ + "bytes", + "ct-logs 0.6.0", + "futures-util", + "hyper", + "log", + "rustls 0.17.0", + "rustls-native-certs 0.3.0", + "tokio", + "tokio-rustls 0.13.1", + "webpki", +] + +[[package]] +name = "hyper-rustls" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" +dependencies = [ + "bytes", + "ct-logs 0.7.0", + "futures-util", + "hyper", + "log", + "rustls 0.18.1", + "rustls-native-certs 0.4.0", + "tokio", + "tokio-rustls 0.14.1", + "webpki", +] + +[[package]] +name = "indexmap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + [[package]] name = "itoa" version = "0.4.6" @@ -312,6 +777,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -352,8 +827,95 @@ dependencies = [ ] [[package]] -name = "num-bigint" -version = "0.2.6" +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + +[[package]] +name = "mio" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +dependencies = [ + "cfg-if", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.1", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log", + "mio", + "miow 0.3.5", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" +dependencies = [ + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "net2" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" +dependencies = [ + "cfg-if", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "num-bigint" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ @@ -393,6 +955,56 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b9e280448854bd91559252582173b3bd1f8e094a0e644791c0628ca9b1f144f" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8c8b352676bc6a4c3d71970560b913cea444a7a921cc2e2d920225e4b91edaa" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e555d9e657502182ac97b539fb3dae8b79cda19e3e4f8ffb5e8de4f18df93c95" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "polyval" version = "0.4.0" @@ -417,186 +1029,591 @@ checksum = "74c293e32d9ab6c9afadf14e14d76df204f17618f4b74b38fa2adb18b352fbf2" dependencies = [ "aes-ctr", "aes-gcm", - "base64", + "base64 0.12.3", "rand 0.7.3", "ring", "thiserror", ] [[package]] -name = "proc-macro2" -version = "1.0.21" +name = "proc-macro-hack" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom", + "redox_syscall", + "rust-argon2", +] + +[[package]] +name = "regex" +version = "1.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ring" +version = "0.16.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "952cd6b98c85bbc30efa1ba5783b8abf12fec8b3287ffa52605b9432313e34e4" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + +[[package]] +name = "rle-decode-fast" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cabe4fa914dec5870285fa7f71f602645da47c486e68486d2b4ceb4a343e90ac" + +[[package]] +name = "rusoto_core" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e977941ee0658df96fca7291ecc6fc9a754600b21ad84b959eb1dbbc9d5abcc7" +dependencies = [ + "async-trait", + "base64 0.12.3", + "bytes", + "crc32fast", + "futures", + "http", + "hyper", + "hyper-rustls 0.20.0", + "lazy_static", + "log", + "md5", + "percent-encoding", + "pin-project", + "rusoto_credential", + "rusoto_signature", + "rustc_version", + "serde", + "serde_json", + "tokio", + "xml-rs", +] + +[[package]] +name = "rusoto_credential" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac05563f83489b19b4d413607a30821ab08bbd9007d14fa05618da3ef09d8b" +dependencies = [ + "async-trait", + "chrono", + "dirs", + "futures", + "hyper", + "pin-project", + "regex", + "serde", + "serde_json", + "shlex", + "tokio", + "zeroize", +] + +[[package]] +name = "rusoto_mock" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff14fcb9ae32511e705349da464e117bac945b541ec87283ad5dc93c1280250" +dependencies = [ + "async-trait", + "chrono", + "futures", + "http", + "rusoto_core", + "serde", + "serde_json", +] + +[[package]] +name = "rusoto_s3" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1146e37a7c1df56471ea67825fe09bbbd37984b5f6e201d8b2e0be4ee15643d8" +dependencies = [ + "async-trait", + "bytes", + "futures", + "rusoto_core", + "xml-rs", +] + +[[package]] +name = "rusoto_signature" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a740a88dde8ded81b6f2cff9cd5e054a5a2e38a38397260f7acdd2c85d17dd" +dependencies = [ + "base64 0.12.3", + "bytes", + "futures", + "hex", + "hmac", + "http", + "hyper", + "log", + "md5", + "percent-encoding", + "pin-project", + "rusoto_credential", + "rustc_version", + "serde", + "sha2", + "time 0.2.22", + "tokio", +] + +[[package]] +name = "rust-argon2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19" +dependencies = [ + "base64 0.12.3", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" +dependencies = [ + "base64 0.11.0", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" +dependencies = [ + "base64 0.12.3", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75ffeb84a6bd9d014713119542ce415db3a3e4748f0bfce1e1416cd224a23a5" +dependencies = [ + "openssl-probe", + "rustls 0.17.0", + "schannel", + "security-framework 0.4.4", +] + +[[package]] +name = "rustls-native-certs" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629d439a7672da82dd955498445e496ee2096fe2117b9f796558a43fdb9e59b8" +dependencies = [ + "openssl-probe", + "rustls 0.18.1", + "schannel", + "security-framework 1.0.0", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + +[[package]] +name = "sct" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c" +checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" dependencies = [ - "unicode-xid", + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys 0.4.3", ] [[package]] -name = "quote" -version = "1.0.7" +name = "security-framework" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +checksum = "ad502866817f0575705bd7be36e2b2535cc33262d493aa733a2ec862baa2bc2b" dependencies = [ - "proc-macro2", + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys 1.0.0", ] [[package]] -name = "rand" -version = "0.4.6" +name = "security-framework-sys" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" dependencies = [ - "fuchsia-cprng", + "core-foundation-sys", "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", ] [[package]] -name = "rand" -version = "0.7.3" +name = "security-framework-sys" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +checksum = "51ceb04988b17b6d1dcd555390fa822ca5637b4a14e1f5099f13d351bed4d6c7" dependencies = [ - "getrandom", + "core-foundation-sys", "libc", - "rand_chacha", - "rand_core 0.5.1", - "rand_hc", ] [[package]] -name = "rand_chacha" -version = "0.2.2" +name = "semver" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "semver-parser", ] [[package]] -name = "rand_core" -version = "0.3.1" +name = "semver-parser" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] -name = "rand_core" -version = "0.4.2" +name = "serde" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +checksum = "e54c9a88f2da7238af84b5101443f0c0d0a3bbdc455e34a5c9497b1903ed55d5" +dependencies = [ + "serde_derive", +] [[package]] -name = "rand_core" -version = "0.5.1" +name = "serde_derive" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +checksum = "609feed1d0a73cc36a0182a840a9b37b4a82f0b1150369f0536a9e3f2a31dc48" dependencies = [ - "getrandom", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "rand_hc" -version = "0.2.0" +name = "serde_json" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +checksum = "164eacbdb13512ec2745fb09d51fd5b22b0d65ed294a1dcf7285a360c80a675c" dependencies = [ - "rand_core 0.5.1", + "itoa", + "ryu", + "serde", ] [[package]] -name = "rdrand" -version = "0.4.0" +name = "sha1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "sha2" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1" dependencies = [ - "rand_core 0.3.1", + "block-buffer", + "cfg-if", + "cpuid-bool", + "digest", + "opaque-debug 0.3.0", ] [[package]] -name = "redox_syscall" -version = "0.1.57" +name = "shlex" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" [[package]] -name = "remove_dir_all" -version = "0.5.3" +name = "signal-hook-registry" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +checksum = "a3e12110bc539e657a646068aaf5eb5b63af9d0c1f7b29c97113fad80e15f035" dependencies = [ - "winapi", + "arc-swap", + "libc", ] [[package]] -name = "ring" -version = "0.16.15" +name = "slab" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "952cd6b98c85bbc30efa1ba5783b8abf12fec8b3287ffa52605b9432313e34e4" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "socket2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" dependencies = [ - "cc", + "cfg-if", "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi", + "redox_syscall", + "winapi 0.3.9", ] [[package]] -name = "rle-decode-fast" -version = "1.0.1" +name = "spin" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cabe4fa914dec5870285fa7f71f602645da47c486e68486d2b4ceb4a343e90ac" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] -name = "ryu" -version = "1.0.5" +name = "standback" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "33a71ea1ea5f8747d1af1979bfb7e65c3a025a70609f04ceb78425bc5adad8e6" +dependencies = [ + "version_check", +] [[package]] -name = "serde" -version = "1.0.115" +name = "stdweb" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54c9a88f2da7238af84b5101443f0c0d0a3bbdc455e34a5c9497b1903ed55d5" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ - "serde_derive", + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", ] [[package]] -name = "serde_derive" -version = "1.0.115" +name = "stdweb-derive" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609feed1d0a73cc36a0182a840a9b37b4a82f0b1150369f0536a9e3f2a31dc48" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ "proc-macro2", "quote", + "serde", + "serde_derive", "syn", ] [[package]] -name = "serde_json" -version = "1.0.57" +name = "stdweb-internal-macros" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164eacbdb13512ec2745fb09d51fd5b22b0d65ed294a1dcf7285a360c80a675c" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" dependencies = [ - "itoa", - "ryu", + "base-x", + "proc-macro2", + "quote", "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", ] [[package]] -name = "spin" -version = "0.5.2" +name = "stdweb-internal-runtime" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" [[package]] name = "stream-cipher" @@ -672,7 +1689,7 @@ dependencies = [ "rand 0.7.3", "redox_syscall", "remove_dir_all", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -704,6 +1721,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + [[package]] name = "time" version = "0.1.44" @@ -712,9 +1738,152 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", + "winapi 0.3.9", +] + +[[package]] +name = "time" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b7151c9065e80917fbf285d9a5d1432f60db41d170ccafc749a136b41a93af" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi 0.3.9", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tokio" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio", + "mio-named-pipes", + "mio-uds", + "pin-project-lite", + "signal-hook-registry", + "slab", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4" +dependencies = [ + "futures-core", + "rustls 0.17.0", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-rustls" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" +dependencies = [ + "futures-core", + "rustls 0.18.1", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typed-builder" version = "0.5.1" @@ -798,6 +1967,16 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -874,6 +2053,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab146130f5f790d45f82aeeb09e55a256573373ec64409fc19a6fb82fb1032ae" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + [[package]] name = "winapi" version = "0.3.9" @@ -884,6 +2079,12 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -896,6 +2097,22 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + [[package]] name = "zerocopy" version = "0.3.0" @@ -916,3 +2133,9 @@ dependencies = [ "syn", "synstructure", ] + +[[package]] +name = "zeroize" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f33972566adbd2d3588b0491eb94b98b43695c4ef897903470ede4f3f5a28a" diff --git a/facilitator/Cargo.toml b/facilitator/Cargo.toml index 6bb0e49d6..eda645a83 100644 --- a/facilitator/Cargo.toml +++ b/facilitator/Cargo.toml @@ -11,13 +11,22 @@ avro-rs = "0.11.0" base64 = "0.12.3" chrono = "0.4" clap = "2.33.3" +derivative = "2.1.1" +hyper = "0.13.8" +hyper-rustls = "0.21.0" prio = "0.2" rand = "0.7" ring = { version = "0.16.15", features = ["std"] } +rusoto_core = { version = "0.45.0", default_features = false, features = ["rustls"] } +rusoto_s3 = { version = "0.45.0", default_features = false, features = ["rustls"] } serde = { version = "1.0", features = ["derive"] } tempfile = "3.1.0" thiserror = "1.0" +tokio = { version = "0.2", features = ["rt-core", "io-util"] } uuid = { version = "0.8", features = ["serde", "v4"] } [build-dependencies] vergen = "3" + +[dev-dependencies] +rusoto_mock = { version = "0.45.0", default_features = false, features = ["rustls"] } diff --git a/facilitator/src/batch.rs b/facilitator/src/batch.rs index 05e5ff131..194ee02ae 100644 --- a/facilitator/src/batch.rs +++ b/facilitator/src/batch.rs @@ -1,6 +1,6 @@ use crate::{ idl::{Header, Packet}, - transport::Transport, + transport::{Transport, TransportWriter}, DigestWriter, SidecarWriter, DATE_FORMAT, }; use anyhow::{anyhow, Context, Result}; @@ -13,14 +13,13 @@ use ring::{ }; use std::io::{Cursor, Read, Write}; use std::marker::PhantomData; -use std::path::{Path, PathBuf}; use uuid::Uuid; /// Manages the paths to the different files in a batch pub struct Batch { - header_path: PathBuf, - signature_path: PathBuf, - packet_file_path: PathBuf, + header_path: String, + signature_path: String, + packet_file_path: String, } impl Batch { @@ -51,46 +50,49 @@ impl Batch { aggregation_end: &NaiveDateTime, is_first: bool, ) -> Batch { - let batch_path = PathBuf::new().join(aggregation_name).join(format!( - "{}-{}", + let batch_path = format!( + "{}/{}-{}", + aggregation_name, aggregation_start.format(DATE_FORMAT), aggregation_end.format(DATE_FORMAT) - )); + ); let filename = format!("sum_{}", if is_first { 0 } else { 1 }); Batch { - header_path: batch_path.with_extension(&filename), - signature_path: batch_path.with_extension(format!("{}.sig", &filename)), - packet_file_path: batch_path.with_extension(format!( - "invalid_uuid_{}.avro", + header_path: format!("{}.{}", batch_path, filename), + signature_path: format!("{}.{}.sig", batch_path, filename), + packet_file_path: format!( + "{}.invalid_uuid_{}.avro", + batch_path, if is_first { 0 } else { 1 } - )), + ), } } fn new(aggregation_name: &str, batch_id: &Uuid, date: &NaiveDateTime, filename: &str) -> Batch { - let batch_path = PathBuf::new() - .join(aggregation_name) - .join(date.format(DATE_FORMAT).to_string()) - .join(batch_id.to_hyphenated().to_string()); - + let batch_path = format!( + "{}/{}/{}", + aggregation_name, + date.format(DATE_FORMAT), + batch_id.to_hyphenated() + ); Batch { - header_path: batch_path.with_extension(filename), - signature_path: batch_path.with_extension(format!("{}.sig", filename)), - packet_file_path: batch_path.with_extension(format!("{}.avro", filename)), + header_path: format!("{}.{}", batch_path, filename), + signature_path: format!("{}.{}.sig", batch_path, filename), + packet_file_path: format!("{}.{}.avro", batch_path, filename), } } - fn header_key(&self) -> &Path { - self.header_path.as_path() + fn header_key(&self) -> &str { + self.header_path.as_ref() } - fn signature_key(&self) -> &Path { - self.signature_path.as_path() + fn signature_key(&self) -> &str { + self.signature_path.as_ref() } - fn packet_file_key(&self) -> &Path { - self.packet_file_path.as_path() + fn packet_file_key(&self) -> &str { + self.packet_file_path.as_ref() } } @@ -253,6 +255,10 @@ impl<'a, H: Header, P: Packet> BatchWriter<'a, H, P> { let mut sidecar_writer = SidecarWriter::new(self.transport.put(self.batch.header_key())?, Vec::new()); header.write(&mut sidecar_writer)?; + sidecar_writer + .writer + .complete_upload() + .context("failed to complete batch header upload")?; let header_signature = key .sign(&SystemRandom::new(), &sidecar_writer.sidecar) @@ -268,7 +274,7 @@ impl<'a, H: Header, P: Packet> BatchWriter<'a, H, P> { /// content written by the operation. pub fn packet_file_writer(&mut self, operation: F) -> Result where - F: FnOnce(&mut Writer, DigestWriter>>) -> Result<()>, + F: FnOnce(&mut Writer, DigestWriter>>) -> Result<()>, { let mut writer = Writer::new( &self.packet_schema, @@ -278,22 +284,36 @@ impl<'a, H: Header, P: Packet> BatchWriter<'a, H, P> { ), ); - operation(&mut writer)?; - - Ok(writer + let result = operation(&mut writer); + let mut sidecar_writer = writer .into_inner() - .context("failed to flush writer")? - .sidecar - .finish()) + .with_context(|| format!("failed to flush Avro writer ({:?})", result))?; + + if let Err(e) = result { + sidecar_writer + .writer + .cancel_upload() + .with_context(|| format!("Encountered while handling: {}", e))?; + return Err(e); + } + + sidecar_writer + .writer + .complete_upload() + .context("failed to complete packet file upload")?; + Ok(sidecar_writer.sidecar.finish()) } /// Constructs a signature structure from the provided buffers and writes it /// to the batch's signature file pub fn put_signature(&mut self, signature: &Signature) -> Result<()> { - self.transport - .put(self.batch.signature_key())? + let mut writer = self.transport.put(self.batch.signature_key())?; + writer .write_all(signature.as_ref()) - .context("failed to write signature") + .context("failed to write signature")?; + writer + .complete_upload() + .context("failed to complete signature upload") } } @@ -313,7 +333,7 @@ mod tests { fn roundtrip_batch<'a>( aggregation_name: String, batch_id: Uuid, - base_path: PathBuf, + base_path: String, filenames: &[String], batch_writer: &mut BatchWriter<'a, IngestionHeader, IngestionDataSharePacket>, batch_reader: &BatchReader<'a, IngestionHeader, IngestionDataSharePacket>, @@ -386,7 +406,7 @@ mod tests { // Verify file layout is as expected for extension in filenames { transport - .get(&base_path.with_extension(extension)) + .get(&format!("{}.{}", base_path, extension)) .expect(&format!("could not get batch file {}", extension)); } @@ -450,10 +470,12 @@ mod tests { let batch_reader: BatchReader<'_, IngestionHeader, IngestionDataSharePacket> = BatchReader::new_ingestion(&aggregation_name, &batch_id, &date, &mut read_transport) .unwrap(); - let base_path = PathBuf::new() - .join(aggregation_name) - .join(date.format(DATE_FORMAT).to_string()) - .join(batch_id.to_hyphenated().to_string()); + let base_path = format!( + "{}/{}/{}", + aggregation_name, + date.format(DATE_FORMAT), + batch_id.to_hyphenated() + ); let read_key = if keys_match { default_ingestor_public_key() } else { @@ -525,10 +547,12 @@ mod tests { &mut read_transport, ) .unwrap(); - let base_path = PathBuf::new() - .join(aggregation_name) - .join(date.format(DATE_FORMAT).to_string()) - .join(batch_id.to_hyphenated().to_string()); + let base_path = format!( + "{}/{}/{}", + aggregation_name, + date.format(DATE_FORMAT), + batch_id.to_hyphenated() + ); let first_filenames = &[ "validity_0".to_owned(), "validity_0.avro".to_owned(), @@ -611,11 +635,12 @@ mod tests { &mut read_transport, ) .unwrap(); - let batch_path = PathBuf::new().join(aggregation_name).join(format!( - "{}-{}", + let batch_path = format!( + "{}/{}-{}", + aggregation_name, start.format(DATE_FORMAT), end.format(DATE_FORMAT) - )); + ); let first_filenames = &[ "sum_0".to_owned(), "invalid_uuid_0.avro".to_owned(), diff --git a/facilitator/src/bin/facilitator.rs b/facilitator/src/bin/facilitator.rs index 9a8511abb..88e7043b9 100644 --- a/facilitator/src/bin/facilitator.rs +++ b/facilitator/src/bin/facilitator.rs @@ -1,6 +1,4 @@ -use std::{path::Path, str::FromStr}; - -use anyhow::{anyhow, Context}; +use anyhow::{anyhow, Context, Result}; use chrono::{prelude::Utc, NaiveDateTime}; use clap::{App, Arg, ArgMatches, SubCommand}; use prio::encrypt::PrivateKey; @@ -8,6 +6,8 @@ use ring::signature::{ EcdsaKeyPair, KeyPair, UnparsedPublicKey, ECDSA_P256_SHA256_FIXED, ECDSA_P256_SHA256_FIXED_SIGNING, }; +use rusoto_core::Region; +use std::{path::Path, str::FromStr}; use uuid::Uuid; use facilitator::{ @@ -19,7 +19,7 @@ use facilitator::{ DEFAULT_INGESTOR_PRIVATE_KEY, DEFAULT_PHA_ECIES_PRIVATE_KEY, DEFAULT_PHA_SIGNING_PRIVATE_KEY, }, - transport::LocalFileTransport, + transport::{LocalFileTransport, S3Transport, Transport}, DATE_FORMAT, }; @@ -43,6 +43,39 @@ fn uuid_validator(s: String) -> Result<(), String> { Uuid::parse_str(&s).map(|_| ()).map_err(|e| e.to_string()) } +enum StoragePath<'a> { + S3Path { region: &'a str, bucket: &'a str }, + LocalPath(&'a str), +} + +fn parse_path<'a>(s: &'a str) -> Result> { + match s.strip_prefix("s3://") { + Some(region_and_bucket) => { + if !region_and_bucket.contains("/") { + return Err(anyhow!( + "S3 storage must be like \"s3://{region}/{bucket name}\"" + )); + } + + // All we require is that the string contain a region and a bucket name. + // Further validation of bucket names is left to Amazon servers. + let mut components = region_and_bucket.splitn(2, '/'); + let region = components.next().context("S3 URL missing region")?; + let bucket = components.next().context("S3 URL missing bucket name")?; + // splitn will only return 2 so it should never have more + assert!(components.next().is_none()); + Ok(StoragePath::S3Path { region, bucket }) + } + None => Ok(StoragePath::LocalPath(s)), + } +} + +fn path_validator(s: String) -> Result<(), String> { + parse_path(s.as_ref()) + .map(|_| ()) + .map_err(|e| e.to_string()) +} + fn main() -> Result<(), anyhow::Error> { let matches = App::new("facilitator") .about("Prio data share processor") @@ -67,9 +100,12 @@ fn main() -> Result<(), anyhow::Error> { .long("pha-output") .value_name("DIR") .default_value(".") + .validator(path_validator) .help( - "Directory to write sample data for the PHA (aka \ - first server) into", + "Storage to write sample data for the PHA \ + (aka first server) into. May be either a local \ + filesystem path or an S3 bucket, formatted as \ + \"s3://{region}/{bucket-name}\"", ), ) .arg( @@ -77,9 +113,12 @@ fn main() -> Result<(), anyhow::Error> { .long("facilitator-output") .value_name("DIR") .default_value(".") + .validator(path_validator) .help( - "Directory to write sample data for the \ - facilitator (aka second server) into", + "Storage to write sample data for the \ + facilitator (aka second server) into. May be \ + either a local filesystem path or an S3 bucket, \ + formatted as \"s3://{region}/{bucket-name}\"", ), ) .arg( @@ -292,16 +331,24 @@ fn main() -> Result<(), anyhow::Error> { .long("ingestion-bucket") .value_name("DIR") .default_value(".") - .help("Directory containing ingestion data"), + .validator(path_validator) + .help( + "Directory containing ingestion data. May be \ + either a local filesystem path or an S3 bucket, \ + formatted as \"s3://{region}/{bucket-name}\"", + ), ) .arg( Arg::with_name("validation-bucket") .long("validation-bucket") .value_name("DIR") .default_value(".") + .validator(path_validator) .help( "Peer validation bucket into which to write \ - validation shares", + validation shares. May be either a local \ + filesystem path or an S3 bucket, formatted as \ + \"s3://{region}/{bucket-name}\"", ), ), ) @@ -373,16 +420,24 @@ fn main() -> Result<(), anyhow::Error> { .long("ingestion-bucket") .value_name("DIR") .default_value(".") - .help("Directory containing ingestion data"), + .validator(path_validator) + .help( + "Directory containing ingestion data. May be \ + either a local filesystem path or an S3 bucket, \ + formatted as \"s3://{region}/{bucket-name}\"", + ), ) .arg( Arg::with_name("own-validation-bucket") .long("own-validation-bucket") .value_name("DIR") .default_value(".") + .validator(path_validator) .help( "Bucket in which this share processor's validation \ - shares were written", + shares were written. May be either a local \ + filesystem path or an S3 bucket, formatted as \ + \"s3://{region}/{bucket-name}\"", ), ) .arg( @@ -390,9 +445,12 @@ fn main() -> Result<(), anyhow::Error> { .long("peer-validation-bucket") .value_name("DIR") .default_value(".") + .validator(path_validator) .help( "Bucket in which the peer share processor's \ - validation shares were written", + validation shares were written. May be either a \ + local filesystem path or an S3 bucket, formatted \ + as \"s3://{region}/{bucket-name}\"", ), ) .arg( @@ -400,7 +458,12 @@ fn main() -> Result<(), anyhow::Error> { .long("aggregation-bucket") .value_name("DIR") .default_value(".") - .help("Bucket into which sum parts are to be written."), + .validator(path_validator) + .help( + "Bucket into which sum parts are to be written. May be either a \ + local filesystem path or an S3 bucket, formatted \ + as \"s3://{region}/{bucket-name}\"", + ), ) .arg( Arg::with_name("ecies-private-key") @@ -473,13 +536,13 @@ fn main() -> Result<(), anyhow::Error> { // various parameters are present and valid, so it is safe to use // unwrap() here. ("generate-ingestion-sample", Some(sub_matches)) => { + let mut pha_transport = transport_for_output_path("pha-output", sub_matches)?; + let mut facilitator_transport = + transport_for_output_path("facilitator-output", sub_matches)?; + generate_ingestion_sample( - &mut LocalFileTransport::new( - Path::new(sub_matches.value_of("pha-output").unwrap()).to_path_buf(), - ), - &mut LocalFileTransport::new( - Path::new(sub_matches.value_of("facilitator-output").unwrap()).to_path_buf(), - ), + &mut *pha_transport, + &mut *facilitator_transport, &sub_matches .value_of("batch-id") .map_or_else(|| Uuid::new_v4(), |v| Uuid::parse_str(v).unwrap()), @@ -488,9 +551,14 @@ fn main() -> Result<(), anyhow::Error> { || Utc::now().naive_utc(), |v| NaiveDateTime::parse_from_str(&v, DATE_FORMAT).unwrap(), ), - &PrivateKey::from_base64(sub_matches.value_of("pha-private-key").unwrap()).unwrap(), - &PrivateKey::from_base64(sub_matches.value_of("facilitator-private-key").unwrap()) + &PrivateKey::from_base64(sub_matches.value_of("pha-ecies-private-key").unwrap()) .unwrap(), + &PrivateKey::from_base64( + sub_matches + .value_of("facilitator-ecies-private-key") + .unwrap(), + ) + .unwrap(), &base64::decode(sub_matches.value_of("ingestor-private-key").unwrap()).unwrap(), sub_matches .value_of("dimension") @@ -521,12 +589,10 @@ fn main() -> Result<(), anyhow::Error> { Ok(()) } ("batch-intake", Some(sub_matches)) => { - let mut ingestion_transport = LocalFileTransport::new( - Path::new(sub_matches.value_of("ingestion-bucket").unwrap()).to_path_buf(), - ); - let mut validation_transport = LocalFileTransport::new( - Path::new(sub_matches.value_of("validation-bucket").unwrap()).to_path_buf(), - ); + let mut ingestion_transport = + transport_for_output_path("ingestion-bucket", sub_matches)?; + let mut validation_transport = + transport_for_output_path("validation-bucket", sub_matches)?; let share_processor_ecies_key = PrivateKey::from_base64(sub_matches.value_of("ecies-private-key").unwrap()) @@ -552,8 +618,8 @@ fn main() -> Result<(), anyhow::Error> { || Utc::now().naive_utc(), |v| NaiveDateTime::parse_from_str(&v, DATE_FORMAT).unwrap(), ), - &mut ingestion_transport, - &mut validation_transport, + &mut *ingestion_transport, + &mut *validation_transport, sub_matches.is_present("is-first"), &share_processor_ecies_key, &share_processor_key, @@ -563,18 +629,14 @@ fn main() -> Result<(), anyhow::Error> { Ok(()) } ("aggregate", Some(sub_matches)) => { - let mut ingestion_transport = LocalFileTransport::new( - Path::new(sub_matches.value_of("ingestion-bucket").unwrap()).to_path_buf(), - ); - let mut own_validation_transport = LocalFileTransport::new( - Path::new(sub_matches.value_of("own-validation-bucket").unwrap()).to_path_buf(), - ); - let mut peer_validation_transport = LocalFileTransport::new( - Path::new(sub_matches.value_of("peer-validation-bucket").unwrap()).to_path_buf(), - ); - let mut aggregation_transport = LocalFileTransport::new( - Path::new(sub_matches.value_of("aggregation-bucket").unwrap()).to_path_buf(), - ); + let mut ingestion_transport = + transport_for_output_path("ingestion-bucket", sub_matches)?; + let mut own_validation_transport = + transport_for_output_path("own-validation-bucket", sub_matches)?; + let mut peer_validation_transport = + transport_for_output_path("peer-validation-bucket", sub_matches)?; + let mut aggregation_transport = + transport_for_output_path("aggregation-bucket", sub_matches)?; let ingestor_pub_key = public_key_from_arg("ingestor-public-key", sub_matches); let peer_share_processor_pub_key = @@ -618,10 +680,10 @@ fn main() -> Result<(), anyhow::Error> { |v| NaiveDateTime::parse_from_str(&v, DATE_FORMAT).unwrap(), ), sub_matches.is_present("is-first"), - &mut ingestion_transport, - &mut own_validation_transport, - &mut peer_validation_transport, - &mut aggregation_transport, + &mut *ingestion_transport, + &mut *own_validation_transport, + &mut *peer_validation_transport, + &mut *aggregation_transport, &ingestor_pub_key, &share_processor_key, &peer_share_processor_pub_key, @@ -646,3 +708,16 @@ fn public_key_from_arg(arg: &str, matches: &ArgMatches) -> UnparsedPublicKey UnparsedPublicKey::new(&ECDSA_P256_SHA256_FIXED, key_bytes), } } + +fn transport_for_output_path(arg: &str, matches: &ArgMatches) -> Result> { + let path = parse_path(matches.value_of(arg).unwrap())?; + match path { + StoragePath::S3Path { region, bucket } => Ok(Box::new(S3Transport::new( + Region::from_str(region)?, + bucket.to_string(), + ))), + StoragePath::LocalPath(path) => Ok(Box::new(LocalFileTransport::new( + Path::new(path).to_path_buf(), + ))), + } +} diff --git a/facilitator/src/lib.rs b/facilitator/src/lib.rs index b7a44e61d..1cacd3354 100644 --- a/facilitator/src/lib.rs +++ b/facilitator/src/lib.rs @@ -1,3 +1,4 @@ +use anyhow::Result; use ring::digest; use std::io::Write; @@ -30,10 +31,10 @@ pub enum Error { MalformedDataPacketError(String), #[error("end of file")] EofError, - #[error("I/O error: {0}")] - IoError(String, #[source] std::io::Error), } +/// An implementation of transport::TransportWriter that computes a SHA256 +/// digest over the content it is provided. pub struct DigestWriter { context: digest::Context, } @@ -45,6 +46,7 @@ impl DigestWriter { } } + /// Consumes the DigestWriter and returns the computed SHA256 hash. fn finish(self) -> digest::Digest { self.context.finish() } @@ -61,26 +63,15 @@ impl Write for DigestWriter { } } -/// MemoryWriter is a trait we apply to std::io::Write implementations that are -/// backed by memory and are assumed to not fail or perform short writes in any -/// circumstance short of ENOMEM. -pub trait MemoryWriter: Write {} - -impl MemoryWriter for DigestWriter {} -impl MemoryWriter for Vec {} - /// SidecarWriter wraps an std::io::Write, but also writes any buffers passed to -/// it into a MemoryWriter. The reason sidecar isn't simply some other type that -/// implements std::io::Write is that SidecarWriter::write assumes that short -/// writes to the sidecar can't happen, so we use the trait to ensure that this -/// property holds. -pub struct SidecarWriter { - writer: W, - sidecar: M, +/// it into a second std::io::Write. +pub struct SidecarWriter { + writer: T, + sidecar: W, } -impl SidecarWriter { - fn new(writer: W, sidecar: M) -> SidecarWriter { +impl SidecarWriter { + fn new(writer: T, sidecar: W) -> SidecarWriter { SidecarWriter { writer: writer, sidecar: sidecar, @@ -88,7 +79,7 @@ impl SidecarWriter { } } -impl Write for SidecarWriter { +impl Write for SidecarWriter { fn write(&mut self, buf: &[u8]) -> Result { // We might get a short write into whatever writer is, so make sure the // sidecar writer doesn't get ahead in that case. @@ -98,6 +89,7 @@ impl Write for SidecarWriter { } fn flush(&mut self) -> Result<(), std::io::Error> { - self.writer.flush() + self.writer.flush()?; + self.sidecar.flush() } } diff --git a/facilitator/src/sample.rs b/facilitator/src/sample.rs index e76744027..703cf1058 100644 --- a/facilitator/src/sample.rs +++ b/facilitator/src/sample.rs @@ -160,7 +160,6 @@ mod tests { }, transport::LocalFileTransport, }; - use std::path::PathBuf; #[test] #[ignore] @@ -187,12 +186,10 @@ mod tests { 100, ); assert!(res.is_ok(), "error writing sample data {:?}", res.err()); - let mut expected_path = - PathBuf::from("fake-aggregation/fake-date").join(batch_uuid.to_string()); + let expected_path = format!("fake-aggregation/fake-date/{}.batch", batch_uuid); let transports = &[pha_transport, facilitator_transport]; for transport in transports { - expected_path.set_extension("batch"); let reader = transport.get(&expected_path); assert!(res.is_ok(), "error reading header back {:?}", res.err()); diff --git a/facilitator/src/transport.rs b/facilitator/src/transport.rs index 554368dba..2a5ae089f 100644 --- a/facilitator/src/transport.rs +++ b/facilitator/src/transport.rs @@ -1,18 +1,58 @@ use crate::Error; +use anyhow::{Context, Result}; +use derivative::Derivative; +use hyper_rustls::HttpsConnector; +use rusoto_core::credential::DefaultCredentialsProvider; +use rusoto_core::{ByteStream, Region}; +use rusoto_s3::{ + AbortMultipartUploadRequest, CompleteMultipartUploadRequest, CompletedMultipartUpload, + CompletedPart, CreateMultipartUploadRequest, GetObjectRequest, S3Client, UploadPartRequest, S3, +}; use std::boxed::Box; use std::fs::{create_dir_all, File}; use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; +use std::mem; +use std::path::{PathBuf, MAIN_SEPARATOR}; +use std::pin::Pin; +use std::time::Duration; +use tokio::{ + io::{AsyncRead, AsyncReadExt}, + runtime::{Builder, Runtime}, +}; + +/// A TransportWriter extends std::io::Write but adds methods that explicitly +/// allow callers to complete or cancel an upload. +pub trait TransportWriter: Write { + /// Complete an upload operation, flushing any buffered writes and cleaning + /// up any related resources. Callers must call this method or cancel_upload + /// when they are done with the TransportWriter. + fn complete_upload(&mut self) -> Result<()>; + + /// Cancel an upload operation, cleaning up any related resources. Callers + /// must call this method or complete_upload when they are done with the + /// Transportwriter. + fn cancel_upload(&mut self) -> Result<()>; +} + +impl TransportWriter for Box { + fn complete_upload(&mut self) -> Result<()> { + (**self).complete_upload() + } + + fn cancel_upload(&mut self) -> Result<()> { + (**self).cancel_upload() + } +} /// A transport moves object in and out of some data store, such as a cloud /// object store like Amazon S3, or local files, or buffers in memory. pub trait Transport { /// Returns an std::io::Read instance from which the contents of the value /// of the provided key may be read. - fn get(&self, key: &Path) -> Result, Error>; + fn get(&self, key: &str) -> Result>; /// Returns an std::io::Write instance into which the contents of the value /// may be written. - fn put(&mut self, key: &Path) -> Result, Error>; + fn put(&mut self, key: &str) -> Result>; } /// A transport implementation backed by the local filesystem. @@ -24,47 +64,295 @@ impl LocalFileTransport { /// Creates a LocalFileTransport under the specified path. The key parameter /// provided to `put` or `get` will be interpreted as a relative path. pub fn new(directory: PathBuf) -> LocalFileTransport { - LocalFileTransport { - directory: directory, - } + LocalFileTransport { directory } + } + + /// Callers will construct keys using "/" as a separator. This function + /// attempts to convert the provided key into a relative path valid for the + /// current platform. + fn relative_path(key: &str) -> PathBuf { + PathBuf::from(key.replace("/", &MAIN_SEPARATOR.to_string())) } } impl Transport for LocalFileTransport { - fn get(&self, key: &Path) -> Result, Error> { - let path = self.directory.join(key); - let f = File::open(path.as_path()) - .map_err(|e| Error::IoError(format!("opening {}", path.display()), e))?; + fn get(&self, key: &str) -> Result> { + let path = self.directory.join(LocalFileTransport::relative_path(key)); + let f = + File::open(path.as_path()).with_context(|| format!("opening {}", path.display()))?; Ok(Box::new(f)) } - fn put(&mut self, key: &Path) -> Result, Error> { - let path = self.directory.join(key); + fn put(&mut self, key: &str) -> Result> { + let path = self.directory.join(LocalFileTransport::relative_path(key)); if let Some(parent) = path.parent() { - create_dir_all(parent).map_err(|e| { - Error::IoError( - format!("creating parent directories {}", parent.display()), - e, - ) - })?; + create_dir_all(parent) + .with_context(|| format!("creating parent directories {}", parent.display()))?; } - let f = File::create(path.as_path()) - .map_err(|e| Error::IoError(format!("creating {}", path.display()), e))?; + let f = + File::create(path.as_path()).with_context(|| format!("creating {}", path.display()))?; Ok(Box::new(f)) } } -/// Trivial implementor of std::io::{Read, Write} used in NullTransport. -struct NullReadWriter; +impl TransportWriter for File { + fn complete_upload(&mut self) -> Result<()> { + // This method is a no-op for local files + Ok(()) + } + + fn cancel_upload(&mut self) -> Result<()> { + // This method is a no-op for local files + Ok(()) + } +} + +/// Constructs a basic runtime suitable for use in our single threaded context +fn basic_runtime() -> Result { + Ok(Builder::new().basic_scheduler().enable_all().build()?) +} + +/// Implementation of Transport that reads and writes objects from Amazon S3. +pub struct S3Transport { + region: Region, + bucket: String, + // client_provider allows injection of mock S3Client for testing purposes + client_provider: fn(&Region) -> S3Client, +} + +impl S3Transport { + pub fn new(region: Region, bucket: String) -> S3Transport { + S3Transport::new_with_client(region, bucket, |region| { + // Rusoto uses Hyper which uses connection pools. The default + // timeout for those connections is 90 seconds[1]. Amazon S3's API + // closes idle client connections after 20 seconds[2]. If we use a + // default client via S3Client::new, this mismatch causes uploads to + // fail when Hyper tries to re-use a connection that has been idle + // too long. Until this is fixed in Rusoto[3], we construct our own + // HTTP client dispatcher whose underlying hyper::Client is + // configured to timeout idle connections after 10 seconds. We could + // also implement retries on our layer[4]. + // + // [1]: https://docs.rs/hyper/0.13.8/hyper/client/struct.Builder.html#method.pool_idle_timeout + // [2]: https://aws.amazon.com/premiumsupport/knowledge-center/s3-socket-connection-timeout-error/ + // [3]: https://github.com/rusoto/rusoto/issues/1686 + // [4]: https://github.com/abetterinternet/prio-server/issues/41 + // + // Credentials for authenticating to AWS are automatically sourced + // from environment variables or ~/.aws/credentials. + // https://github.com/rusoto/rusoto/blob/master/AWS-CREDENTIALS.md + let credentials_provider = + DefaultCredentialsProvider::new().expect("failed to create credentials provider"); + + let mut builder = hyper::Client::builder(); + builder.pool_idle_timeout(Duration::from_secs(10)); + let connector = HttpsConnector::new(); + let http_client = rusoto_core::HttpClient::from_builder(builder, connector); + + S3Client::new_with(http_client, credentials_provider, region.clone()) + }) + } + + fn new_with_client( + region: Region, + bucket: String, + client_provider: fn(&Region) -> S3Client, + ) -> S3Transport { + S3Transport { + region, + bucket, + client_provider, + } + } +} + +impl Transport for S3Transport { + fn get(&self, key: &str) -> Result> { + let mut runtime = basic_runtime()?; + let client = (self.client_provider)(&self.region); + let get_output = runtime + .block_on(client.get_object(GetObjectRequest { + bucket: self.bucket.to_owned(), + key: key.to_string(), + ..Default::default() + })) + .context("error getting S3 object")?; + + let body = get_output.body.context("no body in GetObjectResponse")?; + + Ok(Box::new(StreamingBodyReader::new(body, runtime))) + } + + fn put(&mut self, key: &str) -> Result> { + Ok(Box::new(MultipartUploadWriter::new( + self.region.clone(), + self.bucket.to_owned(), + key.to_string(), + // Set buffer size to 5 MB, which is the minimum required by Amazon + // https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html + 5_242_880, + self.client_provider, + )?)) + } +} + +/// StreamingBodyReader is an std::io::Read implementation which reads from the +/// tokio::io::AsyncRead inside the StreamingBody in a Rusoto API request +/// response. +struct StreamingBodyReader { + body_reader: Pin>, + runtime: Runtime, +} + +impl StreamingBodyReader { + fn new(body: ByteStream, runtime: Runtime) -> StreamingBodyReader { + StreamingBodyReader { + body_reader: Box::pin(body.into_async_read()), + runtime: runtime, + } + } +} + +impl Read for StreamingBodyReader { + fn read(&mut self, buf: &mut [u8]) -> Result { + self.runtime.block_on(self.body_reader.read(buf)) + } +} + +/// MultipartUploadWriter is a TransportWriter implementation that uses AWS S3 +/// multi part uploads to permit streaming of objects into S3. On creation, it +/// initiates a multipart upload. It maintains a memory buffer into which it +/// writes the buffers passed by std::io::Write::write, and when there is more +/// than buffer_capacity bytes in it, performs an UploadPart call. On +/// TransportWrite::complete_upload, it calls CompleteMultipartUpload to finish +/// the upload. If any part of the upload fails, it cleans up by calling +/// AbortMultipartUpload as otherwise we would be billed for partial uploads. +/// https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html +#[derive(Derivative)] +#[derivative(Debug)] +struct MultipartUploadWriter { + runtime: Runtime, + #[derivative(Debug = "ignore")] + client: S3Client, + bucket: String, + key: String, + upload_id: String, + completed_parts: Vec, + minimum_upload_part_size: usize, + buffer: Vec, +} + +impl MultipartUploadWriter { + /// Creates a new MultipartUploadWriter with the provided parameters. A real + /// instance of this will fail if buffer_capacity is less than 5 MB, but we + /// allow smaller values for testing purposes. Larger values are also + /// acceptable but smaller values prevent excessive memory usage. + fn new( + region: Region, + bucket: String, + key: String, + minimum_upload_part_size: usize, + client_provider: fn(&Region) -> S3Client, + ) -> Result { + let mut runtime = basic_runtime()?; + let client = client_provider(®ion); + + let create_output = runtime + .block_on( + client.create_multipart_upload(CreateMultipartUploadRequest { + bucket: bucket.to_string(), + key: key.to_string(), + ..Default::default() + }), + ) + .context("error creating multipart upload")?; + + Ok(MultipartUploadWriter { + runtime: runtime, + client: client, + bucket: bucket, + key: key, + upload_id: create_output + .upload_id + .context("no upload ID in CreateMultipartUploadResponse")?, + completed_parts: Vec::new(), + // Upload parts must be at least buffer_capacity, but it's fine if + // they're bigger, so overprovision the buffer to make it unlikely + // that the caller will overflow it. + minimum_upload_part_size: minimum_upload_part_size, + buffer: Vec::with_capacity(minimum_upload_part_size * 2), + }) + } + + /// Upload content in internal buffer, if any, to S3 in an UploadPart call. + /// Returns std::io::Result because it is called by std::io::Write methods + /// and this makes it easier to use ? operator. + fn upload_part(&mut self) -> Result<()> { + if self.buffer.is_empty() { + return Ok(()); + } + + let part_number = (self.completed_parts.len() + 1) as i64; + + let upload_output = self + .runtime + .block_on( + self.client.upload_part(UploadPartRequest { + bucket: self.bucket.to_string(), + key: self.key.to_string(), + upload_id: self.upload_id.clone(), + part_number: part_number as i64, + body: Some( + // Move internal buffer into request object and replace + // it with a new, empty buffer. + mem::replace( + &mut self.buffer, + Vec::with_capacity(self.minimum_upload_part_size * 2), + ) + .into(), + ), + ..Default::default() + }), + ) + .context("failed to upload_part") + .or_else(|e| { + // Clean up botched uploads + if let Err(cancel) = self.cancel_upload() { + return Err(cancel.context(e)); + } + Err(e) + })?; + + let e_tag = upload_output + .e_tag + .context("no ETag in UploadPartOutput") + .or_else(|e| { + if let Err(cancel) = self.cancel_upload() { + return Err(cancel.context(e)); + } + Err(e) + })?; -impl Read for NullReadWriter { - fn read(&mut self, _buf: &mut [u8]) -> std::io::Result { - Ok(0) + let completed_part = CompletedPart { + e_tag: Some(e_tag), + part_number: Some(part_number), + }; + self.completed_parts.push(completed_part); + Ok(()) } } -impl Write for NullReadWriter { +impl Write for MultipartUploadWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { + // Write into memory buffer, and upload to S3 if we have accumulated + // enough content. + self.buffer.extend_from_slice(buf); + if self.buffer.len() >= self.minimum_upload_part_size { + self.upload_part().map_err(|e| { + std::io::Error::new(std::io::ErrorKind::Other, Error::AnyhowError(e)) + })?; + } + Ok(buf.len()) } @@ -73,26 +361,53 @@ impl Write for NullReadWriter { } } -/// Implementation of Transport that simply discards all writes and whose reads -/// always yield 0 bytes. Intended for use in tests, e.g. when a client is set -/// up but only needs to emit shares to one server. -pub struct NullTransport {} +impl TransportWriter for MultipartUploadWriter { + fn complete_upload(&mut self) -> Result<()> { + // Write last part, if any + self.upload_part()?; -impl Transport for NullTransport { - fn get(&self, _key: &Path) -> Result, Error> { - Ok(Box::new(NullReadWriter {})) + // Ignore output for now, but we might want the e_tag to check the + // digest + self.runtime + .block_on( + self.client + .complete_multipart_upload(CompleteMultipartUploadRequest { + bucket: self.bucket.to_string(), + key: self.key.to_string(), + upload_id: self.upload_id.clone(), + multipart_upload: Some(CompletedMultipartUpload { + parts: Some(mem::take(&mut self.completed_parts)), + }), + ..Default::default() + }), + ) + .context("error completing upload")?; + Ok(()) } - fn put(&mut self, _key: &Path) -> Result, Error> { - Ok(Box::new(NullReadWriter {})) + fn cancel_upload(&mut self) -> Result<()> { + // There's nothing useful in the output so discard it + self.runtime.block_on( + self.client + .abort_multipart_upload(AbortMultipartUploadRequest { + bucket: self.bucket.to_string(), + key: self.key.to_string(), + upload_id: self.upload_id.clone(), + ..Default::default() + }), + )?; + Ok(()) } } -// TODO: S3Transport; https://github.com/rusoto/rusoto - #[cfg(test)] mod tests { use super::*; + use rusoto_core::signature::SignedRequest; + use rusoto_mock::{ + MockCredentialsProvider, MockRequestDispatcher, MultipleMockRequestDispatcher, + }; + use rusoto_s3::CreateMultipartUploadError; use std::io::Read; #[test] @@ -100,17 +415,14 @@ mod tests { let tempdir = tempfile::TempDir::new().unwrap(); let mut file_transport = LocalFileTransport::new(tempdir.path().to_path_buf()); let content = vec![1, 2, 3, 4, 5, 6, 7, 8]; - let path = Path::new("path"); - let path2 = Path::new("path2"); - let complex_path = Path::new("path3/with/separators"); { - let ret = file_transport.get(&path2); + let ret = file_transport.get("path2"); assert!(ret.is_err(), "unexpected return value {:?}", ret.err()); } - for path in &[path, complex_path] { - let writer = file_transport.put(&path); + for path in &["path", "path3/with/separators"] { + let writer = file_transport.put(path); assert!(writer.is_ok(), "unexpected error {:?}", writer.err()); writer @@ -118,7 +430,7 @@ mod tests { .write_all(&content) .expect("failed to write"); - let reader = file_transport.get(&path); + let reader = file_transport.get(path); assert!(reader.is_ok(), "create reader failed: {:?}", reader.err()); let mut content_again = Vec::new(); @@ -130,33 +442,337 @@ mod tests { } } - #[test] - fn roundtrip_null_transport() { - let mut null_transport = NullTransport {}; - let content = vec![1, 2, 3, 4, 5, 6, 7, 8]; - let path = Path::new("path"); + // Rusoto provides us the ability to create mock clients and play canned + // responses to API requests. Besides that, we want to verify that we get + // the expected sequence of API requests, for instance to verify that we + // call AbortMultipartUpload if an UploadPart call fails. The mock client + // will crash if it runs out of canned responses to give the client, which + // will catch excess API calls. To make sure we issue the correct API calls, + // we use with_request_checker to examine the outgoing requests. We only get + // to examine a rusoto::signature::SignedRequest, which does not contain an + // explicit indication of which API call the HTTP request represents. So we + // have the `is_*_request` functions below, which compare HTTP method and + // URL parameters against the Amazon API specification to figure out what + // API call we are looking at. - let writer = null_transport.put(&path); - assert!(writer.is_ok(), "unexpected error {:?}", writer.err()); + const TEST_BUCKET: &str = "fake-bucket"; + const TEST_KEY: &str = "fake-key"; - writer - .unwrap() - .write_all(&content) - .expect("failed to write"); + fn is_create_multipart_upload_request(request: &SignedRequest) { + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html + assert_eq!( + request.method, "POST", + "expected CreateMultipartUpload request, found {:?}", + request + ); + assert!( + request.params.contains_key("uploads"), + "expected CreateMultipartUpload request, found {:?}", + request + ); + } - let reader = null_transport.get(&path); - assert!(reader.is_ok(), "create reader failed: {:?}", reader.err()); + fn is_upload_part_request(request: &SignedRequest) { + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html + assert_eq!( + request.method, "PUT", + "expected UploadPart request, found {:?}", + request + ); + assert!( + request.params.contains_key("partNumber"), + "expected UploadPart request, found {:?}", + request + ); + assert!( + request.params.contains_key("uploadId"), + "expected UploadPart request, found {:?}", + request + ); + } + + fn is_abort_multipart_upload_request(request: &SignedRequest) { + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html + assert_eq!( + request.method, "DELETE", + "expected AbortMultipartUpload request, found {:?}", + request + ); + assert!( + request.params.contains_key("uploadId"), + "expected AbortMultipartUpload request, found {:?}", + request + ); + } + + fn is_complete_multipart_upload_request(request: &SignedRequest) { + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html + assert_eq!( + request.method, "POST", + "expected CompleteMultipartUpload request, found {:?}", + request + ); + assert!( + request.params.contains_key("uploadId"), + "expected CompleteMultipartUpload request, found {:?}", + request + ); + } - let mut content_again = Vec::new(); - reader - .unwrap() - .read_to_end(&mut content_again) - .expect("failed to read"); + fn is_get_object_request(request: &SignedRequest) { + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html + assert_eq!( + request.method, "GET", + "expected GetObject request, found {:?}", + request + ); assert_eq!( - content_again.len(), - 0, - "vector unexpectedly contents: {:?}", - content_again + request.path, "/fake-bucket/fake-key", + "expected GetObject request, found {:?}", + request + ); + // Because we pass no options to get_options, our requests end up with + // no parameters, which is what we use to distinguish from e.g. + // GetObjectAcl with would have a param "acl" in it. + assert!( + request.params.is_empty(), + "expected GetObject request, found {:?}", + request ); } + + #[test] + fn multipart_upload_create_fails() { + let err = MultipartUploadWriter::new( + Region::UsWest2, + String::from(TEST_BUCKET), + String::from(TEST_KEY), + 50, + |region| { + S3Client::new_with( + MockRequestDispatcher::with_status(401) + .with_request_checker(is_create_multipart_upload_request), + MockCredentialsProvider, + region.clone(), + ) + }, + ) + .expect_err("expected error"); + assert!( + err.is::>(), + "found unexpected error {:?}", + err + ); + } + + #[test] + fn multipart_upload_create_no_upload_id() { + // Response body format from + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_Operations_Amazon_Simple_Storage_Service.html + MultipartUploadWriter::new( + Region::UsWest2, + String::from(TEST_BUCKET), + String::from(TEST_KEY), + 50, + |region| { + S3Client::new_with( + MockRequestDispatcher::with_status(200) + .with_body( + r#" + + fake-bucket + fake-key +"#, + ) + .with_request_checker(is_create_multipart_upload_request), + MockCredentialsProvider, + region.clone(), + ) + }, + ) + .expect_err("expected error"); + } + + #[test] + fn multipart_upload() { + // Response body format from + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_Operations_Amazon_Simple_Storage_Service.html + let mut writer = MultipartUploadWriter::new( + Region::UsWest2, + String::from(TEST_BUCKET), + String::from(TEST_KEY), + 50, + |region| { + let requests = vec![ + // Response to CreateMultipartUpload + MockRequestDispatcher::with_status(200) + .with_body( + r#" + + fake-bucket + fake-key + upload-id +"#, + ) + .with_request_checker(is_create_multipart_upload_request), + // Response to UploadPart. HTTP 401 will cause failure. + MockRequestDispatcher::with_status(401) + .with_request_checker(is_upload_part_request), + // Response to AbortMultipartUpload, expected because of + // previous UploadPart failure + MockRequestDispatcher::with_status(204) + .with_request_checker(is_abort_multipart_upload_request), + // Response to UploadPart. HTTP 200 but no ETag header will + // cause failure. + MockRequestDispatcher::with_status(200) + .with_request_checker(is_upload_part_request), + // Response to AbortMultipartUpload, expected because of + // previous UploadPart failure + MockRequestDispatcher::with_status(204) + .with_request_checker(is_abort_multipart_upload_request), + // Well formed response to UploadPart. + MockRequestDispatcher::with_status(200) + .with_request_checker(is_upload_part_request) + .with_header("ETag", "fake-etag"), + // Well formed response to UploadPart + MockRequestDispatcher::with_status(200) + .with_request_checker(is_upload_part_request) + .with_header("ETag", "fake-etag"), + // Well formed response to CompleteMultipartUpload + MockRequestDispatcher::with_status(200) + .with_request_checker(is_complete_multipart_upload_request) + .with_body( + r#" + + string + fake-bucket + fake-key + fake-etag +"#, + ), + // Well formed response to CompleteMultipartUpload + MockRequestDispatcher::with_status(200) + .with_request_checker(is_complete_multipart_upload_request) + .with_body( + r#" + + string + fake-bucket + fake-key + fake-etag +"#, + ), + // Failure response to CompleteMultipartUpload + MockRequestDispatcher::with_status(400) + .with_request_checker(is_complete_multipart_upload_request), + ]; + S3Client::new_with( + MultipleMockRequestDispatcher::new(requests), + MockCredentialsProvider, + region.clone(), + ) + }, + ) + .expect("failed to create multipart upload writer"); + + // First write will fail due to HTTP 401 + writer.write(&[0; 51]).unwrap_err(); + // Second write will fail because response is missing ETag + writer.write(&[0; 51]).unwrap_err(); + // Third write will work + writer.write(&[0; 51]).unwrap(); + // This write will put some content in the buffer, but not enough to + // cause an UploadPart + writer.write(&[0; 25]).unwrap(); + // Flush will cause writer to UploadPart the last part and then complete + // upload + writer.complete_upload().unwrap(); + // The buffer is empty now, so another flush will not cause an + // UploadPart call + writer.complete_upload().unwrap(); + writer.complete_upload().unwrap_err(); + } + + #[test] + fn roundtrip_s3_transport() { + let transport = + S3Transport::new_with_client(Region::UsWest2, TEST_BUCKET.to_string(), |region| { + S3Client::new_with( + // Failed GetObject request + MockRequestDispatcher::with_status(404) + .with_request_checker(is_get_object_request), + MockCredentialsProvider, + region.clone(), + ) + }); + + let ret = transport.get(TEST_KEY); + assert!(ret.is_err(), "unexpected return value {:?}", ret.err()); + + let transport = + S3Transport::new_with_client(Region::UsWest2, TEST_BUCKET.to_string(), |region| { + S3Client::new_with( + // Successful GetObject request + MockRequestDispatcher::with_status(200) + .with_request_checker(is_get_object_request) + .with_body("fake-content"), + MockCredentialsProvider, + region.clone(), + ) + }); + + let mut reader = transport + .get(TEST_KEY) + .expect("unexpected error getting reader"); + let mut content = Vec::new(); + reader.read_to_end(&mut content).expect("failed to read"); + assert_eq!(Vec::from("fake-content"), content); + + let mut transport = + S3Transport::new_with_client(Region::UsWest2, TEST_BUCKET.to_string(), |region| { + let requests = vec![ + // Response to CreateMultipartUpload + MockRequestDispatcher::with_status(200) + .with_body( + r#" + + fake-bucket + fake-key + upload-id +"#, + ) + .with_request_checker(is_create_multipart_upload_request), + // Well formed response to UploadPart + MockRequestDispatcher::with_status(200) + .with_request_checker(is_upload_part_request) + .with_header("ETag", "fake-etag"), + // Well formed response to CompleteMultipartUpload + MockRequestDispatcher::with_status(200) + .with_request_checker(is_complete_multipart_upload_request) + .with_body( + r#" + + string + fake-bucket + fake-key + fake-etag +"#, + ), + // Response to AbortMultipartUpload, expected because of + // cancel_upload call + MockRequestDispatcher::with_status(204) + .with_request_checker(is_abort_multipart_upload_request), + ]; + S3Client::new_with( + MultipleMockRequestDispatcher::new(requests), + MockCredentialsProvider, + region.clone(), + ) + }); + + let mut writer = transport.put(TEST_KEY).unwrap(); + writer.write_all(b"fake-content").unwrap(); + writer.complete_upload().unwrap(); + writer.cancel_upload().unwrap(); + } }