diff --git a/Cargo.lock b/Cargo.lock index cb5d491e9792b..115347044629f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,12 +79,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "antidote" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" - [[package]] name = "anyhow" version = "1.0.38" @@ -102,9 +96,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "0.4.8" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8" +checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73" [[package]] name = "arrayref" @@ -187,7 +181,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite 0.2.4", - "tokio 0.2.25", + "tokio 1.3.0", "zstd", "zstd-safe", ] @@ -235,9 +229,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "2.5.0" +version = "2.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82833e5a5a5c77b8f9ecbc003db996df7432e8afffd6ad8de2fb0dc1436d1f35" +checksum = "d44e9cd1eedaa85207eea845e59280f87d8892b9c2c8057712c9f110726fb571" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -275,9 +269,9 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "2.5.0" +version = "2.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ddce4448d5f5f7c5a8ace5f8273725cb904011df9b8a8933345aad333fcbfbd" +checksum = "91b828ac0ddb654259a9cf3aa0101bc47428e8798d97690f188f0bdd6fddd431" dependencies = [ "Inflector", "async-graphql-parser", @@ -291,9 +285,9 @@ dependencies = [ [[package]] name = "async-graphql-parser" -version = "2.5.4" +version = "2.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "274680c34fda08547d1f4f6268f1e57b2b8e8b9f96d160975c915f1ff955137d" +checksum = "53e2f7bc6173bf7a30def040a0b541d87cb044da90ec3c8d23de7b47185379ea" dependencies = [ "async-graphql-value", "pest", @@ -304,9 +298,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "2.0.5" +version = "2.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d3aa3cd3696ffd8decb10f5053affc78cb33ecfc545e480072bbc600e6723d" +checksum = "3f8134ed15d6ebe6025977026a9dcfac18f94c99d762fccc2df6a4a7f7f1226f" dependencies = [ "serde", "serde_json", @@ -314,9 +308,9 @@ dependencies = [ [[package]] name = "async-graphql-warp" -version = "2.5.0" +version = "2.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e05242b044bfb1bebf1a3f7e980f8f408d421a60df1d3fd6831b8fe8a2d43416" +checksum = "0d56a0a9045ef99ab5a2c966789ad19aac27dc99c746a1110ef0ea91972ca04b" dependencies = [ "async-graphql", "futures-util", @@ -743,45 +737,44 @@ dependencies = [ [[package]] name = "bollard" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcee456f2103b858b1ca8eb2edd35e413ebba1f32f04b1db52cd366b9ad4fe40" +checksum = "699194c00f3a2effd3358d47f880646818e3d483190b17ebcdf598c654fb77e9" dependencies = [ - "base64 0.12.3", + "base64 0.13.0", "bollard-stubs", - "bytes 0.5.6", + "bytes 1.0.1", "chrono", "ct-logs", - "dirs-next 1.0.2", + "dirs-next", "futures-core", "futures-util", "hex", "http", - "hyper 0.13.10", - "hyper-rustls", + "hyper 0.14.4", + "hyper-rustls 0.22.1", "hyper-unix-connector", "log", - "mio-named-pipes", - "pin-project 0.4.27", - "rustls 0.18.1", - "rustls-native-certs 0.4.0", + "pin-project 1.0.6", + "rustls 0.19.0", + "rustls-native-certs", "serde", "serde_derive", "serde_json", - "serde_urlencoded 0.6.1", + "serde_urlencoded 0.7.0", "thiserror", - "tokio 0.2.25", - "tokio-util", + "tokio 1.3.0", + "tokio-util 0.6.3", "url", - "webpki-roots 0.20.0", + "webpki-roots 0.21.0", "winapi 0.3.9", ] [[package]] name = "bollard-stubs" -version = "1.40.6" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf72b3eeb9a5cce41979def2c7522cb830356c0621ca29c0b766128c4e7fded" +checksum = "ed2f2e73fffe9455141e170fb9c1feb0ac521ec7e7dcd47a7cab72a658490fb8" dependencies = [ "chrono", "serde", @@ -867,15 +860,15 @@ name = "bytes" version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" -dependencies = [ - "serde", -] [[package]] name = "bytes" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" +dependencies = [ + "serde", +] [[package]] name = "bytesize" @@ -1040,9 +1033,9 @@ dependencies = [ name = "codec" version = "0.1.0" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "serde_json", - "tokio-util", + "tokio-util 0.6.3", "tracing 0.1.23", ] @@ -1123,32 +1116,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" -[[package]] -name = "core-foundation" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" -dependencies = [ - "core-foundation-sys 0.7.0", - "libc", -] - [[package]] name = "core-foundation" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" dependencies = [ - "core-foundation-sys 0.8.2", + "core-foundation-sys", "libc", ] -[[package]] -name = "core-foundation-sys" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" - [[package]] name = "core-foundation-sys" version = "0.8.2" @@ -1512,16 +1489,6 @@ dependencies = [ "subtle 1.0.0", ] -[[package]] -name = "crypto-mac" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" -dependencies = [ - "generic-array 0.14.4", - "subtle 2.4.0", -] - [[package]] name = "crypto-mac" version = "0.9.1" @@ -1566,9 +1533,9 @@ dependencies = [ [[package]] name = "ct-logs" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c8e13110a84b6315df212c045be706af261fd364791cad863285439ebba672e" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" dependencies = [ "sct", ] @@ -1594,7 +1561,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "socket2", + "socket2 0.3.19", "winapi 0.3.9", ] @@ -1849,26 +1816,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "dirs" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" -dependencies = [ - "cfg-if 0.1.10", - "dirs-sys", -] - -[[package]] -name = "dirs-next" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf36e65a80337bea855cd4ef9b8401ffce06a7baedf2e85ec467b1ac3f6e82b6" -dependencies = [ - "cfg-if 1.0.0", - "dirs-sys-next", -] - [[package]] name = "dirs-next" version = "2.0.0" @@ -1879,17 +1826,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" -dependencies = [ - "libc", - "redox_users 0.3.5", - "winapi 0.3.9", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -2067,10 +2003,9 @@ checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" [[package]] name = "evmap" version = "10.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e3ea06a83f97d3dc2eb06e51e7a729b418f0717a5558a5c870e3d5156dc558d" +source = "git+https://github.com/lukesteensen/evmap.git?rev=45ba973c22715a68c5e99efad4b072421f7ad40b#45ba973c22715a68c5e99efad4b072421f7ad40b" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "hashbag", "slab", "smallvec", @@ -2150,7 +2085,7 @@ name = "file-source" version = "0.1.0" dependencies = [ "bstr", - "bytes 0.5.6", + "bytes 1.0.1", "chrono", "crc", "criterion", @@ -2165,7 +2100,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "tokio 0.2.25", + "tokio 1.3.0", "tracing 0.1.23", "winapi 0.3.9", ] @@ -2546,21 +2481,21 @@ dependencies = [ [[package]] name = "goauth" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c55b7ac37895bd6e4ca0b357c074248358c95e20cf1cf2b462603121f7b87" +checksum = "d94101e84ede813c04773b0a43396c01b5a3a9376537dbce1125858ae090ae60" dependencies = [ "arc-swap", "futures 0.3.13", "log", - "reqwest", + "reqwest 0.11.1", "serde", "serde_derive", "serde_json", "simpl", "smpl_jwt", "time 0.2.25", - "tokio 0.2.25", + "tokio 1.3.0", ] [[package]] @@ -2649,11 +2584,30 @@ dependencies = [ "indexmap", "slab", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", "tracing 0.1.23", "tracing-futures 0.2.5", ] +[[package]] +name = "h2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78" +dependencies = [ + "bytes 1.0.1", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.3.0", + "tokio-util 0.6.3", + "tracing 0.1.23", +] + [[package]] name = "half" version = "1.7.1" @@ -2734,7 +2688,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767e6e47cf88abe7c9a5ebb4df82f180d30d9c0ba0269b6d166482461765834" dependencies = [ "cfg-if 1.0.0", - "core-foundation 0.9.1", + "core-foundation", "futures-core", "futures-util", "lazy_static", @@ -2773,7 +2727,7 @@ checksum = "75603ff3868851c04954ee86bf610a6bd45be2732a0e81c35fd72b2b90fa4718" dependencies = [ "bitflags", "cfg-if 1.0.0", - "core-foundation 0.9.1", + "core-foundation", "heim-common", "heim-runtime", "libc", @@ -2916,16 +2870,6 @@ dependencies = [ "digest 0.8.1", ] -[[package]] -name = "hmac" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" -dependencies = [ - "crypto-mac 0.8.0", - "digest 0.9.0", -] - [[package]] name = "hmac" version = "0.9.0" @@ -3024,7 +2968,7 @@ dependencies = [ "serde", "serde_json", "serde_regex", - "tokio 1.4.0", + "tokio 1.3.0", ] [[package]] @@ -3045,20 +2989,21 @@ dependencies = [ [[package]] name = "hyper" version = "0.13.10" -source = "git+https://github.com/hyperium/hyper?rev=d7495a75abca34646b1d6d047589c1b8110d0fa5#d7495a75abca34646b1d6d047589c1b8110d0fa5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" dependencies = [ "bytes 0.5.6", "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.2.7", "http", "http-body 0.3.1", "httparse", "httpdate", "itoa", "pin-project 1.0.6", - "socket2", + "socket2 0.3.19", "tokio 0.2.25", "tower-service", "tracing 0.1.23", @@ -3075,14 +3020,15 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2 0.3.1", "http", "http-body 0.4.0", "httparse", "httpdate", "itoa", "pin-project 1.0.6", - "socket2", - "tokio 1.4.0", + "socket2 0.3.19", + "tokio 1.3.0", "tower-service", "tracing 0.1.23", "want", @@ -3090,20 +3036,19 @@ dependencies = [ [[package]] name = "hyper-openssl" -version = "0.8.1" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90705b797966f4774ffbc5e4dbda9e5b95f2a49991a3187f753171cdd7db3c58" +checksum = "a9d52322a69f0a93f177d76ca82073fcec8d5b4eb6e28525d5b3142fa718195c" dependencies = [ - "antidote", - "bytes 0.5.6", "http", - "hyper 0.13.10", + "hyper 0.14.4", "linked_hash_set", "once_cell", "openssl", "openssl-sys", - "tokio 0.2.25", - "tokio-openssl", + "parking_lot", + "tokio 1.3.0", + "tokio-openssl 0.6.1", "tower-layer", ] @@ -3114,17 +3059,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37743cc83e8ee85eacfce90f2f4102030d9ff0a95244098d781e9bee4a90abb6" dependencies = [ "bytes 0.5.6", - "ct-logs", "futures-util", "hyper 0.13.10", "log", "rustls 0.18.1", - "rustls-native-certs 0.4.0", "tokio 0.2.25", "tokio-rustls 0.14.1", "webpki", ] +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper 0.14.4", + "log", + "rustls 0.19.0", + "rustls-native-certs", + "tokio 1.3.0", + "tokio-rustls 0.22.0", + "webpki", +] + [[package]] name = "hyper-tls" version = "0.4.3" @@ -3138,18 +3098,30 @@ dependencies = [ "tokio-tls", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes 1.0.1", + "hyper 0.14.4", + "native-tls", + "tokio 1.3.0", + "tokio-native-tls 0.3.0", +] + [[package]] name = "hyper-unix-connector" -version = "0.1.5" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42b66be14087ec25c5150c9d1228a1e9bbbfe7fe2506ff85daed350724980319" +checksum = "24ef1fd95d34b4ff007d3f0590727b5cf33572cace09b42032fc817dc8b16557" dependencies = [ "anyhow", - "futures-util", "hex", - "hyper 0.13.10", - "pin-project 0.4.27", - "tokio 0.2.25", + "hyper 0.14.4", + "pin-project 1.0.6", + "tokio 1.3.0", ] [[package]] @@ -3211,11 +3183,11 @@ dependencies = [ [[package]] name = "input_buffer" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a8a95243d5a0398cae618ec29477c6e3cb631152be5c19481f80bc71559754" +checksum = "f97967975f448f1a7ddb12b0bc41069d09ed6a1c161a92687e057325db35d413" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", ] [[package]] @@ -3264,7 +3236,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" dependencies = [ - "socket2", + "socket2 0.3.19", "widestring", "winapi 0.3.9", "winreg 0.6.2", @@ -3300,15 +3272,6 @@ dependencies = [ "waker-fn", ] -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.9.0" @@ -3387,9 +3350,9 @@ dependencies = [ "k8s-openapi", "k8s-test-framework", "regex", - "reqwest", + "reqwest 0.10.10", "serde_json", - "tokio 0.2.25", + "tokio 1.3.0", ] [[package]] @@ -3417,7 +3380,7 @@ dependencies = [ "once_cell", "serde_json", "tempfile", - "tokio 0.2.25", + "tokio 1.3.0", ] [[package]] @@ -4162,7 +4125,7 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" dependencies = [ - "socket2", + "socket2 0.3.19", "winapi 0.3.9", ] @@ -4194,14 +4157,14 @@ dependencies = [ "pbkdf2 0.3.0", "percent-encoding", "rand 0.7.3", - "reqwest", + "reqwest 0.10.10", "rustls 0.17.0", "serde", "serde_bytes", "serde_with", "sha-1 0.8.2", "sha2 0.8.2", - "socket2", + "socket2 0.3.19", "stringprep", "strsim 0.10.0", "take_mut", @@ -4255,8 +4218,8 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework 2.0.0", - "security-framework-sys 2.0.0", + "security-framework", + "security-framework-sys", "tempfile", ] @@ -4280,7 +4243,7 @@ dependencies = [ "parking_lot", "regex", "rustls 0.19.0", - "rustls-native-certs 0.5.0", + "rustls-native-certs", "webpki", "winapi 0.3.9", ] @@ -4292,7 +4255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "670361df1bc2399ee1ff50406a0d422587dd3bb0da596e1978fe8e05dabddf4f" dependencies = [ "libc", - "socket2", + "socket2 0.3.19", ] [[package]] @@ -4633,9 +4596,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-src" -version = "111.15.0+1.1.1k" +version = "111.13.0+1.1.1i" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a5f6ae2ac04393b217ea9f700cd04fa9bf3d93fae2872069f3d15d908af70a" +checksum = "045e4dc48af57aad93d665885789b43222ae26f4886494da12d1ed58d309dcb6" dependencies = [ "cc", ] @@ -4985,7 +4948,7 @@ dependencies = [ "futures 0.3.13", "openssl", "tokio 0.2.25", - "tokio-openssl", + "tokio-openssl 0.4.0", "tokio-postgres", ] @@ -5164,9 +5127,9 @@ dependencies = [ "indexmap", "nom 6.1.2", "num_enum", - "prost 0.6.1", - "prost-build 0.6.1", - "prost-types 0.6.1", + "prost", + "prost-build", + "prost-types", "shared", "snafu", ] @@ -5202,16 +5165,6 @@ dependencies = [ "vrl-parser", ] -[[package]] -name = "prost" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce49aefe0a6144a45de32927c77bd2859a5f7677b55f220ae5b744e87389c212" -dependencies = [ - "bytes 0.5.6", - "prost-derive 0.6.1", -] - [[package]] name = "prost" version = "0.7.0" @@ -5219,25 +5172,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e6984d2f1a23009bd270b8bb56d0926810a3d483f59c987d77969e9d8e840b2" dependencies = [ "bytes 1.0.1", - "prost-derive 0.7.0", -] - -[[package]] -name = "prost-build" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26" -dependencies = [ - "bytes 0.5.6", - "heck", - "itertools 0.8.2", - "log", - "multimap", - "petgraph", - "prost 0.6.1", - "prost-types 0.6.1", - "tempfile", - "which 3.1.1", + "prost-derive", ] [[package]] @@ -5252,23 +5187,10 @@ dependencies = [ "log", "multimap", "petgraph", - "prost 0.7.0", - "prost-types 0.7.0", + "prost", + "prost-types", "tempfile", - "which 4.0.2", -] - -[[package]] -name = "prost-derive" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537aa19b95acde10a12fec4301466386f757403de4cd4e5b4fa78fb5ecb18f72" -dependencies = [ - "anyhow", - "itertools 0.8.2", - "proc-macro2 1.0.24", - "quote 1.0.9", - "syn 1.0.62", + "which", ] [[package]] @@ -5284,16 +5206,6 @@ dependencies = [ "syn 1.0.62", ] -[[package]] -name = "prost-types" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1834f67c0697c001304b75be76f67add9c89742eda3a085ad8ee0bb38c3417aa" -dependencies = [ - "bytes 0.5.6", - "prost 0.6.1", -] - [[package]] name = "prost-types" version = "0.7.0" @@ -5301,7 +5213,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b518d7cdd93dab1d1122cf07fa9a60771836c668dde9d9e2a139f957f0d9f1bb" dependencies = [ "bytes 1.0.1", - "prost 0.7.0", + "prost", ] [[package]] @@ -5321,14 +5233,14 @@ dependencies = [ "native-tls", "nom 6.1.2", "pem", - "prost 0.7.0", - "prost-build 0.7.0", - "prost-derive 0.7.0", + "prost", + "prost-build", + "prost-derive", "rand 0.8.3", "regex", "tokio 0.2.25", - "tokio-native-tls", - "tokio-util", + "tokio-native-tls 0.1.0", + "tokio-util 0.3.1", "url", ] @@ -5673,9 +5585,9 @@ dependencies = [ [[package]] name = "rdkafka" -version = "0.24.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db594dc221933be6f2ad804b997b48a57a63436c26ab924222c28e9a36ad210a" +checksum = "af78bc431a82ef178c4ad6db537eb9cc25715a8591d27acc30455ee7227a76f4" dependencies = [ "futures 0.3.13", "libc", @@ -5684,14 +5596,15 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "tokio 0.2.25", + "slab", + "tokio 1.3.0", ] [[package]] name = "rdkafka-sys" -version = "2.1.0+1.5.0" +version = "4.0.0+1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d3f17044cba41c7309facedc72ca9bf25f177bf1e06756318e010f043713017" +checksum = "54f24572851adfeb525fdc4a1d51185898e54fed4e8d8dba4fadb90c6b4f0422" dependencies = [ "cmake", "libc", @@ -5809,8 +5722,8 @@ dependencies = [ "http", "http-body 0.3.1", "hyper 0.13.10", - "hyper-rustls", - "hyper-tls", + "hyper-rustls 0.21.0", + "hyper-tls 0.4.3", "ipnet", "js-sys", "lazy_static", @@ -5835,6 +5748,41 @@ dependencies = [ "winreg 0.7.0", ] +[[package]] +name = "reqwest" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0460542b551950620a3648c6aa23318ac6b3cd779114bd873209e6e8b5eb1c34" +dependencies = [ + "base64 0.13.0", + "bytes 1.0.1", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body 0.4.0", + "hyper 0.14.4", + "hyper-tls 0.5.0", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite 0.2.4", + "serde", + "serde_json", + "serde_urlencoded 0.7.0", + "tokio 1.3.0", + "tokio-native-tls 0.3.0", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.7.0", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -5891,12 +5839,12 @@ dependencies = [ [[package]] name = "rusoto_cloudwatch" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5057642d3b77125bdae53ac923ed46c931a4387a9409d192dedd6ae03926dc6" +checksum = "17dc69132670b25e3cc2f5cdb3a56605615c19f563be3d6ee5e359ccfa400fa8" dependencies = [ "async-trait", - "bytes 0.5.6", + "bytes 1.0.1", "futures 0.3.13", "rusoto_core", "serde_urlencoded 0.6.1", @@ -5905,61 +5853,56 @@ dependencies = [ [[package]] name = "rusoto_core" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e977941ee0658df96fca7291ecc6fc9a754600b21ad84b959eb1dbbc9d5abcc7" +checksum = "02aff20978970d47630f08de5f0d04799497818d16cafee5aec90c4b4d0806cf" dependencies = [ "async-trait", - "base64 0.12.3", - "bytes 0.5.6", + "base64 0.13.0", + "bytes 1.0.1", "crc32fast", "flate2", "futures 0.3.13", "http", - "hyper 0.13.10", - "hyper-tls", + "hyper 0.14.4", + "hyper-tls 0.5.0", "lazy_static", "log", - "md5", - "percent-encoding", - "pin-project 0.4.27", "rusoto_credential", "rusoto_signature", "rustc_version", "serde", "serde_json", - "tokio 0.2.25", + "tokio 1.3.0", "xml-rs", ] [[package]] name = "rusoto_credential" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ac05563f83489b19b4d413607a30821ab08bbd9007d14fa05618da3ef09d8b" +checksum = "8e91e4c25ea8bfa6247684ff635299015845113baaa93ba8169b9e565701b58e" dependencies = [ "async-trait", "chrono", - "dirs 2.0.2", + "dirs-next", "futures 0.3.13", - "hyper 0.13.10", - "pin-project 0.4.27", - "regex", + "hyper 0.14.4", "serde", "serde_json", "shlex", - "tokio 0.2.25", + "tokio 1.3.0", "zeroize", ] [[package]] name = "rusoto_es" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12a49c8259ada24cb2c64d2a2aee0ca150eed3aef91d102063d8efca747ccaa3" +checksum = "b3b437c8c6fe3247f91de943997db5fcae5e0a1162c524916ea7789634c599b3" dependencies = [ "async-trait", - "bytes 0.5.6", + "bytes 1.0.1", "futures 0.3.13", "rusoto_core", "serde", @@ -5969,12 +5912,12 @@ dependencies = [ [[package]] name = "rusoto_firehose" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3c0c212bda09bad1e6b34cf2cdc9b6960214ff52935300567984c80389e29a" +checksum = "cd6eb649178c314d7a6b5dc9ce0d8b4d094cc2a4039ad91753595c19a3bbe2b6" dependencies = [ "async-trait", - "bytes 0.5.6", + "bytes 1.0.1", "futures 0.3.13", "rusoto_core", "serde", @@ -5983,12 +5926,12 @@ dependencies = [ [[package]] name = "rusoto_kinesis" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb8a3baabc67851ca472d25c9fe848b229969a1412cbc50ad00ce3af00e9aeae" +checksum = "15603c1ee187cd789647e2b3ec2e9bec259b247b4ed04f6e6002616b39bc9d51" dependencies = [ "async-trait", - "bytes 0.5.6", + "bytes 1.0.1", "futures 0.3.13", "rusoto_core", "serde", @@ -5997,12 +5940,12 @@ dependencies = [ [[package]] name = "rusoto_logs" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47079cf81cf7301d96195e495f2649cd731bedfe592925741997eb5a02d253e6" +checksum = "df86e14b61ecefc43b92603c9080db61d598d71c166bab94e8af09b09265c72b" dependencies = [ "async-trait", - "bytes 0.5.6", + "bytes 1.0.1", "futures 0.3.13", "rusoto_core", "serde", @@ -6011,12 +5954,12 @@ dependencies = [ [[package]] name = "rusoto_s3" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1146e37a7c1df56471ea67825fe09bbbd37984b5f6e201d8b2e0be4ee15643d8" +checksum = "abc3f56f14ccf91f880b9a9c2d0556d8523e8c155041c54db155b384a1dd1119" dependencies = [ "async-trait", - "bytes 0.5.6", + "bytes 1.0.1", "futures 0.3.13", "rusoto_core", "xml-rs", @@ -6024,37 +5967,37 @@ dependencies = [ [[package]] name = "rusoto_signature" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a740a88dde8ded81b6f2cff9cd5e054a5a2e38a38397260f7acdd2c85d17dd" +checksum = "5486e6b1673ab3e0ba1ded284fb444845fe1b7f41d13989a54dd60f62a7b2baa" dependencies = [ - "base64 0.12.3", - "bytes 0.5.6", + "base64 0.13.0", + "bytes 1.0.1", "futures 0.3.13", "hex", - "hmac 0.8.1", + "hmac 0.10.1", "http", - "hyper 0.13.10", + "hyper 0.14.4", "log", "md5", "percent-encoding", - "pin-project 0.4.27", + "pin-project-lite 0.2.4", "rusoto_credential", "rustc_version", "serde", "sha2 0.9.3", "time 0.2.25", - "tokio 0.2.25", + "tokio 1.3.0", ] [[package]] name = "rusoto_sqs" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcd228a1b4ce3f3a40541ee8cef526ff3702b58a4779fb05c31e739174efda5e" +checksum = "054901b35dd1ed5f7be0e490a9d3476dc0b236ca8383fe600d3fe851e0a32e67" dependencies = [ "async-trait", - "bytes 0.5.6", + "bytes 1.0.1", "futures 0.3.13", "rusoto_core", "serde_urlencoded 0.6.1", @@ -6063,17 +6006,16 @@ dependencies = [ [[package]] name = "rusoto_sts" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3815b8c0fc1c50caf9e87603f23daadfedb18d854de287b361c69f68dc9d49e0" +checksum = "2f93005e0c3b9e40a424b50ca71886d2445cc19bb6cdac3ac84c2daff482eb59" dependencies = [ "async-trait", - "bytes 0.5.6", + "bytes 1.0.1", "chrono", "futures 0.3.13", "rusoto_core", "serde_urlencoded 0.6.1", - "tempfile", "xml-rs", ] @@ -6160,18 +6102,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "rustls-native-certs" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629d439a7672da82dd955498445e496ee2096fe2117b9f796558a43fdb9e59b8" -dependencies = [ - "openssl-probe", - "rustls 0.18.1", - "schannel", - "security-framework 1.0.0", -] - [[package]] name = "rustls-native-certs" version = "0.5.0" @@ -6181,7 +6111,7 @@ dependencies = [ "openssl-probe", "rustls 0.19.0", "schannel", - "security-framework 2.0.0", + "security-framework", ] [[package]] @@ -6318,19 +6248,6 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" -[[package]] -name = "security-framework" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad502866817f0575705bd7be36e2b2535cc33262d493aa733a2ec862baa2bc2b" -dependencies = [ - "bitflags", - "core-foundation 0.7.0", - "core-foundation-sys 0.7.0", - "libc", - "security-framework-sys 1.0.0", -] - [[package]] name = "security-framework" version = "2.0.0" @@ -6338,20 +6255,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69" dependencies = [ "bitflags", - "core-foundation 0.9.1", - "core-foundation-sys 0.8.2", - "libc", - "security-framework-sys 2.0.0", -] - -[[package]] -name = "security-framework-sys" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51ceb04988b17b6d1dcd555390fa822ca5637b4a14e1f5099f13d351bed4d6c7" -dependencies = [ - "core-foundation-sys 0.7.0", + "core-foundation", + "core-foundation-sys", "libc", + "security-framework-sys", ] [[package]] @@ -6360,7 +6267,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b" dependencies = [ - "core-foundation-sys 0.8.2", + "core-foundation-sys", "libc", ] @@ -6619,7 +6526,7 @@ dependencies = [ name = "shared" version = "0.1.0" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "chrono", "chrono-tz", "derivative 2.2.0", @@ -6754,11 +6661,11 @@ dependencies = [ [[package]] name = "smpl_jwt" -version = "0.5.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "547e9c1059500ce0fe6cfa325f868b5621214957922be60a49d86e3e844ee9dc" +checksum = "4370044f8b20f944e05c35d77edd3518e6f21fc4de77e593919f287c6a3f428a" dependencies = [ - "base64 0.12.3", + "base64 0.13.0", "log", "openssl", "serde", @@ -6809,6 +6716,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "socket2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "spin" version = "0.5.2" @@ -6896,14 +6813,13 @@ checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" [[package]] name = "stream-cancel" -version = "0.6.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcbeca004dfaec7b6fd818d8ae6359eaea21770d134f137c4cb8fb5fa92b5a33" +checksum = "f36848ff9e3e8af125e00ab244aca7af0a8b270d4c6afcc9ccb4e523f7972c4c" dependencies = [ "futures-core", - "futures-util", - "pin-project 0.4.27", - "tokio 0.2.25", + "pin-project 1.0.6", + "tokio 1.3.0", ] [[package]] @@ -7135,7 +7051,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" dependencies = [ "byteorder", - "dirs 1.0.5", + "dirs", "winapi 0.3.9", ] @@ -7192,11 +7108,11 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.3" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" dependencies = [ - "once_cell", + "lazy_static", ] [[package]] @@ -7308,15 +7224,18 @@ dependencies = [ [[package]] name = "tokio" -version = "1.4.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134af885d758d645f0f0505c9a8b3f9bf8a348fd822e112ab5248138348f1722" +checksum = "8d56477f6ed99e10225f38f9f75f872f29b8b8bd8c0b946f63345bb144e9eeda" dependencies = [ "autocfg 1.0.1", + "bytes 1.0.1", "libc", + "memchr", "mio 0.7.7", "num_cpus", "once_cell", + "parking_lot", "pin-project-lite 0.2.4", "signal-hook-registry", "tokio-macros 1.1.0", @@ -7376,6 +7295,16 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio 1.3.0", +] + [[package]] name = "tokio-openssl" version = "0.4.0" @@ -7386,6 +7315,18 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-openssl" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1bec5c0a4aa71e3459802c7a12e8912c2091ce2151004f9ce95cc5d1c6124e" +dependencies = [ + "futures 0.3.13", + "openssl", + "pin-project 1.0.6", + "tokio 1.3.0", +] + [[package]] name = "tokio-postgres" version = "0.5.5" @@ -7405,7 +7346,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", ] [[package]] @@ -7433,37 +7374,38 @@ dependencies = [ ] [[package]] -name = "tokio-stream" -version = "0.1.3" +name = "tokio-rustls" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ - "futures-core", - "pin-project-lite 0.2.4", - "tokio 1.4.0", + "rustls 0.19.0", + "tokio 1.3.0", + "webpki", ] [[package]] -name = "tokio-test" -version = "0.2.1" +name = "tokio-stream" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed0049c119b6d505c4447f5c64873636c7af6c75ab0d45fd9f618d82acb8016d" +checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea" dependencies = [ - "bytes 0.5.6", "futures-core", - "tokio 0.2.25", + "pin-project-lite 0.2.4", + "tokio 1.3.0", + "tokio-util 0.6.3", ] [[package]] name = "tokio-test" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f58403903e94d4bc56805e46597fced893410b2e753e229d3f7f22423ea03f67" +checksum = "7c7d205f6f59b03f9e824ac86eaba635a98395f287756ecc8a06464779c399bf" dependencies = [ "async-stream", "bytes 1.0.1", "futures-core", - "tokio 1.4.0", + "tokio 1.3.0", "tokio-stream", ] @@ -7491,16 +7433,16 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e878ad426ca286e4dcae09cbd4e1973a7f8987d97570e2469703dd7f5720c" +checksum = "e1a5f475f1b9d077ea1017ecbc60890fda8e54942d680ca0b1d2b47cfa2d861b" dependencies = [ "futures-util", "log", "native-tls", - "pin-project 0.4.27", - "tokio 0.2.25", - "tokio-native-tls", + "pin-project 1.0.6", + "tokio 1.3.0", + "tokio-native-tls 0.3.0", "tungstenite", ] @@ -7518,6 +7460,21 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-util" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.4", + "slab", + "tokio 1.3.0", +] + [[package]] name = "tokio01-test" version = "0.1.1" @@ -7540,13 +7497,15 @@ dependencies = [ [[package]] name = "tower" -version = "0.3.1" -source = "git+https://github.com/tower-rs/tower?rev=43168944220ed32dab83cb4f11f7b97abc5818d5#43168944220ed32dab83cb4f11f7b97abc5818d5" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f715efe02c0862926eb463e49368d38ddb119383475686178e32e26d15d06a66" dependencies = [ "futures-core", "futures-util", - "pin-project 0.4.27", - "tokio 0.2.25", + "pin-project 1.0.6", + "tokio 1.3.0", + "tokio-util 0.6.3", "tower-layer", "tower-service", "tracing 0.1.23", @@ -7554,9 +7513,9 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35d656f2638b288b33495d1053ea74c40dc05ec0b92084dd71ca5566c4ed1dc" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" [[package]] name = "tower-make" @@ -7575,14 +7534,14 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tower-test" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba4bbc2c1e4a8543c30d4c13a4c8314ed72d6e07581910f665aa13fde0153c8" +checksum = "a4546773ffeab9e4ea02b8872faa49bb616a80a7da66afc2f32688943f97efa7" dependencies = [ "futures-util", - "pin-project 0.4.27", - "tokio 0.2.25", - "tokio-test 0.2.1", + "pin-project 1.0.6", + "tokio 1.3.0", + "tokio-test", "tower-layer", "tower-service", ] @@ -7808,19 +7767,19 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0308d80d86700c5878b9ef6321f020f29b1bb9d5ff3cab25e75e23f3a492a23" +checksum = "8ada8297e8d70872fa9a551d93250a9f407beb9f37ef86494eb20012a2ff7c24" dependencies = [ - "base64 0.12.3", + "base64 0.13.0", "byteorder", - "bytes 0.5.6", + "bytes 1.0.1", "http", "httparse", "input_buffer", "log", "native-tls", - "rand 0.7.3", + "rand 0.8.3", "sha-1 0.9.4", "url", "utf-8", @@ -8006,12 +7965,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "urlencoding" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" - [[package]] name = "userfaultfd" version = "0.2.1" @@ -8119,7 +8072,7 @@ dependencies = [ "bloom", "bollard", "built", - "bytes 0.5.6", + "bytes 1.0.1", "bytesize", "chrono", "cidr-utils", @@ -8131,7 +8084,7 @@ dependencies = [ "db-key", "derivative 2.2.0", "derive_is_enum_variant", - "dirs-next 2.0.0", + "dirs-next", "dyn-clone", "encoding_rs", "evmap", @@ -8150,7 +8103,7 @@ dependencies = [ "hostname", "http", "httpmock", - "hyper 0.13.10", + "hyper 0.14.4", "hyper-openssl", "indexmap", "indoc", @@ -8192,15 +8145,15 @@ dependencies = [ "postgres-openssl", "pretty_assertions", "prometheus-parser", - "prost 0.6.1", - "prost-build 0.6.1", - "prost-types 0.6.1", + "prost", + "prost-build", + "prost-types", "pulsar", "rand 0.8.3", "rand_distr", "rdkafka", "regex", - "reqwest", + "reqwest 0.11.1", "rlua", "rusoto_cloudwatch", "rusoto_core", @@ -8216,7 +8169,7 @@ dependencies = [ "rusty-fork", "schannel", "seahash", - "security-framework 2.0.0", + "security-framework", "semver 0.11.0", "serde", "serde_json", @@ -8225,21 +8178,23 @@ dependencies = [ "smpl_jwt", "snafu", "snap", - "socket2", + "socket2 0.4.0", "stream-cancel", "strip-ansi-escapes", "structopt 0.3.21", "syslog", "syslog_loose", "tempfile", - "tokio 0.2.25", - "tokio-openssl", + "tokio 1.3.0", + "tokio-openssl 0.6.1", "tokio-postgres", - "tokio-test 0.4.1", - "tokio-util", + "tokio-stream", + "tokio-test", + "tokio-util 0.6.3", "tokio01-test", "toml", "tower", + "tower-layer", "tower-test", "tracing 0.1.23", "tracing-core 0.1.17", @@ -8268,14 +8223,16 @@ name = "vector-api-client" version = "0.1.1" dependencies = [ "anyhow", + "async-stream", "async-trait", "chrono", "futures 0.3.13", "graphql_client", - "reqwest", + "reqwest 0.11.1", "serde", "serde_json", - "tokio 0.2.25", + "tokio 1.3.0", + "tokio-stream", "tokio-tungstenite", "url", "uuid 0.8.2", @@ -8314,7 +8271,7 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" name = "vrl" version = "0.1.0" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "indoc", "ordered-float", "thiserror", @@ -8327,7 +8284,7 @@ dependencies = [ name = "vrl-cli" version = "0.1.0" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "exitcode", "indoc", "lazy_static", @@ -8347,7 +8304,7 @@ name = "vrl-compiler" version = "0.1.0" dependencies = [ "bitflags", - "bytes 0.5.6", + "bytes 1.0.1", "chrono", "criterion", "dyn-clone", @@ -8390,7 +8347,7 @@ version = "0.1.0" dependencies = [ "anyhow", "base64 0.13.0", - "bytes 0.5.6", + "bytes 1.0.1", "chrono", "cidr-utils", "criterion", @@ -8505,29 +8462,31 @@ dependencies = [ [[package]] name = "warp" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f41be6df54c97904af01aa23e613d4521eed7ab23537cede692d4058f6449407" +checksum = "3dafd0aac2818a94a34df0df1100a7356c493d8ede4393875fd0b5c51bb6bc80" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "futures 0.3.13", "headers", "http", - "hyper 0.13.10", + "hyper 0.14.4", "log", "mime", "mime_guess", - "pin-project 0.4.27", + "percent-encoding", + "pin-project 1.0.6", "scoped-tls", "serde", "serde_json", - "serde_urlencoded 0.6.1", - "tokio 0.2.25", + "serde_urlencoded 0.7.0", + "tokio 1.3.0", + "tokio-stream", "tokio-tungstenite", + "tokio-util 0.6.3", "tower-service", "tracing 0.1.23", "tracing-futures 0.2.5", - "urlencoding", ] [[package]] @@ -8708,21 +8667,21 @@ dependencies = [ ] [[package]] -name = "wepoll-sys" -version = "3.0.1" +name = "webpki-roots" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" +checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" dependencies = [ - "cc", + "webpki", ] [[package]] -name = "which" -version = "3.1.1" +name = "wepoll-sys" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" +checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" dependencies = [ - "libc", + "cc", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 002b51738ced8..511ed394f3da5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,12 +95,13 @@ vector-api-client = { path = "lib/vector-api-client", optional = true } vrl-cli = { path = "lib/vrl/cli", optional = true } # Tokio / Futures -async-trait = "0.1" +async-trait = "0.1.42" futures = { version = "0.3", default-features = false, features = ["compat", "io-compat"] } futures01 = { package = "futures", version = "0.1.25" } -tokio = { version = "0.2.13", features = ["blocking", "fs", "io-std", "macros", "process", "rt-core", "rt-threaded", "signal", "stream", "sync", "time", "udp", "uds"] } -tokio-openssl = "0.4.0" -tokio-util = { version = "0.3.1", features = ["codec"] } +tokio = { version = "1.3.0", features = ["full"] } +tokio-openssl = "0.6.1" +tokio-stream = { version = "0.1.3", features = ["net", "sync"] } +tokio-util = { version = "0.6.2", features = ["codec", "time"] } # Tracing tracing = "0.1.15" @@ -119,20 +120,21 @@ metrics-util = "=0.4.0-alpha.10" metrics-macros = "=0.1.0-alpha.9" # Aws -rusoto_cloudwatch = { version = "0.45.0", optional = true } -rusoto_core = { version = "0.45.0", features = ["encoding"], optional = true } -rusoto_credential = { version = "0.45.0", optional = true } -rusoto_es = { version = "0.45.0", optional = true } -rusoto_firehose = { version = "0.45.0", optional = true } -rusoto_kinesis = { version = "0.45.0", optional = true } -rusoto_logs = { version = "0.45.0", optional = true } -rusoto_s3 = { version = "0.45.0", optional = true } -rusoto_signature = { version = "0.45.0", optional = true } -rusoto_sqs = { version = "0.45.0", optional = true } -rusoto_sts = { version = "0.45.0", optional = true } +rusoto_cloudwatch = { version = "0.46.0", optional = true } +rusoto_core = { version = "0.46.0", features = ["encoding"], optional = true } +rusoto_credential = { version = "0.46.0", optional = true } +rusoto_es = { version = "0.46.0", optional = true } +rusoto_firehose = { version = "0.46.0", optional = true } +rusoto_kinesis = { version = "0.46.0", optional = true } +rusoto_logs = { version = "0.46.0", optional = true } +rusoto_s3 = { version = "0.46.0", optional = true } +rusoto_signature = { version = "0.46.0", optional = true } +rusoto_sqs = { version = "0.46.0", optional = true } +rusoto_sts = { version = "0.46.0", optional = true } # Tower -tower = { version = "0.3.1", git = "https://github.com/tower-rs/tower", rev = "43168944220ed32dab83cb4f11f7b97abc5818d5", features = ["buffer", "limit", "retry", "timeout", "util"] } +tower = { version = "0.4.0", features = ["buffer", "limit", "retry", "timeout", "util"] } +tower-layer = "0.3.1" # Serde serde = { version = "1.0.117", features = ["derive"] } @@ -140,16 +142,16 @@ serde_json = { version = "1.0.33", features = ["raw_value"] } serde_yaml = { version ="0.8.13" } # Prost -prost = "0.6.1" -prost-types = "0.6.1" +prost = "0.7.0" +prost-types = "0.7.0" # GCP -goauth = { version = "0.8.1", optional = true } -smpl_jwt = { version = "0.5.0", optional = true } +goauth = { version = "0.9.0", optional = true } +smpl_jwt = { version = "0.6.1", optional = true } # API -async-graphql = { version = "=2.5.0", optional = true } -async-graphql-warp = { version = "=2.5.0", optional = true } +async-graphql = { version = "2.5.9", optional = true } +async-graphql-warp = { version = "2.5.9", optional = true } itertools = { version = "0.10.0", optional = true } # API client @@ -164,12 +166,12 @@ vrl-stdlib = { path = "lib/vrl/stdlib" } # External libs anyhow = "1.0.37" -async-compression = { version = "0.3.7", features = ["tokio-02", "gzip", "zstd"] } +async-compression = { version = "0.3.7", features = ["tokio", "gzip", "zstd"] } avro-rs = { version = "0.13.0", optional = true } base64 = { version = "0.13.0", optional = true } bloom = { version = "0.3.2", optional = true } -bollard = { version = "0.9.1", features = ["ssl"], optional = true } -bytes = { version = "0.5.6", features = ["serde"] } +bollard = { version = "0.10.1", features = ["ssl"], optional = true } +bytes = { version = "1.0.0", features = ["serde"] } bytesize = { version = "1.0.0", optional = true } chrono = { version = "0.4.19", features = ["serde"] } cidr-utils = "0.5.1" @@ -180,7 +182,7 @@ derivative = "2.1.1" dirs-next = { version = "2.0.0", optional = true } dyn-clone = "1.0.3" encoding_rs = { version = "0.8", features = ["serde"] } -evmap = { version = "10.0.2", features = ["bytes"], optional = true } +evmap = { git = "https://github.com/lukesteensen/evmap.git", rev = "45ba973c22715a68c5e99efad4b072421f7ad40b", features = ["bytes"], optional = true } exitcode = "1.1.2" flate2 = "1.0.19" getset = "0.1.1" @@ -190,8 +192,8 @@ headers = "0.3" heim = { version = "0.1.0-rc.1", features = ["full"], optional = true } hostname = "0.3.1" http = "0.2" -hyper = "0.13" -hyper-openssl = "0.8" +hyper = { version = "0.14", features = ["full"] } +hyper-openssl = "0.9.1" indexmap = {version = "1.6.2", features = ["serde"]} indoc = "1.0.3" inventory = "0.1.10" @@ -219,7 +221,7 @@ postgres-openssl = { version = "0.3.0", optional = true } pulsar = { version = "1.0.0", default-features = false, features = ["tokio-runtime"], optional = true } rand = { version = "0.8.0", features = ["small_rng"] } rand_distr = "0.4.0" -rdkafka = { version = "0.24.0", features = ["libz", "ssl", "zstd"], optional = true } +rdkafka = { version = "0.26.0", features = ["libz", "ssl", "zstd"], optional = true } regex = "1.4.5" # make sure to update the external docs when the Lua version changes rlua = { version = "0.17.0", optional = true } @@ -227,8 +229,8 @@ seahash = { version = "4.0.1", optional = true } semver = { version = "0.11.0", features = ["serde"], optional = true } snafu = { version = "0.6.10", features = ["futures", "futures-01"] } snap = { version = "1.0.3", optional = true } -socket2 = { version = "0.3.19", optional = true } -stream-cancel = "0.6.2" +socket2 = "0.4.0" +stream-cancel = "0.8.0" strip-ansi-escapes = "0.1.0" structopt = "0.3.21" syslog = { version = "5", optional = true } @@ -239,7 +241,7 @@ typetag = "0.1.6" twox-hash = "1.6" url = "2.2.1" uuid = { version = "0.8", features = ["serde", "v4"], optional = true } -warp = { version = "0.2.5", default-features = false, optional = true } +warp = { version = "0.3.0", default-features = false, optional = true } # For WASM async-stream = "0.3.0" @@ -260,7 +262,7 @@ atty = "0.2" nix = "0.20.0" [build-dependencies] -prost-build = "0.6.1" +prost-build = "0.7.0" built = { version = "0.4.4", features = ["chrono", "git2"] } [dev-dependencies] @@ -273,13 +275,13 @@ libc = "0.2.80" libz-sys = "1.1.2" matches = "0.1.8" pretty_assertions = "0.7.1" -reqwest = { version = "0.10.9", features = ["json"] } +reqwest = { version = "0.11.0", features = ["json"] } rusty-fork = "0.3.0" tempfile = "3.0.6" -tokio = { version = "0.2", features = ["test-util"] } -tokio-test = "0.4" +tokio = { version = "1.3.0", features = ["test-util"] } +tokio-test = "0.4.0" tokio01-test = "0.1.1" -tower-test = "0.3.0" +tower-test = "0.4.0" walkdir = "2.2.7" [features] @@ -336,6 +338,7 @@ api = [ "async-graphql-warp", "base64", "itertools", + "warp", ] # API client @@ -401,16 +404,16 @@ sources-mongodb_metrics = ["mongodb"] sources-nginx_metrics = ["nom"] sources-postgresql_metrics = ["postgres-openssl", "tokio-postgres"] sources-prometheus = ["prometheus-parser", "sinks-prometheus", "sources-utils-http", "warp"] -sources-socket = ["bytesize", "listenfd", "tokio-util/udp", "sources-utils-udp", "sources-utils-tcp-keepalive", "sources-utils-tcp-socket", "sources-utils-tls", "sources-utils-unix"] +sources-socket = ["bytesize", "listenfd", "tokio-util/net", "sources-utils-udp", "sources-utils-tcp-keepalive", "sources-utils-tcp-socket", "sources-utils-tls", "sources-utils-unix"] sources-splunk_hec = ["bytesize", "sources-utils-tls", "warp"] -sources-statsd = ["listenfd", "sources-utils-tcp-keepalive", "sources-utils-tcp-socket", "sources-utils-tls", "sources-utils-udp", "sources-utils-unix", "tokio-util/udp"] +sources-statsd = ["listenfd", "sources-utils-tcp-keepalive", "sources-utils-tcp-socket", "sources-utils-tls", "sources-utils-udp", "sources-utils-unix", "tokio-util/net"] sources-stdin = ["bytesize"] -sources-syslog = ["bytesize", "listenfd", "tokio-util/udp", "sources-utils-udp", "sources-utils-tcp-keepalive", "sources-utils-tcp-socket", "sources-utils-tls", "sources-utils-unix", "syslog_loose"] +sources-syslog = ["bytesize", "listenfd", "tokio-util/net", "sources-utils-udp", "sources-utils-tcp-keepalive", "sources-utils-tcp-socket", "sources-utils-tls", "sources-utils-unix", "syslog_loose"] sources-utils-http = ["snap", "sources-utils-tls", "warp"] sources-utils-tcp-keepalive = [] sources-utils-tcp-socket = [] sources-utils-tls = [] -sources-utils-udp = ["socket2"] +sources-utils-udp = [] sources-utils-unix = [] sources-vector = ["listenfd", "sources-utils-tcp-keepalive", "sources-utils-tcp-socket", "sources-utils-tls"] @@ -563,8 +566,8 @@ sinks-pulsar = ["avro-rs", "pulsar"] sinks-sematext = ["sinks-elasticsearch", "sinks-influxdb"] sinks-socket = ["sinks-utils-udp"] sinks-splunk_hec = ["bytesize"] -sinks-statsd = ["sinks-utils-udp", "tokio-util/udp"] -sinks-utils-udp = ["socket2"] +sinks-statsd = ["sinks-utils-udp", "tokio-util/net"] +sinks-utils-udp = [] sinks-vector = ["sinks-utils-udp"] # Identifies that the build is a nightly build @@ -683,10 +686,3 @@ required-features = ["metrics-benches"] name = "distribution_statistic" harness = false required-features = ["statistic-benches"] - -[patch.'https://github.com/tower-rs/tower'] -tower-layer = "=0.3.0" - -[patch.crates-io] -# TODO: update to the next 0.13.x (after 0.13.10, if any) or 0.14 (or higher) -hyper = { version = "0.13", git = "https://github.com/hyperium/hyper", rev = "d7495a75abca34646b1d6d047589c1b8110d0fa5" } diff --git a/benches/batch.rs b/benches/batch.rs index 7269a78387cf9..89560ffb7a156 100644 --- a/benches/batch.rs +++ b/benches/batch.rs @@ -61,7 +61,7 @@ fn benchmark_batching(c: &mut Criterion) { batch_sink, ) }, - |(mut rt, input, batch_sink)| rt.block_on(input.forward(batch_sink)).unwrap(), + |(rt, input, batch_sink)| rt.block_on(input.forward(batch_sink)).unwrap(), criterion::BatchSize::LargeInput, ) }, @@ -86,7 +86,7 @@ fn benchmark_batching(c: &mut Criterion) { (rt, stream::iter(input.clone()).map(Ok), batch_sink) }, - |(mut rt, input, batch_sink)| rt.block_on(input.forward(batch_sink)).unwrap(), + |(rt, input, batch_sink)| rt.block_on(input.forward(batch_sink)).unwrap(), criterion::BatchSize::LargeInput, ) }); diff --git a/benches/buffering.rs b/benches/buffering.rs index b02fabf6b7ad5..dd2593f42b1ac 100644 --- a/benches/buffering.rs +++ b/benches/buffering.rs @@ -34,7 +34,7 @@ fn benchmark_buffers(c: &mut Criterion) { when_full: Default::default(), }; - let mut rt = runtime(); + let rt = runtime(); let (output_lines, topology) = rt.block_on(async move { let output_lines = CountReceiver::receive_lines(out_addr); let (topology, _crash) = start_topology(config.build().unwrap(), false).await; @@ -44,7 +44,7 @@ fn benchmark_buffers(c: &mut Criterion) { (rt, topology, output_lines) }, - |(mut rt, topology, output_lines)| { + |(rt, topology, output_lines)| { rt.block_on(async move { let lines = random_lines(line_size).take(num_lines); send_lines(in_addr, lines).await.unwrap(); @@ -82,7 +82,7 @@ fn benchmark_buffers(c: &mut Criterion) { when_full: Default::default(), }; config.global.data_dir = Some(data_dir.path().to_path_buf()); - let mut rt = runtime(); + let rt = runtime(); let (output_lines, topology) = rt.block_on(async move { let output_lines = CountReceiver::receive_lines(out_addr); let (topology, _crash) = start_topology(config.build().unwrap(), false).await; @@ -91,7 +91,7 @@ fn benchmark_buffers(c: &mut Criterion) { }); (rt, topology, output_lines) }, - |(mut rt, topology, output_lines)| { + |(rt, topology, output_lines)| { rt.block_on(async move { let lines = random_lines(line_size).take(num_lines); send_lines(in_addr, lines).await.unwrap(); @@ -130,7 +130,7 @@ fn benchmark_buffers(c: &mut Criterion) { //when_full: Default::default(), //}; //config.global.data_dir = Some(data_dir.path().to_path_buf()); - //let mut rt = runtime(); + //let rt = runtime(); //let (output_lines, topology) = rt.block_on(async move { //let output_lines = CountReceiver::receive_lines(out_addr); //let (topology, _crash) = start_topology(config.build().unwrap(), false).await; @@ -139,7 +139,7 @@ fn benchmark_buffers(c: &mut Criterion) { //}); //(rt, topology, output_lines) //}, - //|(mut rt, topology, output_lines)| { + //|(rt, topology, output_lines)| { //rt.block_on(async move { //let lines = random_lines(line_size).take(num_lines); //send_lines(in_addr, lines).await.unwrap(); diff --git a/benches/files.rs b/benches/files.rs index 0f9ac4c00cdb5..02377ec790feb 100644 --- a/benches/files.rs +++ b/benches/files.rs @@ -59,7 +59,7 @@ fn benchmark_files_without_partitions(c: &mut Criterion) { }, ); - let mut rt = runtime(); + let rt = runtime(); let (topology, input) = rt.block_on(async move { let (topology, _crash) = start_topology(config.build().unwrap(), false).await; @@ -74,7 +74,7 @@ fn benchmark_files_without_partitions(c: &mut Criterion) { }); (rt, topology, input) }, - |(mut rt, topology, input)| { + |(rt, topology, input)| { rt.block_on(async move { let lines = random_lines(line_size).take(num_lines).map(|mut line| { line.push('\n'); diff --git a/benches/http.rs b/benches/http.rs index 28853824d0eb3..8ce6e69ac290d 100644 --- a/benches/http.rs +++ b/benches/http.rs @@ -58,7 +58,7 @@ fn benchmark_http(c: &mut Criterion) { }, ); - let mut rt = runtime(); + let rt = runtime(); let topology = rt.block_on(async move { let (topology, _crash) = start_topology(config.build().unwrap(), false).await; @@ -67,7 +67,7 @@ fn benchmark_http(c: &mut Criterion) { }); (rt, topology) }, - |(mut rt, topology)| { + |(rt, topology)| { rt.block_on(async move { let lines = random_lines(line_size).take(num_lines); send_lines(in_addr, lines).await.unwrap(); diff --git a/benches/isolated_buffering.rs b/benches/isolated_buffering.rs index efb1a6ddc04a6..39899679a1b81 100644 --- a/benches/isolated_buffering.rs +++ b/benches/isolated_buffering.rs @@ -7,6 +7,7 @@ use futures::{ }; use futures01::{stream, Sink, Stream}; use tempfile::tempdir; +use tokio_stream::wrappers::ReceiverStream; use vector::{ buffers::{ disk::{leveldb_buffer, DiskBuffer}, @@ -55,7 +56,7 @@ fn benchmark_buffers(c: &mut Criterion) { (rt, writer, read_loop) }, - |(mut rt, writer, read_loop)| { + |(rt, writer, read_loop)| { let send = writer.send_all(random_events(line_size).take(num_lines as u64)); let read_handle = rt.spawn(read_loop.compat()); @@ -81,7 +82,7 @@ fn benchmark_buffers(c: &mut Criterion) { (rt, writer, read_handle) }, - |(mut rt, mut writer, read_handle)| { + |(rt, mut writer, read_handle)| { let write_handle = rt.spawn(async move { let mut stream = random_events(line_size).take(num_lines as u64).compat(); while let Some(e) = stream.next().await { @@ -101,13 +102,14 @@ fn benchmark_buffers(c: &mut Criterion) { || { let rt = runtime(); - let (writer, mut reader) = tokio::sync::mpsc::channel(100); + let (writer, reader) = tokio::sync::mpsc::channel(100); + let mut stream = ReceiverStream::new(reader); - let read_handle = rt.spawn(async move { while reader.next().await.is_some() {} }); + let read_handle = rt.spawn(async move { while stream.next().await.is_some() {} }); (rt, writer, read_handle) }, - |(mut rt, mut writer, read_handle)| { + |(rt, writer, read_handle)| { let write_handle = rt.spawn(async move { let mut stream = random_events(line_size).take(num_lines as u64).compat(); while let Some(e) = stream.next().await { @@ -136,7 +138,7 @@ fn benchmark_buffers(c: &mut Criterion) { (rt, writer) }, - |(mut rt, writer)| { + |(rt, writer)| { let send = writer.send_all(random_events(line_size).take(num_lines as u64)); let write_handle = rt.spawn(send.compat()); let _ = rt.block_on(write_handle).unwrap().unwrap(); @@ -150,7 +152,7 @@ fn benchmark_buffers(c: &mut Criterion) { || { let data_dir = tempdir().unwrap(); - let mut rt = runtime(); + let rt = runtime(); let plenty_of_room = num_lines * line_size * 2; let (writer, reader, acker) = @@ -170,7 +172,7 @@ fn benchmark_buffers(c: &mut Criterion) { (rt, read_loop) }, - |(mut rt, read_loop)| { + |(rt, read_loop)| { let read_handle = rt.spawn(read_loop); rt.block_on(read_handle).unwrap().unwrap(); }, @@ -198,7 +200,7 @@ fn benchmark_buffers(c: &mut Criterion) { (rt, writer, read_loop) }, - |(mut rt, writer, read_loop)| { + |(rt, writer, read_loop)| { let send = writer.send_all(random_events(line_size).take(num_lines as u64)); let read_handle = rt.spawn(read_loop); diff --git a/benches/languages.rs b/benches/languages.rs index e8b0cf47f5eba..c41270e410dd0 100644 --- a/benches/languages.rs +++ b/benches/languages.rs @@ -375,7 +375,7 @@ fn benchmark_configs( let config = config::load_from_str(&config, Some(config::Format::Toml)) .expect(&format!("invalid TOML configuration: {}", &config)); - let mut rt = runtime(); + let rt = runtime(); let (output_lines, topology) = rt.block_on(async move { let output_lines = CountReceiver::receive_lines(out_addr); let (topology, _crash) = start_topology(config, false).await; @@ -385,7 +385,7 @@ fn benchmark_configs( let lines = lines.clone(); (rt, lines, topology, output_lines) }, - |(mut rt, lines, topology, output_lines)| { + |(rt, lines, topology, output_lines)| { rt.block_on(async move { send_lines(in_addr, lines).await.unwrap(); diff --git a/benches/lua.rs b/benches/lua.rs index ea279cf7253b0..620964ea25cb8 100644 --- a/benches/lua.rs +++ b/benches/lua.rs @@ -100,7 +100,7 @@ fn bench_field_filter(c: &mut Criterion) { let benchmarks: Vec<(&str, Transform)> = vec![ ("native", { - let mut rt = runtime(); + let rt = runtime(); rt.block_on(async move { transforms::field_filter::FieldFilterConfig { field: "the_field".to_string(), diff --git a/benches/metrics_bench_util/mod.rs b/benches/metrics_bench_util/mod.rs index 9ea56b287edc0..5d9078542267c 100644 --- a/benches/metrics_bench_util/mod.rs +++ b/benches/metrics_bench_util/mod.rs @@ -142,7 +142,7 @@ fn bench_topology(c: &mut Criterion, bench_name: &'static str) { ), ); - let mut rt = runtime(); + let rt = runtime(); let (output_lines, topology) = rt.block_on(async move { let output_lines = CountReceiver::receive_lines(out_addr); let (topology, _crash) = @@ -153,7 +153,7 @@ fn bench_topology(c: &mut Criterion, bench_name: &'static str) { (input_lines, rt, topology, output_lines) }, - |(input_lines, mut rt, topology, output_lines)| { + |(input_lines, rt, topology, output_lines)| { rt.block_on(async move { let sends = stream::iter(input_lines) .map(|lines| send_lines(in_addr, lines)) diff --git a/benches/regex.rs b/benches/regex.rs index ee852337192be..79f33154510b1 100644 --- a/benches/regex.rs +++ b/benches/regex.rs @@ -21,7 +21,7 @@ fn benchmark_regex(c: &mut Criterion) { let input: Vec = lines.into_iter().map(|l| l.into()).collect(); group.bench_function("regex", |b| { - let mut rt = runtime(); + let rt = runtime(); let mut parser = rt.block_on(async move { transforms::regex_parser::RegexParserConfig { diff --git a/benches/remap.rs b/benches/remap.rs index 117734b86da47..3085196efa5c1 100644 --- a/benches/remap.rs +++ b/benches/remap.rs @@ -25,7 +25,7 @@ criterion_group!( criterion_main!(benches); fn benchmark_remap(c: &mut Criterion) { - let mut rt = runtime(); + let rt = runtime(); let add_fields_runner = |tform: &mut Box, event: Event| { let mut result = Vec::with_capacity(1); tform.transform(&mut result, event); diff --git a/benches/topology.rs b/benches/topology.rs index 954db808d89bc..ecfdfe5cb608c 100644 --- a/benches/topology.rs +++ b/benches/topology.rs @@ -43,7 +43,7 @@ fn benchmark_simple_pipes(c: &mut Criterion) { ), ); - let mut rt = runtime(); + let rt = runtime(); let (output_lines, topology) = rt.block_on(async move { let output_lines = CountReceiver::receive_lines(out_addr); let (topology, _crash) = @@ -53,7 +53,7 @@ fn benchmark_simple_pipes(c: &mut Criterion) { }); (rt, topology, output_lines) }, - |(mut rt, topology, output_lines)| { + |(rt, topology, output_lines)| { rt.block_on(async move { let sends = stream::iter(0..*num_writers) .map(|_| { @@ -117,7 +117,7 @@ fn benchmark_interconnected(c: &mut Criterion) { sinks::socket::SocketSinkConfig::make_basic_tcp_config(out_addr2.to_string()), ); - let mut rt = runtime(); + let rt = runtime(); let (output_lines1, output_lines2, topology) = rt.block_on(async move { let output_lines1 = CountReceiver::receive_lines(out_addr1); let output_lines2 = CountReceiver::receive_lines(out_addr2); @@ -128,7 +128,7 @@ fn benchmark_interconnected(c: &mut Criterion) { }); (rt, topology, output_lines1, output_lines2) }, - |(mut rt, topology, output_lines1, output_lines2)| { + |(rt, topology, output_lines1, output_lines2)| { rt.block_on(async move { let lines1 = random_lines(line_size).take(num_lines); send_lines(in_addr1, lines1).await.unwrap(); @@ -197,7 +197,7 @@ fn benchmark_transforms(c: &mut Criterion) { sinks::socket::SocketSinkConfig::make_basic_tcp_config(out_addr.to_string()), ); - let mut rt = runtime(); + let rt = runtime(); let (output_lines, topology) = rt.block_on(async move { let output_lines = CountReceiver::receive_lines(out_addr); let (topology, _crash) = start_topology(config.build().unwrap(), false).await; @@ -206,7 +206,7 @@ fn benchmark_transforms(c: &mut Criterion) { }); (rt, topology, output_lines) }, - |(mut rt, topology, output_lines)| { + |(rt, topology, output_lines)| { rt.block_on(async move { let lines = random_lines(line_size) .map(|l| l + "status=404") @@ -335,7 +335,7 @@ fn benchmark_complex(c: &mut Criterion) { ), ); - let mut rt = runtime(); + let rt = runtime(); let ( output_lines_all, output_lines_sampled, @@ -372,7 +372,7 @@ fn benchmark_complex(c: &mut Criterion) { ) }, |( - mut rt, + rt, topology, output_lines_all, output_lines_sampled, @@ -704,7 +704,7 @@ fn benchmark_real_world_1(c: &mut Criterion) { ), ); - let mut rt = runtime(); + let rt = runtime(); let ( output_lines_company_api, output_lines_company_admin, @@ -754,7 +754,7 @@ fn benchmark_real_world_1(c: &mut Criterion) { ) }, |( - mut rt, + rt, topology, output_lines_company_api, output_lines_company_admin, diff --git a/docs/reference/components/sinks/socket.cue b/docs/reference/components/sinks/socket.cue index a8d3be4907c02..8ea4d8cbb08dc 100644 --- a/docs/reference/components/sinks/socket.cue +++ b/docs/reference/components/sinks/socket.cue @@ -27,7 +27,7 @@ components: sinks: socket: { } send_buffer_bytes: { enabled: true - relevant_when: "mode = `tcp` or mode = `udp` && os = `unix`" + relevant_when: "mode = `tcp` or mode = `udp`" } keepalive: enabled: true request: enabled: false diff --git a/docs/reference/components/sinks/statsd.cue b/docs/reference/components/sinks/statsd.cue index f88a723565cfa..408461673055f 100644 --- a/docs/reference/components/sinks/statsd.cue +++ b/docs/reference/components/sinks/statsd.cue @@ -17,7 +17,7 @@ components: sinks: statsd: { request: sinks.socket.features.send.request send_buffer_bytes: { enabled: true - relevant_when: "mode = `tcp` or mode = `udp` && os = `unix`" + relevant_when: "mode = `tcp` or mode = `udp`" } tls: sinks.socket.features.send.tls to: { diff --git a/docs/reference/components/sources/socket.cue b/docs/reference/components/sources/socket.cue index a637737aa8d57..80d3bd4242f6d 100644 --- a/docs/reference/components/sources/socket.cue +++ b/docs/reference/components/sources/socket.cue @@ -28,7 +28,7 @@ components: sources: socket: { } receive_buffer_bytes: { enabled: true - relevant_when: "mode = `tcp` or mode = `udp` && os = `unix`" + relevant_when: "mode = `tcp` or mode = `udp`" } keepalive: enabled: true tls: { diff --git a/docs/reference/components/sources/statsd.cue b/docs/reference/components/sources/statsd.cue index 582065ce3645e..576e5a49d43e3 100644 --- a/docs/reference/components/sources/statsd.cue +++ b/docs/reference/components/sources/statsd.cue @@ -32,7 +32,7 @@ components: sources: statsd: { } receive_buffer_bytes: { enabled: true - relevant_when: "mode = `tcp` or mode = `udp` && os = `unix`" + relevant_when: "mode = `tcp` or mode = `udp`" } keepalive: enabled: true tls: enabled: false diff --git a/docs/reference/components/sources/syslog.cue b/docs/reference/components/sources/syslog.cue index c37f47cf97b62..d4506fcf9837b 100644 --- a/docs/reference/components/sources/syslog.cue +++ b/docs/reference/components/sources/syslog.cue @@ -27,7 +27,7 @@ components: sources: syslog: { } receive_buffer_bytes: { enabled: true - relevant_when: "mode = `tcp` or mode = `udp` && os = `unix`" + relevant_when: "mode = `tcp` or mode = `udp`" } keepalive: enabled: true tls: sources.socket.features.receive.tls diff --git a/lib/codec/Cargo.toml b/lib/codec/Cargo.toml index e5aa487d751b1..3583c8ce89cc5 100644 --- a/lib/codec/Cargo.toml +++ b/lib/codec/Cargo.toml @@ -7,8 +7,8 @@ publish = false license = "MPL-2.0" [dependencies] -bytes = "0.5" -tokio-util = { version = "0.3.1", features = ["codec"] } +bytes = "1.0.0" +tokio-util = { version = "0.6.0", features = ["codec"] } tracing = "0.1.15" [dev-dependencies] diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index 0880cf40454ce..1bc97fb4725a7 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -21,7 +21,7 @@ default-features = false features = [] [dependencies.bytes] -version = "0.5" +version = "1.0.0" default-features = false features = [] @@ -66,9 +66,9 @@ default-features = false features = [] [dependencies.tokio] -version = "0.2" +version = "1.3.0" default-features = false -features = ["rt-core", "blocking", "time"] +features = ["full"] [dev-dependencies] criterion = "0.3" diff --git a/lib/file-source/src/file_server.rs b/lib/file-source/src/file_server.rs index 062eb8f4b80bf..bd69e3c3615b7 100644 --- a/lib/file-source/src/file_server.rs +++ b/lib/file-source/src/file_server.rs @@ -21,7 +21,7 @@ use std::{ sync::Arc, time::{self, Duration}, }; -use tokio::time::delay_for; +use tokio::time::sleep; use tracing::{debug, error, info, trace}; /// `FileServer` is a Source which cooperatively schedules reads over files, @@ -141,17 +141,17 @@ where // aren't going to get away with anything, but none of it should have // any perf impact. let mut shutdown = shutdown.shared(); - let shutdown2 = shutdown.clone(); + let mut shutdown2 = shutdown.clone(); let emitter = self.emitter.clone(); let checkpointer = Arc::new(checkpointer); let sleep_duration = self.glob_minimum_cooldown; self.handle.spawn(async move { let mut done = false; loop { - let sleep = tokio::time::delay_for(sleep_duration); - match select(shutdown2.clone(), sleep).await { - Either::Left((_, _)) => done = true, - Either::Right((_, _)) => {} + let sleep = sleep(sleep_duration); + tokio::select! { + _ = &mut shutdown2 => done = true, + _ = sleep => {}, } let emitter = emitter.clone(); @@ -369,7 +369,7 @@ where // all of these requirements. let sleep = async move { if backoff > 0 { - delay_for(Duration::from_millis(backoff as u64)).await; + sleep(Duration::from_millis(backoff as u64)).await; } }; futures::pin_mut!(sleep); diff --git a/lib/k8s-e2e-tests/Cargo.toml b/lib/k8s-e2e-tests/Cargo.toml index 883347da330f8..64a365cb485f8 100644 --- a/lib/k8s-e2e-tests/Cargo.toml +++ b/lib/k8s-e2e-tests/Cargo.toml @@ -14,7 +14,7 @@ k8s-test-framework = { version = "0.1", path = "../k8s-test-framework" } regex = "1" reqwest = { version = "0.10", features = ["json"] } serde_json = "1" -tokio = { version = "0.2", features = ["macros", "rt-threaded", "time"] } +tokio = { version = "1.3.0", features = ["full"] } indoc = "1.0.3" [features] diff --git a/lib/k8s-e2e-tests/src/lib.rs b/lib/k8s-e2e-tests/src/lib.rs index 63a9e02eab060..4c1ecebd3db63 100644 --- a/lib/k8s-e2e-tests/src/lib.rs +++ b/lib/k8s-e2e-tests/src/lib.rs @@ -127,7 +127,7 @@ where lines_till_we_give_up -= 1; if lines_till_we_give_up <= 0 { println!("Giving up"); - log_reader.kill()?; + log_reader.kill().await?; break; } @@ -157,7 +157,7 @@ where // killed. // This doesn't immediately stop the reading because we want to // process the pending buffers first. - log_reader.kill()?; + log_reader.kill().await?; } } } diff --git a/lib/k8s-e2e-tests/src/metrics.rs b/lib/k8s-e2e-tests/src/metrics.rs index 925ff0a759360..43a253a20200d 100644 --- a/lib/k8s-e2e-tests/src/metrics.rs +++ b/lib/k8s-e2e-tests/src/metrics.rs @@ -88,7 +88,7 @@ pub async fn wait_for_vector_started( .saturating_duration_since(std::time::Instant::now()) .as_secs_f64(), ); - tokio::time::delay_for(next_attempt_delay).await; + tokio::time::sleep(next_attempt_delay).await; } Ok(()) } diff --git a/lib/k8s-e2e-tests/tests/vector-agent.rs b/lib/k8s-e2e-tests/tests/vector-agent.rs index 37c7d2b66edc0..0a2884e9c818e 100644 --- a/lib/k8s-e2e-tests/tests/vector-agent.rs +++ b/lib/k8s-e2e-tests/tests/vector-agent.rs @@ -326,7 +326,7 @@ async fn preexisting() -> Result<(), Box> { .await?; // Wait for some extra time to ensure pod completes. - tokio::time::delay_for(std::time::Duration::from_secs(10)).await; + tokio::time::sleep(std::time::Duration::from_secs(10)).await; let vector = framework .vector( @@ -718,7 +718,7 @@ async fn pod_filtering() -> Result<(), Box> { // time to pick them up and spit them out. let duration = std::time::Duration::from_secs(120); println!("Starting stop timer, due in {} seconds", duration.as_secs()); - tokio::time::delay_for(duration).await; + tokio::time::sleep(duration).await; println!("Stop timer complete"); stop_tx.send(()).await.unwrap(); }); @@ -898,7 +898,7 @@ async fn custom_selectors() -> Result<(), Box> { // time to pick them up and spit them out. let duration = std::time::Duration::from_secs(120); println!("Starting stop timer, due in {} seconds", duration.as_secs()); - tokio::time::delay_for(duration).await; + tokio::time::sleep(duration).await; println!("Stop timer complete"); stop_tx.send(()).await.unwrap(); }); @@ -1043,7 +1043,7 @@ async fn container_filtering() -> Result<(), Box> { // time to pick them up and spit them out. let duration = std::time::Duration::from_secs(30); println!("Starting stop timer, due in {} seconds", duration.as_secs()); - tokio::time::delay_for(duration).await; + tokio::time::sleep(duration).await; println!("Stop timer complete"); stop_tx.send(()).await.unwrap(); }); @@ -1192,7 +1192,7 @@ async fn glob_pattern_filtering() -> Result<(), Box> { // time to pick them up and spit them out. let duration = std::time::Duration::from_secs(30); println!("Starting stop timer, due in {} seconds", duration.as_secs()); - tokio::time::delay_for(duration).await; + tokio::time::sleep(duration).await; println!("Stop timer complete"); stop_tx.send(()).await.unwrap(); }); @@ -1439,7 +1439,7 @@ async fn metrics_pipeline() -> Result<(), Box> { // We give Vector some reasonable time to perform this initial bootstrap, // and capture the `processed_events` value afterwards. println!("Waiting for Vector bootstrap"); - tokio::time::delay_for(std::time::Duration::from_secs(30)).await; + tokio::time::sleep(std::time::Duration::from_secs(30)).await; println!("Done waiting for Vector bootstrap"); // Capture events processed before deploying the test pod. @@ -1499,7 +1499,7 @@ async fn metrics_pipeline() -> Result<(), Box> { // Due to how `internal_metrics` are implemented, we have to wait for it's // scraping period to pass before we can observe the updates. println!("Waiting for `internal_metrics` to update"); - tokio::time::delay_for(std::time::Duration::from_secs(6)).await; + tokio::time::sleep(std::time::Duration::from_secs(6)).await; println!("Done waiting for `internal_metrics` to update"); // Capture events processed after the test pod has finished. @@ -1562,7 +1562,7 @@ async fn host_metrics() -> Result<(), Box> { // collecting them takes some time to boot (15s roughly). // We wait twice as much, so the bootstrap is guaranteed. println!("Waiting for Vector bootstrap"); - tokio::time::delay_for(std::time::Duration::from_secs(30)).await; + tokio::time::sleep(std::time::Duration::from_secs(30)).await; println!("Done waiting for Vector bootstrap"); // Ensure the host metrics are exposed in the Prometheus endpoint. diff --git a/lib/k8s-test-framework/Cargo.toml b/lib/k8s-test-framework/Cargo.toml index 82f89a591234e..06ac599bc266c 100644 --- a/lib/k8s-test-framework/Cargo.toml +++ b/lib/k8s-test-framework/Cargo.toml @@ -12,7 +12,4 @@ k8s-openapi = { version = "0.11.0", default-features = false, features = ["v1_16 once_cell = "1" serde_json = "1" tempfile = "3" -tokio = { version = "0.2", features = ["io-util", "process"] } - -[dev-dependencies] -tokio = { version = "0.2", features = ["macros", "rt-threaded"] } +tokio = { version = "1.3.0", features = ["full"] } diff --git a/lib/k8s-test-framework/src/port_forward.rs b/lib/k8s-test-framework/src/port_forward.rs index c32095180e047..3de1eaec0005a 100644 --- a/lib/k8s-test-framework/src/port_forward.rs +++ b/lib/k8s-test-framework/src/port_forward.rs @@ -116,11 +116,11 @@ impl PortForwarder { /// Wait for the `kubectl port-forward` process to exit and return the exit /// code. pub async fn wait(&mut self) -> std::io::Result { - (&mut self.child).await + (&mut self.child).wait().await } /// Send a termination signal to the `kubectl port-forward` process. - pub fn kill(&mut self) -> std::io::Result<()> { - self.child.kill() + pub async fn kill(&mut self) -> std::io::Result<()> { + self.child.kill().await } } diff --git a/lib/k8s-test-framework/src/reader.rs b/lib/k8s-test-framework/src/reader.rs index 2fa1ffb985b95..1881316f53551 100644 --- a/lib/k8s-test-framework/src/reader.rs +++ b/lib/k8s-test-framework/src/reader.rs @@ -32,12 +32,12 @@ impl Reader { /// Wait for the `kubectl logs` process to exit and return the exit code. pub async fn wait(&mut self) -> std::io::Result { - (&mut self.child).await + (&mut self.child).wait().await } /// Send a termination signal to the `kubectl logs` process. - pub fn kill(&mut self) -> std::io::Result<()> { - self.child.kill() + pub async fn kill(&mut self) -> std::io::Result<()> { + self.child.kill().await } /// Read one line from the stdout of the `kubectl logs` process. @@ -98,7 +98,7 @@ mod tests { // On line 100 issue a `kill` to stop the infinite stream. if expected_num == 100 { - reader.kill().expect("process already stopped") + reader.kill().await.expect("process already stopped") } // If we are past 200 it means we issued `kill` at 100 and it wasn't diff --git a/lib/k8s-test-framework/src/util.rs b/lib/k8s-test-framework/src/util.rs index 87c53585589b8..42fcbdea5247d 100644 --- a/lib/k8s-test-framework/src/util.rs +++ b/lib/k8s-test-framework/src/util.rs @@ -1,7 +1,7 @@ use crate::Result; pub async fn run_command(mut command: tokio::process::Command) -> Result<()> { - let exit_status = command.spawn()?.await?; + let exit_status = command.spawn()?.wait().await?; if !exit_status.success() { return Err(format!("exec failed: {:?}", command).into()); } @@ -9,8 +9,7 @@ pub async fn run_command(mut command: tokio::process::Command) -> Result<()> { } pub fn run_command_blocking(mut command: std::process::Command) -> Result<()> { - let mut child = command.spawn()?; - let exit_status = child.wait()?; + let exit_status = command.spawn()?.wait()?; if !exit_status.success() { return Err(format!("exec failed: {:?}", command).into()); } diff --git a/lib/prometheus-parser/Cargo.toml b/lib/prometheus-parser/Cargo.toml index e997af44b75c7..e521c12c40046 100644 --- a/lib/prometheus-parser/Cargo.toml +++ b/lib/prometheus-parser/Cargo.toml @@ -12,12 +12,12 @@ license = "MPL-2.0" indexmap = "1.6.2" nom = "6.0.1" num_enum = "0.5.1" -prost = "0.6.1" -prost-types = "0.6.1" +prost = "0.7.0" +prost-types = "0.7.0" snafu = { version = "0.6" } [build-dependencies] -prost-build = "0.6.1" +prost-build = "0.7.0" [dev-dependencies] shared = { path = "../shared", features = ["btreemap"] } diff --git a/lib/shared/Cargo.toml b/lib/shared/Cargo.toml index 3e57600a4b218..b1ff896c21e97 100644 --- a/lib/shared/Cargo.toml +++ b/lib/shared/Cargo.toml @@ -7,7 +7,7 @@ publish = false license = "MPL-2.0" [dependencies] -bytes = { version = "0.5.6", optional = true } +bytes = { version = "1.0.0", optional = true } chrono = { version = "0.4", optional = true } chrono-tz = "0.5.3" derivative = "2.1.3" diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 4061c183f601d..0d2df50afeeaf 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -16,16 +16,18 @@ serde_json = { version = "1.0.33", features = ["raw_value"] } anyhow = "1.0.28" # Tokio / Futures +async-stream = "0.3.0" async-trait = "0.1" futures = { version = "0.3", default-features = false, features = ["compat", "io-compat"] } -tokio = { version = "0.2.13", features = ["blocking", "fs", "io-std", "macros", "rt-core", "rt-threaded", "signal", "sync", "uds"] } +tokio = { version = "1.3.0", features = ["full"] } +tokio-stream = { version = "0.1.3", features = ["sync"] } # GraphQL graphql_client = "0.9.0" # HTTP / WebSockets -reqwest = { version = "0.10.9", features = ["json"] } -tokio-tungstenite = { version = "0.11.0", features = ["tls"] } +reqwest = { version = "0.11.0", features = ["json"] } +tokio-tungstenite = { version = "0.13.0", features = ["tls"] } # External libs chrono = { version = "0.4.6", features = ["serde"] } diff --git a/lib/vector-api-client/src/subscription.rs b/lib/vector-api-client/src/subscription.rs index 7e35839140eb7..70bf285869e1a 100644 --- a/lib/vector-api-client/src/subscription.rs +++ b/lib/vector-api-client/src/subscription.rs @@ -6,10 +6,8 @@ use std::{ pin::Pin, sync::{Arc, Mutex, Weak}, }; -use tokio::{ - stream::{Stream, StreamExt}, - sync::{broadcast, mpsc, oneshot}, -}; +use tokio::sync::{broadcast, mpsc, oneshot}; +use tokio_stream::{wrappers::BroadcastStream, Stream, StreamExt}; use tokio_tungstenite::{connect_async, tungstenite::Message}; use url::Url; use uuid::Uuid; @@ -98,7 +96,7 @@ impl Subscription { /// Send a payload down the channel. This is synchronous because broadcast::Sender::send /// is also synchronous - fn receive(&self, payload: Payload) -> Result> { + fn receive(&self, payload: Payload) -> Result> { self.tx.send(payload) } @@ -125,13 +123,15 @@ impl Drop for Subscription { } } -impl Receiver for Subscription { +impl Receiver for Subscription +where + T: GraphQLQuery + Send + Sync, + ::ResponseData: Unpin + Send + Sync + 'static, +{ /// Returns a stream of `Payload` responses, received from the GraphQL server fn stream(&self) -> StreamResponse { Box::pin( - self.tx - .subscribe() - .into_stream() + BroadcastStream::new(self.tx.subscribe()) .filter(Result::is_ok) .map(|p| p.unwrap().response::()), ) @@ -166,7 +166,7 @@ impl SubscriptionClient { _ = &mut shutdown_rx => break, // Handle receiving payloads back _from_ the server - Some(p) = rx.next() => { + Some(p) = rx.recv() => { let s = subscriptions_clone.lock().unwrap().get::(&p.id); if let Some(s) = s as Option> @@ -186,10 +186,14 @@ impl SubscriptionClient { } /// Start a new subscription request - pub fn start( + pub fn start( &self, request_body: &graphql_client::QueryBody, - ) -> BoxedSubscription { + ) -> BoxedSubscription + where + T: GraphQLQuery + Send + Sync, + ::ResponseData: Unpin + Send + Sync + 'static, + { // Generate a unique ID for the subscription. Subscriptions can be multiplexed // over a single connection, so we'll keep a copy of this against the client to // handling routing responses back to the relevant subscriber. @@ -229,7 +233,7 @@ pub async fn connect_subscription_client( // Forwarded received messages back upstream to the GraphQL server tokio::spawn(async move { loop { - if let Some(p) = send_rx.next().await { + if let Some(p) = send_rx.recv().await { let _ = ws_tx .send(Message::Text(serde_json::to_string(&p).unwrap())) .await; diff --git a/lib/vrl/cli/Cargo.toml b/lib/vrl/cli/Cargo.toml index dd0f7a932ce38..8fe0ac92b066b 100644 --- a/lib/vrl/cli/Cargo.toml +++ b/lib/vrl/cli/Cargo.toml @@ -12,7 +12,7 @@ path = "src/main.rs" [dependencies] vrl = { path = "../core" } -bytes = "0.5.6" +bytes = "1.0.0" exitcode = "1" prettytable-rs = { version = "0.8", default-features = false, optional = true } regex = { version = "1", default-features = false, optional = true } diff --git a/lib/vrl/compiler/Cargo.toml b/lib/vrl/compiler/Cargo.toml index b50f27da25bdb..b44a8985e26a7 100644 --- a/lib/vrl/compiler/Cargo.toml +++ b/lib/vrl/compiler/Cargo.toml @@ -10,7 +10,7 @@ diagnostic = { package = "vrl-diagnostic", path = "../diagnostic" } parser = { package = "vrl-parser", path = "../parser" } bitflags = "1" -bytes = "0.5.6" +bytes = "1.0.0" chrono = "0.4" lalrpop-util = "0.19" ngrammatic = "0.3" diff --git a/lib/vrl/core/Cargo.toml b/lib/vrl/core/Cargo.toml index 1931aab122878..dbe6906ad250c 100644 --- a/lib/vrl/core/Cargo.toml +++ b/lib/vrl/core/Cargo.toml @@ -10,7 +10,7 @@ compiler = { package = "vrl-compiler", path = "../compiler" } diagnostic = { package = "vrl-diagnostic", path = "../diagnostic" } parser = { package = "vrl-parser", path = "../parser" } -bytes = "0.5.6" +bytes = "1.0.0" indoc = "1" ordered-float = "2" thiserror = "1" diff --git a/lib/vrl/stdlib/Cargo.toml b/lib/vrl/stdlib/Cargo.toml index 048729a5ead6e..dd94c13f44e8d 100644 --- a/lib/vrl/stdlib/Cargo.toml +++ b/lib/vrl/stdlib/Cargo.toml @@ -10,7 +10,7 @@ license = "MPL-2.0" vrl = { path = "../core" } base64 = { version = "0.13", optional = true } -bytes = { version = "0.5.6", optional = true } +bytes = { version = "1.0.0", optional = true } chrono = { version = "0.4", optional = true } cidr-utils = { version = "0.5", optional = true } csv = { version = "1.1", optional = true } diff --git a/src/api/schema/components/mod.rs b/src/api/schema/components/mod.rs index d61105e590138..67ef05b60de18 100644 --- a/src/api/schema/components/mod.rs +++ b/src/api/schema/components/mod.rs @@ -18,7 +18,7 @@ use std::{ cmp, collections::{HashMap, HashSet}, }; -use tokio::stream::{Stream, StreamExt}; +use tokio_stream::{wrappers::BroadcastStream, Stream, StreamExt}; #[derive(Debug, Clone, Interface)] #[graphql( @@ -229,24 +229,18 @@ pub struct ComponentsSubscription; impl ComponentsSubscription { /// Subscribes to all newly added components async fn component_added(&self) -> impl Stream { - COMPONENT_CHANGED - .subscribe() - .into_stream() - .filter_map(|c| match c { - Ok(ComponentChanged::Added(c)) => Some(c), - _ => None, - }) + BroadcastStream::new(COMPONENT_CHANGED.subscribe()).filter_map(|c| match c { + Ok(ComponentChanged::Added(c)) => Some(c), + _ => None, + }) } /// Subscribes to all removed components async fn component_removed(&self) -> impl Stream { - COMPONENT_CHANGED - .subscribe() - .into_stream() - .filter_map(|c| match c { - Ok(ComponentChanged::Removed(c)) => Some(c), - _ => None, - }) + BroadcastStream::new(COMPONENT_CHANGED.subscribe()).filter_map(|c| match c { + Ok(ComponentChanged::Removed(c)) => Some(c), + _ => None, + }) } } diff --git a/src/api/schema/health.rs b/src/api/schema/health.rs index 62a0621c99e17..9aee8c23131e8 100644 --- a/src/api/schema/health.rs +++ b/src/api/schema/health.rs @@ -1,9 +1,7 @@ use async_graphql::{validators::IntRange, Object, SimpleObject, Subscription}; use chrono::{DateTime, Utc}; -use tokio::{ - stream::{Stream, StreamExt}, - time::Duration, -}; +use tokio::time::Duration; +use tokio_stream::{wrappers::IntervalStream, Stream, StreamExt}; #[derive(SimpleObject)] pub struct Heartbeat { @@ -37,6 +35,9 @@ impl HealthSubscription { &self, #[graphql(default = 1000, validator(IntRange(min = "10", max = "60_000")))] interval: i32, ) -> impl Stream { - tokio::time::interval(Duration::from_millis(interval as u64)).map(|_| Heartbeat::new()) + IntervalStream::new(tokio::time::interval(Duration::from_millis( + interval as u64, + ))) + .map(|_| Heartbeat::new()) } } diff --git a/src/api/schema/metrics/filter.rs b/src/api/schema/metrics/filter.rs index 81baccbdcde21..d61b4915d0364 100644 --- a/src/api/schema/metrics/filter.rs +++ b/src/api/schema/metrics/filter.rs @@ -6,10 +6,8 @@ use crate::{ use async_stream::stream; use lazy_static::lazy_static; use std::{collections::BTreeMap, sync::Arc}; -use tokio::{ - stream::{Stream, StreamExt}, - time::Duration, -}; +use tokio::time::Duration; +use tokio_stream::{Stream, StreamExt}; lazy_static! { static ref GLOBAL_CONTROLLER: Arc<&'static Controller> = diff --git a/src/api/schema/metrics/mod.rs b/src/api/schema/metrics/mod.rs index 5d033b040b534..f83866704f7e5 100644 --- a/src/api/schema/metrics/mod.rs +++ b/src/api/schema/metrics/mod.rs @@ -10,7 +10,7 @@ mod uptime; use async_graphql::{validators::IntRange, Interface, Object, Subscription}; use chrono::{DateTime, Utc}; -use tokio::stream::{Stream, StreamExt}; +use tokio_stream::{Stream, StreamExt}; pub use errors::{ComponentErrorsTotal, ErrorsTotal}; pub use filter::*; diff --git a/src/app.rs b/src/app.rs index e5a64d926bf06..1d1df01bd2cb6 100644 --- a/src/app.rs +++ b/src/app.rs @@ -4,12 +4,12 @@ use crate::topology::RunningTopology; use crate::{ config, generate, heartbeat, list, metrics, signal, topology, trace, unit_test, validate, }; -use std::cmp::max; use std::collections::HashMap; use std::path::PathBuf; use futures::StreamExt; use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; #[cfg(feature = "sources-host_metrics")] use crate::sources::host_metrics; @@ -93,12 +93,9 @@ impl Application { } } - let mut rt = { - let threads = root_opts.threads.unwrap_or_else(|| max(1, num_cpus::get())); - runtime::Builder::new() - .threaded_scheduler() + let rt = { + runtime::Builder::new_multi_thread() .enable_all() - .core_threads(threads) .build() .expect("Unable to create async runtime") }; @@ -186,9 +183,9 @@ impl Application { } pub fn run(self) { - let mut rt = self.runtime; + let rt = self.runtime; - let mut graceful_crash = self.config.graceful_crash; + let mut graceful_crash = UnboundedReceiverStream::new(self.config.graceful_crash); let mut topology = self.config.topology; let mut config_paths = self.config.config_paths; diff --git a/src/async_read.rs b/src/async_read.rs index d072d8fd0fdeb..8f90ac209784a 100644 --- a/src/async_read.rs +++ b/src/async_read.rs @@ -1,11 +1,10 @@ use pin_project::pin_project; use std::{ future::Future, - mem::MaybeUninit, pin::Pin, task::{Context, Poll}, }; -use tokio::io::{AsyncRead, Result as IoResult}; +use tokio::io::{AsyncRead, ReadBuf, Result as IoResult}; pub trait VecAsyncReadExt: AsyncRead { /// Read data from this reader until the given future resolves. @@ -37,6 +36,10 @@ impl AllowReadUntil { pub fn get_ref(&self) -> &S { &self.reader } + + pub fn get_mut(&mut self) -> &mut S { + &mut self.reader + } } impl AsyncRead for AllowReadUntil @@ -44,15 +47,15 @@ where S: AsyncRead, F: Future, { - fn poll_read(self: Pin<&mut Self>, cx: &mut Context, buf: &mut [u8]) -> Poll> { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context, + buf: &mut ReadBuf<'_>, + ) -> Poll> { let this = self.project(); match this.until.poll(cx) { - Poll::Ready(_) => Poll::Ready(Ok(0)), + Poll::Ready(_) => Poll::Ready(Ok(())), Poll::Pending => this.reader.poll_read(cx, buf), } } - - unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [MaybeUninit]) -> bool { - self.reader.prepare_uninitialized_buffer(buf) - } } diff --git a/src/buffers/mod.rs b/src/buffers/mod.rs index 3796e8c7c8b6d..c9cf5369493a4 100644 --- a/src/buffers/mod.rs +++ b/src/buffers/mod.rs @@ -15,7 +15,7 @@ use std::{ task::{Context, Poll}, }; #[cfg(feature = "leveldb")] -use tokio::stream::StreamExt; +use tokio_stream::StreamExt; #[cfg(feature = "leveldb")] pub mod disk; @@ -249,22 +249,21 @@ impl + Unpin> Sink for DropWhenFull { #[cfg(test)] mod test { use super::{Acker, BufferConfig, DropWhenFull, WhenFull}; - use crate::sink::BoundedSink; + use futures::channel::mpsc; use futures::{future, Sink, Stream}; use futures01::task::AtomicTask; use std::{ sync::{atomic::AtomicUsize, Arc}, task::Poll, }; - use tokio::sync::mpsc; use tokio01_test::task::MockTask; #[tokio::test] async fn drop_when_full() { future::lazy(|cx| { - let (tx, rx) = mpsc::channel(3); + let (tx, rx) = mpsc::channel(2); - let mut tx = Box::pin(DropWhenFull::new(BoundedSink::new(tx))); + let mut tx = Box::pin(DropWhenFull::new(tx)); assert_eq!(tx.as_mut().poll_ready(cx), Poll::Ready(Ok(()))); assert_eq!(tx.as_mut().start_send(1), Ok(())); diff --git a/src/dns.rs b/src/dns.rs index 1bbf026c90e9a..5834fad0e9bf7 100644 --- a/src/dns.rs +++ b/src/dns.rs @@ -1,5 +1,5 @@ use futures::{future::BoxFuture, FutureExt}; -use hyper::client::connect::dns::Name as Name13; +use hyper::client::connect::dns::Name; use snafu::ResultExt; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr, ToSocketAddrs}, @@ -55,7 +55,7 @@ impl Iterator for LookupIp { } } -impl Service for Resolver { +impl Service for Resolver { type Response = LookupIp; type Error = DnsError; type Future = BoxFuture<'static, Result>; @@ -64,7 +64,7 @@ impl Service for Resolver { Ok(()).into() } - fn call(&mut self, name: Name13) -> Self::Future { + fn call(&mut self, name: Name) -> Self::Future { self.lookup_ip(name.as_str().to_owned()).boxed() } } diff --git a/src/expiring_hash_map.rs b/src/expiring_hash_map.rs index 281f6f2709416..796592a7bfad2 100644 --- a/src/expiring_hash_map.rs +++ b/src/expiring_hash_map.rs @@ -7,7 +7,8 @@ use std::collections::HashMap; use std::fmt; use std::hash::Hash; use std::time::{Duration, Instant}; -use tokio::time::{delay_queue, DelayQueue, Error}; +use tokio::time::error::Error; +use tokio_util::time::{delay_queue, DelayQueue}; /// An expired item, holding the value and the key with an expiration /// information. @@ -154,7 +155,7 @@ where /// # Examples /// /// ```rust - /// # let mut rt = tokio::runtime::Runtime::new().unwrap(); + /// # let rt = tokio::runtime::Runtime::new().unwrap(); /// # rt.block_on(async { /// use vector::expiring_hash_map::ExpiringHashMap; /// use std::time::Duration; @@ -173,7 +174,7 @@ where /// } /// Some(Err(error)) => panic!(format!("Timer error: {:?}", error)), /// }, - /// _ = tokio::time::delay_for(Duration::from_millis(100)) => map.insert( + /// _ = tokio::time::sleep(Duration::from_millis(100)) => map.insert( /// "key".to_owned(), /// "val".to_owned(), /// Duration::from_millis(30), @@ -275,7 +276,7 @@ mod tests { // Sleep twice the ttl, to guarantee we're over the deadline. assert_eq!(fut.is_woken(), false); - tokio::time::delay_for(ttl * 2).await; + tokio::time::sleep(ttl * 2).await; assert_eq!(fut.is_woken(), true); // Then, after deadline, has to be ready. diff --git a/src/http.rs b/src/http.rs index 00f809cdf337d..799f0b322f808 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1,5 +1,4 @@ use crate::{ - dns::Resolver, internal_events::http_client, tls::{tls_connector_builder, MaybeTlsSettings, TlsError}, }; @@ -37,7 +36,7 @@ pub enum HttpError { pub type HttpClientFuture = >>::Future; pub struct HttpClient { - client: Client>, B>, + client: Client, B>, span: Span, user_agent: HeaderValue, } @@ -49,7 +48,7 @@ where B::Error: Into, { pub fn new(tls_settings: impl Into) -> Result, HttpError> { - let mut http = HttpConnector::new_with_resolver(Resolver); + let mut http = HttpConnector::new(); http.enforce_http(false); let settings = tls_settings.into(); diff --git a/src/kubernetes/debounce.rs b/src/kubernetes/debounce.rs index 3fc9e709b9094..6f9d11e9b11d2 100644 --- a/src/kubernetes/debounce.rs +++ b/src/kubernetes/debounce.rs @@ -4,7 +4,7 @@ //! and the [`Debounce::debounced`] will be resolved only once. use std::{future::pending, time::Duration}; -use tokio::time::{delay_until, Instant}; +use tokio::time::{sleep_until, Instant}; /// Provides an arbitrary signal debouncing. pub struct Debounce { @@ -40,7 +40,7 @@ impl Debounce { None => pending().await, }; - delay_until(sequence_start).await; + sleep_until(sequence_start).await; self.sequence_start = None; } diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index af82ccabaf618..47f75c25bcde3 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -13,7 +13,7 @@ use k8s_openapi::{ use snafu::Snafu; use std::convert::Infallible; use std::time::Duration; -use tokio::{select, time::delay_for}; +use tokio::{select, time::sleep}; /// Watches remote Kubernetes resources and maintains a local representation of /// the remote state. "Reflects" the remote state locally. @@ -77,7 +77,7 @@ where 'outer: loop { // For the next pause duration we won't get any updates. // This is better than flooding k8s api server with requests. - delay_for(self.pause_between_requests).await; + sleep(self.pause_between_requests).await; let invocation_result = self.issue_request().await; let stream = match invocation_result { diff --git a/src/kubernetes/state/delayed_delete.rs b/src/kubernetes/state/delayed_delete.rs index c9f8d8262fbea..c6d3e426b2bdf 100644 --- a/src/kubernetes/state/delayed_delete.rs +++ b/src/kubernetes/state/delayed_delete.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use futures::{future::BoxFuture, FutureExt}; use std::{collections::VecDeque, time::Duration}; -use tokio::time::{delay_until, Instant}; +use tokio::time::{sleep_until, timeout_at, Instant}; /// A [`super::Write`] implementation that wraps another [`super::Write`] and /// delays the delete calls. @@ -15,7 +15,7 @@ where { inner: T, queue: VecDeque<(::Item, Instant)>, - delay_for: Duration, + sleep: Duration, } impl Writer @@ -24,12 +24,12 @@ where ::Item: Send + Sync, { /// Take a [`super::Write`] and return it wrapped with [`Writer`]. - pub fn new(inner: T, delay_for: Duration) -> Self { + pub fn new(inner: T, sleep: Duration) -> Self { let queue = VecDeque::new(); Self { inner, queue, - delay_for, + sleep, } } } @@ -41,7 +41,7 @@ where { /// Schedules the delayed deletion of the item at the future. pub fn schedule_delete(&mut self, item: ::Item) { - let deadline = Instant::now() + self.delay_for; + let deadline = Instant::now() + self.sleep; self.queue.push_back((item, deadline)); } @@ -92,7 +92,7 @@ where } async fn delete(&mut self, item: Self::Item) { - let deadline = Instant::now() + self.delay_for; + let deadline = Instant::now() + self.sleep; self.queue.push_back((item, deadline)); } @@ -109,16 +109,17 @@ where ::Item: Send + Sync, { fn maintenance_request(&mut self) -> Option> { - let delayed_delete_deadline = self.next_deadline().map(delay_until); + let delayed_delete_deadline = self.next_deadline(); let downstream = self.inner.maintenance_request(); match (downstream, delayed_delete_deadline) { (Some(downstream), Some(delayed_delete_deadline)) => { - let fut = futures::future::select(downstream, delayed_delete_deadline) - .map(|either| either.factor_first().0); + let fut = timeout_at(delayed_delete_deadline, downstream).map(|_| ()); Some(Box::pin(fut)) } - (None, Some(delayed_delete_deadline)) => Some(Box::pin(delayed_delete_deadline)), + (None, Some(delayed_delete_deadline)) => { + Some(Box::pin(sleep_until(delayed_delete_deadline))) + } (Some(downstream), None) => Some(downstream), (None, None) => None, } diff --git a/src/kubernetes/state/evmap.rs b/src/kubernetes/state/evmap.rs index eead9467a1126..23c388afd463d 100644 --- a/src/kubernetes/state/evmap.rs +++ b/src/kubernetes/state/evmap.rs @@ -214,7 +214,7 @@ mod tests { assert_eq!(state_reader.is_empty(), true); - tokio::time::delay_for(flush_debounce_timeout * 2).await; + tokio::time::sleep(flush_debounce_timeout * 2).await; let mut state_writer = join.await.unwrap(); assert_eq!(state_reader.is_empty(), false); diff --git a/src/kubernetes/stream.rs b/src/kubernetes/stream.rs index 319f49673ea63..f5a4a8c361923 100644 --- a/src/kubernetes/stream.rs +++ b/src/kubernetes/stream.rs @@ -24,8 +24,8 @@ where pin_mut!(body); while let Some(buf) = body.data().await { - let mut buf = buf.context(Reading)?; - let chunk = buf.to_bytes(); + let buf = buf.context(Reading)?; + let chunk = buf.chunk(); let responses = decoder.process_next_chunk(chunk.as_ref()); emit!(internal_events::ChunkProcessed{ byte_size: chunk.len() }); for response in responses { diff --git a/src/lib.rs b/src/lib.rs index f303b8c354775..d97b414c1f1dc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -74,7 +74,6 @@ pub mod trace; pub mod transforms; pub mod trigger; pub mod types; -#[cfg(any(feature = "sources-utils-udp", feature = "sinks-utils-udp"))] pub mod udp; pub mod unit_test; pub(crate) mod utilization; diff --git a/src/line_agg.rs b/src/line_agg.rs index 745182d01cede..8bc231506c75e 100644 --- a/src/line_agg.rs +++ b/src/line_agg.rs @@ -14,7 +14,7 @@ use std::{ pin::Pin, task::{Context, Poll}, }; -use tokio::time::DelayQueue; +use tokio_util::time::DelayQueue; /// The mode of operation of the line aggregator. #[derive(Debug, Hash, Clone, PartialEq, Deserialize, Serialize)] diff --git a/src/pipeline.rs b/src/pipeline.rs index 13f6c49a514fe..814bac21d71b5 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -1,7 +1,6 @@ use crate::{internal_events::EventOut, transforms::FunctionTransform, Event}; -use futures::{task::Poll, Sink}; +use futures::{channel::mpsc, task::Poll, Sink}; use std::{collections::VecDeque, fmt, pin::Pin, task::Context}; -use tokio::sync::mpsc; #[derive(Debug)] pub struct ClosedError; @@ -31,8 +30,6 @@ impl Pipeline { &mut self, cx: &mut Context<'_>, ) -> Poll>::Error>> { - use mpsc::error::TrySendError::*; - while let Some(event) = self.enqueued.pop_front() { match self.inner.poll_ready(cx) { Poll::Pending => { @@ -49,15 +46,16 @@ impl Pipeline { Ok(()) => { // we good, keep looping } - Err(Full(_item)) => { + Err(error) if error.is_full() => { // We only try to send after a successful call to poll_ready, which reserves // space for us in the channel. That makes this branch unreachable as long as // the channel implementation fulfills its own contract. panic!("Channel was both ready and full; this is a bug.") } - Err(Closed(_item)) => { + Err(error) if error.is_disconnected() => { return Poll::Ready(Err(ClosedError)); } + Err(_) => unreachable!(), } } Poll::Ready(Ok(())) diff --git a/src/sink.rs b/src/sink.rs index 87a8d4b9920c9..c45805507c972 100644 --- a/src/sink.rs +++ b/src/sink.rs @@ -31,7 +31,6 @@ use std::{ pin::Pin, task::{Context, Poll}, }; -use tokio::sync::mpsc; impl VecSinkExt for T where T: Sink {} @@ -91,34 +90,3 @@ where } } } - -/// Wrapper for mpsc::Sender to turn it into a Sink. -pub struct BoundedSink { - sender: mpsc::Sender, -} - -impl BoundedSink { - pub fn new(sender: mpsc::Sender) -> Self { - Self { sender } - } -} - -impl Sink for BoundedSink { - type Error = (); - fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.sender - .poll_ready(cx) - .map_err(|error| error!(message = "Sender error.", %error)) - } - fn start_send(mut self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> { - self.sender - .try_send(item) - .map_err(|error| error!(message = "Sender error.", %error)) - } - fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - fn poll_close(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } -} diff --git a/src/sinks/aws_kinesis_firehose.rs b/src/sinks/aws_kinesis_firehose.rs index f18adfee32f02..3a4a7bd1e28d6 100644 --- a/src/sinks/aws_kinesis_firehose.rs +++ b/src/sinks/aws_kinesis_firehose.rs @@ -305,7 +305,7 @@ mod integration_tests { use rusoto_es::{CreateElasticsearchDomainRequest, Es, EsClient}; use rusoto_firehose::{CreateDeliveryStreamInput, ElasticsearchDestinationConfiguration}; use serde_json::{json, Value}; - use tokio::time::{delay_for, Duration}; + use tokio::time::{sleep, Duration}; #[tokio::test] async fn firehose_put_records() { @@ -349,7 +349,7 @@ mod integration_tests { let _ = sink.send_all(&mut events).await.unwrap(); - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let config = ElasticSearchConfig { auth: Some(ElasticSearchAuth::Aws(AwsAuthentication::Default {})), diff --git a/src/sinks/aws_kinesis_streams.rs b/src/sinks/aws_kinesis_streams.rs index e571fc8b237fb..af616577d336e 100644 --- a/src/sinks/aws_kinesis_streams.rs +++ b/src/sinks/aws_kinesis_streams.rs @@ -398,7 +398,7 @@ mod integration_tests { use rusoto_core::Region; use rusoto_kinesis::{Kinesis, KinesisClient}; use std::sync::Arc; - use tokio::time::{delay_for, Duration}; + use tokio::time::{sleep, Duration}; #[tokio::test] async fn kinesis_put_records() { @@ -438,7 +438,7 @@ mod integration_tests { let _ = sink.send_all(&mut events).await.unwrap(); - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let timestamp = timestamp as f64 / 1000.0; let records = fetch_records(stream, timestamp, region).await.unwrap(); @@ -509,7 +509,7 @@ mod integration_tests { // // I initially tried using `wait_for` with `DescribeStream` but localstack would // successfully return the stream before it was able to accept PutRecords requests - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; } fn gen_stream() -> String { diff --git a/src/sinks/aws_s3.rs b/src/sinks/aws_s3.rs index 8250a00cccc5b..da457acce7e24 100644 --- a/src/sinks/aws_s3.rs +++ b/src/sinks/aws_s3.rs @@ -239,6 +239,7 @@ impl S3SinkConfig { pub async fn healthcheck(self, client: S3Client) -> crate::Result<()> { let req = client.head_bucket(HeadBucketRequest { bucket: self.bucket.clone(), + expected_bucket_owner: None, }); match req.await { @@ -572,7 +573,7 @@ mod integration_tests { assert_downcast_matches, test_util::{random_lines_with_stream, random_string}, }; - use bytes::{buf::BufExt, BytesMut}; + use bytes::{Buf, BytesMut}; use flate2::read::GzDecoder; use pretty_assertions::assert_eq; use rusoto_core::region::Region; diff --git a/src/sinks/aws_sqs.rs b/src/sinks/aws_sqs.rs index 02cab26aab0cc..bdaff3fa5d452 100644 --- a/src/sinks/aws_sqs.rs +++ b/src/sinks/aws_sqs.rs @@ -320,7 +320,7 @@ mod integration_tests { use rusoto_core::Region; use rusoto_sqs::{CreateQueueRequest, GetQueueUrlRequest, ReceiveMessageRequest}; use std::collections::HashMap; - use tokio::time::{delay_for, Duration}; + use tokio::time::{sleep, Duration}; #[tokio::test] async fn sqs_send_message_batch() { @@ -354,7 +354,7 @@ mod integration_tests { let (mut input_lines, events) = random_lines_with_stream(100, 10); sink.send_all(&mut events.map(Ok)).await.unwrap(); - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let response = client .receive_message(ReceiveMessageRequest { diff --git a/src/sinks/blackhole.rs b/src/sinks/blackhole.rs index 0ff99c7ca016e..bd7624225c481 100644 --- a/src/sinks/blackhole.rs +++ b/src/sinks/blackhole.rs @@ -10,7 +10,7 @@ use async_trait::async_trait; use futures::{future, stream::BoxStream, FutureExt, StreamExt}; use serde::{Deserialize, Serialize}; use std::time::{Duration, Instant}; -use tokio::time::delay_until; +use tokio::time::sleep_until; pub struct BlackholeSink { total_events: usize, @@ -85,7 +85,7 @@ impl StreamSink for BlackholeSink { if let Some(rate) = self.config.rate { let until = self.last.unwrap_or_else(Instant::now) + Duration::from_secs_f32(1.0 / rate as f32); - delay_until(until.into()).await; + sleep_until(until.into()).await; self.last = Some(until); } diff --git a/src/sinks/file/mod.rs b/src/sinks/file/mod.rs index 79d157a7b83e0..d99c1407e6d95 100644 --- a/src/sinks/file/mod.rs +++ b/src/sinks/file/mod.rs @@ -11,7 +11,7 @@ use crate::{ }, template::Template, }; -use async_compression::tokio_02::write::GzipEncoder; +use async_compression::tokio::write::GzipEncoder; use async_trait::async_trait; use bytes::Bytes; use futures::{ @@ -349,7 +349,7 @@ mod tests { lines_from_file, lines_from_gzip_file, random_events_with_stream, random_lines_with_stream, temp_dir, temp_file, trace_init, }; - use futures::stream; + use futures::{stream, SinkExt}; use std::convert::TryInto; #[test] @@ -509,7 +509,7 @@ mod tests { let mut sink = FileSink::new(&config, Acker::Null); let (mut input, _events) = random_lines_with_stream(10, 64); - let (mut tx, rx) = tokio::sync::mpsc::channel(1); + let (mut tx, rx) = futures::channel::mpsc::channel(0); let _ = tokio::spawn(async move { sink.run(Box::pin(rx)).await }); @@ -519,7 +519,7 @@ mod tests { } // wait for file to go idle and be closed - tokio::time::delay_for(Duration::from_secs(2)).await; + tokio::time::sleep(Duration::from_secs(2)).await; // trigger another write let last_line = "i should go at the end"; @@ -527,7 +527,7 @@ mod tests { input.push(String::from(last_line)); // wait for another flush - tokio::time::delay_for(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_secs(1)).await; // make sure we appended instead of overwriting let output = lines_from_file(template); diff --git a/src/sinks/gcp/mod.rs b/src/sinks/gcp/mod.rs index 3f588a984ae00..4addeb76c7f64 100644 --- a/src/sinks/gcp/mod.rs +++ b/src/sinks/gcp/mod.rs @@ -15,6 +15,7 @@ use smpl_jwt::Jwt; use snafu::{ResultExt, Snafu}; use std::sync::{Arc, RwLock}; use std::time::Duration; +use tokio_stream::wrappers::IntervalStream; pub mod cloud_storage; pub mod pubsub; @@ -142,7 +143,7 @@ impl GcpCredentials { let this = self.clone(); let period = this.token.read().unwrap().expires_in() as u64 / 2; - let interval = tokio::time::interval(Duration::from_secs(period)); + let interval = IntervalStream::new(tokio::time::interval(Duration::from_secs(period))); let task = interval.for_each(move |_| { let this = this.clone(); async move { diff --git a/src/sinks/http.rs b/src/sinks/http.rs index 06f699da43308..3f9d8447636bf 100644 --- a/src/sinks/http.rs +++ b/src/sinks/http.rs @@ -303,15 +303,14 @@ mod tests { }, test_util::{next_addr, random_lines_with_stream}, }; - use bytes::{buf::BufExt, Bytes}; + use bytes::{Buf, Bytes}; use flate2::read::GzDecoder; - use futures::{stream, StreamExt}; + use futures::{channel::mpsc, stream, StreamExt}; use headers::{Authorization, HeaderMapExt}; use http::request::Parts; use hyper::Method; use serde::Deserialize; use std::io::{BufRead, BufReader}; - use tokio::sync::mpsc::Receiver; #[test] fn generate_config() { @@ -575,7 +574,7 @@ mod tests { // its accepting socket. The delay below ensures that the sink // attempts to connect at least once before creating the // listening socket. - tokio::time::delay_for(std::time::Duration::from_secs(2)).await; + tokio::time::sleep(std::time::Duration::from_secs(2)).await; let (rx, trigger, server) = build_test_server(in_addr); tokio::spawn(server); @@ -650,7 +649,7 @@ mod tests { } async fn get_received( - rx: Receiver<(Parts, Bytes)>, + rx: mpsc::Receiver<(Parts, Bytes)>, assert_parts: impl Fn(Parts), ) -> Vec { rx.flat_map(|(parts, body)| { diff --git a/src/sinks/kafka.rs b/src/sinks/kafka.rs index 499851acbfa4a..015a4cb1e720f 100644 --- a/src/sinks/kafka.rs +++ b/src/sinks/kafka.rs @@ -17,7 +17,7 @@ use futures::{ }; use rdkafka::{ consumer::{BaseConsumer, Consumer}, - error::{KafkaError, RDKafkaError}, + error::{KafkaError, RDKafkaErrorCode}, producer::{DeliveryFuture, FutureProducer, FutureRecord}, ClientConfig, }; @@ -30,7 +30,7 @@ use std::{ sync::Arc, task::{Context, Poll}, }; -use tokio::time::{delay_for, Duration}; +use tokio::time::{sleep, Duration}; // Maximum number of futures blocked by [send_result](https://docs.rs/rdkafka/0.24.0/rdkafka/producer/future_producer/struct.FutureProducer.html#method.send_result) const SEND_RESULT_LIMIT: usize = 5; @@ -313,11 +313,11 @@ impl Sink for KafkaSink { // See item 4 on GitHub: https://github.com/timberio/vector/pull/101#issue-257150924 // https://docs.rs/rdkafka/0.24.0/src/rdkafka/producer/future_producer.rs.html#296 Err((error, future_record)) - if error == KafkaError::MessageProduction(RDKafkaError::QueueFull) => + if error == KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull) => { debug!(message = "The rdkafka queue full.", %error, %seqno, internal_log_rate_secs = 1); record = future_record; - delay_for(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; } Err((error, _)) => break Err(error), } @@ -798,7 +798,9 @@ mod integration_test { let _ = kafka_auth.apply(&mut client_config).unwrap(); let mut tpl = TopicPartitionList::new(); - tpl.add_partition(&topic, 0).set_offset(Offset::Beginning); + tpl.add_partition(&topic, 0) + .set_offset(Offset::Beginning) + .unwrap(); let consumer: BaseConsumer = client_config.create().unwrap(); consumer.assign(&tpl).unwrap(); diff --git a/src/sinks/new_relic_logs.rs b/src/sinks/new_relic_logs.rs index bc4dbfdc08413..e0c21c79f8e07 100644 --- a/src/sinks/new_relic_logs.rs +++ b/src/sinks/new_relic_logs.rs @@ -172,7 +172,7 @@ mod tests { test_util::next_addr, Event, }; - use bytes::buf::BufExt; + use bytes::Buf; use futures::{stream, StreamExt}; use hyper::Method; use serde_json::Value; diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 8638366ef7aa1..0e7b23ee9541b 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -346,6 +346,7 @@ mod tests { use indoc::indoc; use pretty_assertions::assert_eq; use tokio::{sync::mpsc, time}; + use tokio_stream::wrappers::UnboundedReceiverStream; #[test] fn generate_config() { @@ -407,13 +408,13 @@ mod tests { }; let (sink, _) = config.build(SinkContext::new_test()).await.unwrap(); let (tx, rx) = mpsc::unbounded_channel(); - tokio::spawn(sink.run(Box::pin(rx))); + tokio::spawn(sink.run(Box::pin(UnboundedReceiverStream::new(rx)))); for event in events { tx.send(event).expect("Failed to send event."); } - time::delay_for(time::Duration::from_millis(100)).await; + time::sleep(time::Duration::from_millis(100)).await; let request = Request::get(format!("{}://{}/metrics", proto, address)) .body(Body::empty()) @@ -542,6 +543,7 @@ mod integration_tests { use chrono::Utc; use serde_json::Value; use tokio::{sync::mpsc, time}; + use tokio_stream::wrappers::UnboundedReceiverStream; const PROMETHEUS_ADDRESS: &str = "127.0.0.1:9101"; @@ -550,7 +552,7 @@ mod integration_tests { trace_init(); prometheus_scrapes_metrics().await; - time::delay_for(time::Duration::from_millis(500)).await; + time::sleep(time::Duration::from_millis(500)).await; reset_on_flush_period().await; } @@ -563,13 +565,13 @@ mod integration_tests { }; let (sink, _) = config.build(SinkContext::new_test()).await.unwrap(); let (tx, rx) = mpsc::unbounded_channel(); - tokio::spawn(sink.run(Box::pin(rx))); + tokio::spawn(sink.run(Box::pin(UnboundedReceiverStream::new(rx)))); let (name, event) = tests::create_metric_gauge(None, 123.4); tx.send(event).expect("Failed to send."); // Wait a bit for the prometheus server to scrape the metrics - time::delay_for(time::Duration::from_secs(2)).await; + time::sleep(time::Duration::from_secs(2)).await; // Now try to download them from prometheus let result = prometheus_query(&name).await; @@ -596,7 +598,7 @@ mod integration_tests { }; let (sink, _) = config.build(SinkContext::new_test()).await.unwrap(); let (tx, rx) = mpsc::unbounded_channel(); - tokio::spawn(sink.run(Box::pin(rx))); + tokio::spawn(sink.run(Box::pin(UnboundedReceiverStream::new(rx)))); let (name1, event) = tests::create_metric_set(None, vec!["0", "1", "2"]); tx.send(event).expect("Failed to send."); @@ -604,7 +606,7 @@ mod integration_tests { tx.send(event).expect("Failed to send."); // Wait a bit for the prometheus server to scrape the metrics - time::delay_for(time::Duration::from_secs(2)).await; + time::sleep(time::Duration::from_secs(2)).await; // Now try to download them from prometheus let result = prometheus_query(&name1).await; @@ -619,7 +621,7 @@ mod integration_tests { ); // Wait a bit for expired metrics - time::delay_for(time::Duration::from_secs(3)).await; + time::sleep(time::Duration::from_secs(3)).await; let (name1, event) = tests::create_metric_set(Some(name1), vec!["6", "7"]); tx.send(event).expect("Failed to send."); @@ -627,7 +629,7 @@ mod integration_tests { tx.send(event).expect("Failed to send."); // Wait a bit for the prometheus server to scrape the metrics - time::delay_for(time::Duration::from_secs(2)).await; + time::sleep(time::Duration::from_secs(2)).await; // Now try to download them from prometheus let result = prometheus_query(&name1).await; diff --git a/src/sinks/socket.rs b/src/sinks/socket.rs index 0403c70d7d173..a3110c28dec03 100644 --- a/src/sinks/socket.rs +++ b/src/sinks/socket.rs @@ -96,8 +96,9 @@ mod test { }; use tokio::{ net::TcpListener, - time::{delay_for, timeout, Duration}, + time::{sleep, timeout, Duration}, }; + use tokio_stream::wrappers::TcpListenerStream; use tokio_util::codec::{FramedRead, LinesCodec}; #[test] @@ -188,9 +189,8 @@ mod test { #[tokio::test] async fn tcp_stream_detects_disconnect() { use crate::tls::{self, MaybeTlsIncomingStream, MaybeTlsSettings, TlsConfig, TlsOptions}; - use futures::{future, FutureExt, StreamExt}; + use futures::{channel::mpsc, future, FutureExt, SinkExt, StreamExt}; use std::{ - net::Shutdown, pin::Pin, sync::{ atomic::{AtomicUsize, Ordering}, @@ -199,12 +199,12 @@ mod test { task::Poll, }; use tokio::{ - io::AsyncRead, + io::{AsyncRead, AsyncWriteExt, ReadBuf}, net::TcpStream, - sync::mpsc, task::yield_now, time::{interval, Duration}, }; + use tokio_stream::wrappers::IntervalStream; trace_init(); @@ -228,7 +228,7 @@ mod test { }; let context = SinkContext::new_test(); let (sink, _healthcheck) = config.build(context).await.unwrap(); - let (mut sender, receiver) = mpsc::channel::>(1); + let (mut sender, receiver) = mpsc::channel::>(0); let jh1 = tokio::spawn(async move { let stream = receiver .take_while(|event| ready(event.is_some())) @@ -261,17 +261,26 @@ mod test { let msg_counter1 = Arc::clone(&msg_counter1); let mut stream: MaybeTlsIncomingStream = connection.unwrap(); + future::poll_fn(move |cx| loop { if let Some(fut) = close_rx.as_mut() { if let Poll::Ready(()) = fut.poll_unpin(cx) { - stream.get_ref().unwrap().shutdown(Shutdown::Write).unwrap(); + stream + .get_mut() + .unwrap() + .shutdown() + .now_or_never() + .unwrap() + .unwrap(); close_rx = None; } } - return match Pin::new(&mut stream).poll_read(cx, &mut [0u8; 11]) { - Poll::Ready(Ok(n)) => { - if n == 0 { + let mut buf = [0u8; 11]; + let mut buf = ReadBuf::new(&mut buf); + return match Pin::new(&mut stream).poll_read(cx, &mut buf) { + Poll::Ready(Ok(())) => { + if buf.filled().is_empty() { Poll::Ready(()) } else { msg_counter1.fetch_add(1, Ordering::SeqCst); @@ -294,7 +303,7 @@ mod test { // Loop and check for 10 events, we should always get 10 events. Once, // we have 10 events we can tell the server to shutdown to simulate the // remote shutting down on an idle connection. - interval(Duration::from_millis(100)) + IntervalStream::new(interval(Duration::from_millis(100))) .take(500) .take_while(|_| ready(msg_counter.load(Ordering::SeqCst) != 10)) .for_each(|_| ready(())) @@ -342,9 +351,7 @@ mod test { // First listener let mut count = 20usize; - TcpListener::bind(addr) - .await - .unwrap() + TcpListenerStream::new(TcpListener::bind(addr).await.unwrap()) .next() .await .unwrap() @@ -365,7 +372,7 @@ mod test { // Disconnect if cfg!(windows) { // Gives Windows time to release the addr port. - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; } // Second listener diff --git a/src/sinks/splunk_hec.rs b/src/sinks/splunk_hec.rs index cc2729c346252..0be6ea15180fa 100644 --- a/src/sinks/splunk_hec.rs +++ b/src/sinks/splunk_hec.rs @@ -457,7 +457,7 @@ mod integration_tests { use futures::stream; use serde_json::Value as JsonValue; use std::{future::ready, net::SocketAddr}; - use tokio::time::{delay_for, Duration}; + use tokio::time::{sleep, Duration}; use warp::Filter; const USERNAME: &str = "admin"; @@ -576,7 +576,7 @@ mod integration_tests { break; } - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; } assert!(found_all); diff --git a/src/sinks/statsd.rs b/src/sinks/statsd.rs index 111e1f9eace09..11fd9c2ea1c1b 100644 --- a/src/sinks/statsd.rs +++ b/src/sinks/statsd.rs @@ -240,9 +240,8 @@ mod test { use super::*; use crate::{event::Metric, test_util::*}; use bytes::Bytes; - use futures::TryStreamExt; + use futures::{channel::mpsc, TryStreamExt}; use tokio::net::UdpSocket; - use tokio::sync::mpsc; use tokio_util::{codec::BytesCodec, udp::UdpFramed}; #[cfg(feature = "sources-statsd")] @@ -428,7 +427,7 @@ mod test { .with_namespace(Some("vector")), ), ]; - let (mut tx, rx) = mpsc::channel(1); + let (mut tx, rx) = mpsc::channel(0); let socket = UdpSocket::bind(addr).await.unwrap(); tokio::spawn(async move { diff --git a/src/sinks/util/adaptive_concurrency/semaphore.rs b/src/sinks/util/adaptive_concurrency/semaphore.rs index 4f73300304ab9..f24778ecd1e96 100644 --- a/src/sinks/util/adaptive_concurrency/semaphore.rs +++ b/src/sinks/util/adaptive_concurrency/semaphore.rs @@ -2,7 +2,10 @@ // clippy's warning that an AtomicUsize would work better is incorrect. #![allow(clippy::mutex_atomic)] -use futures::{future::BoxFuture, ready}; +use futures::{ + future::{BoxFuture, FutureExt}, + ready, +}; use std::future::Future; use std::mem::{drop, replace}; use std::pin::Pin; @@ -31,7 +34,11 @@ impl ShrinkableSemaphore { ) -> impl Future + Send + 'static { MaybeForgetFuture { master: Arc::clone(&self), - future: Box::pin(Arc::clone(&self.semaphore).acquire_owned()), + future: Box::pin( + Arc::clone(&self.semaphore) + .acquire_owned() + .map(|r| r.expect("Semaphore has been closed")), + ), } } @@ -84,7 +91,9 @@ impl Future for MaybeForgetFuture { let permit = ready!(self.future.as_mut().poll(cx)); permit.forget(); *to_forget -= 1; - let future = Arc::clone(&self.master.semaphore).acquire_owned(); + let future = Arc::clone(&self.master.semaphore) + .acquire_owned() + .map(|r| r.expect("Semaphore is closed")); drop(replace(&mut self.future, Box::pin(future))); } drop(to_forget); diff --git a/src/sinks/util/adaptive_concurrency/tests.rs b/src/sinks/util/adaptive_concurrency/tests.rs index 1e4ed455fd6f7..463450c0b5d68 100644 --- a/src/sinks/util/adaptive_concurrency/tests.rs +++ b/src/sinks/util/adaptive_concurrency/tests.rs @@ -37,7 +37,7 @@ use std::{ sync::{Arc, Mutex}, task::Poll, }; -use tokio::time::{self, delay_for, Duration, Instant}; +use tokio::time::{self, sleep, Duration, Instant}; use tower::Service; #[derive(Copy, Clone, Debug, Derivative, Deserialize, Serialize)] @@ -263,7 +263,7 @@ fn respond_after( stats: Arc>, ) -> BoxFuture<'static, Result> { Box::pin(async move { - delay_for(Duration::from_secs_f64(delay)).await; + sleep(Duration::from_secs_f64(delay)).await; let mut stats = stats.lock().expect("Poisoned stats lock"); stats.end_request(Instant::now(), matches!(response, Ok(Response::Ok))); response @@ -382,7 +382,7 @@ async fn run_test(params: TestParams) -> TestResults { // This is crude and dumb, but it works, and the tests run fast and // the results are highly repeatable. while stats.lock().expect("Poisoned stats lock").completed < params.requests { - time::advance(Duration::from_millis(1)).await; + time::advance(Duration::from_millis(0)).await; } topology.stop().await; @@ -644,7 +644,7 @@ async fn all_tests() { // The first delay takes just slightly longer than all the rest, // which causes the first test to run differently than all the // others. Throw in a dummy delay to take up this delay "slack". - delay_for(Duration::from_millis(1)).await; + sleep(Duration::from_millis(1)).await; time::advance(Duration::from_millis(1)).await; // Then run all the tests diff --git a/src/sinks/util/http.rs b/src/sinks/util/http.rs index d90e903b3d076..ca220429a8f33 100644 --- a/src/sinks/util/http.rs +++ b/src/sinks/util/http.rs @@ -358,7 +358,10 @@ where let response = http_client.call(request).await?; let (parts, body) = response.into_parts(); let mut body = body::aggregate(body).await?; - Ok(hyper::Response::from_parts(parts, body.to_bytes())) + Ok(hyper::Response::from_parts( + parts, + body.copy_to_bytes(body.remaining()), + )) }) } } @@ -481,10 +484,10 @@ mod test { let mut tx = tx.clone(); async move { - let body = hyper::body::aggregate(req.into_body()) + let mut body = hyper::body::aggregate(req.into_body()) .await .map_err(|error| format!("error: {}", error))?; - let string = String::from_utf8(body.bytes().into()) + let string = String::from_utf8(body.copy_to_bytes(body.remaining()).to_vec()) .map_err(|_| "Wasn't UTF-8".to_string())?; tx.try_send(string).map_err(|_| "Send error".to_string())?; @@ -501,7 +504,7 @@ mod test { } }); - tokio::time::delay_for(std::time::Duration::from_millis(50)).await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; service.call(request).await.unwrap(); let (body, _rest) = rx.into_future().await; diff --git a/src/sinks/util/mod.rs b/src/sinks/util/mod.rs index 2f48f444c7f46..26c8af74bd28a 100644 --- a/src/sinks/util/mod.rs +++ b/src/sinks/util/mod.rs @@ -11,7 +11,6 @@ pub mod statistic; pub mod tcp; #[cfg(test)] pub mod test; -#[cfg(feature = "socket2")] pub mod udp; #[cfg(all(any(feature = "sinks-socket", feature = "sinks-statsd"), unix))] pub mod unix; diff --git a/src/sinks/util/retries.rs b/src/sinks/util/retries.rs index fc96738d8b7a2..c42500253ee21 100644 --- a/src/sinks/util/retries.rs +++ b/src/sinks/util/retries.rs @@ -7,7 +7,7 @@ use std::{ task::{Context, Poll}, time::Duration, }; -use tokio::time::{delay_for, Delay}; +use tokio::time::{sleep, Sleep}; use tower::{retry::Policy, timeout::error::Elapsed}; pub enum RetryAction { @@ -41,7 +41,7 @@ pub struct FixedRetryPolicy { } pub struct RetryPolicyFuture { - delay: Delay, + delay: Pin>, policy: FixedRetryPolicy, } @@ -79,7 +79,7 @@ impl FixedRetryPolicy { fn build_retry(&self) -> RetryPolicyFuture { let policy = self.advance(); - let delay = delay_for(self.backoff()); + let delay = Box::pin(sleep(self.backoff())); debug!(message = "Retrying request.", delay_ms = %self.backoff().as_millis()); RetryPolicyFuture { delay, policy } diff --git a/src/sinks/util/sink.rs b/src/sinks/util/sink.rs index 29c7bb7350d87..bfa432a59a65c 100644 --- a/src/sinks/util/sink.rs +++ b/src/sinks/util/sink.rs @@ -55,7 +55,7 @@ use std::{ }; use tokio::{ sync::oneshot, - time::{delay_for, Delay, Duration}, + time::{sleep, Duration, Sleep}, }; use tower::{Service, ServiceBuilder}; use tracing_futures::Instrument; @@ -184,7 +184,7 @@ where batch: StatefulBatch, partitions: HashMap>, timeout: Duration, - lingers: HashMap, + lingers: HashMap>>, closing: bool, } @@ -252,8 +252,8 @@ where let batch = self.batch.fresh(); self.partitions.insert(partition.clone(), batch); - let delay = delay_for(self.timeout); - self.lingers.insert(partition.clone(), delay); + let delay = sleep(self.timeout); + self.lingers.insert(partition.clone(), Box::pin(delay)); }; if let PushResult::Overflow(item) = batch.push(item) { @@ -551,7 +551,7 @@ mod tests { _ => unreachable!(), }; - delay_for(duration).await; + sleep(duration).await; Ok::<(), Infallible>(()) }); diff --git a/src/sinks/util/tcp.rs b/src/sinks/util/tcp.rs index 21c9f32820e38..1b32a0d80c447 100644 --- a/src/sinks/util/tcp.rs +++ b/src/sinks/util/tcp.rs @@ -32,7 +32,11 @@ use std::{ task::{Context, Poll}, time::Duration, }; -use tokio::{io::AsyncRead, net::TcpStream, time::delay_for}; +use tokio::{ + io::{AsyncRead, ReadBuf}, + net::TcpStream, + time::sleep, +}; #[derive(Debug, Snafu)] enum TcpError { @@ -178,7 +182,7 @@ impl TcpConnector { } Err(error) => { emit!(TcpSocketConnectionFailed { error }); - delay_for(backoff.next().unwrap()).await; + sleep(backoff.next().unwrap()).await; } } } @@ -228,9 +232,11 @@ impl TcpSink { // If this returns `Poll::Pending` we know the connection is still // valid and the write will most likely succeed. let mut cx = Context::from_waker(noop_waker_ref()); - match Pin::new(stream).poll_read(&mut cx, &mut [0u8; 1]) { + let mut buf = [0u8; 1]; + let mut buf = ReadBuf::new(&mut buf); + match Pin::new(stream).poll_read(&mut cx, &mut buf) { Poll::Ready(Err(error)) => ShutdownCheck::Error(error), - Poll::Ready(Ok(0)) => { + Poll::Ready(Ok(())) if buf.filled().is_empty() => { // Maybe this is only a sign to close the channel, // in which case we should try to flush our buffers // before disconnecting. diff --git a/src/sinks/util/test.rs b/src/sinks/util/test.rs index 4126cc78b565d..bd4e387b68f87 100644 --- a/src/sinks/util/test.rs +++ b/src/sinks/util/test.rs @@ -3,14 +3,13 @@ use crate::{ Error, }; use bytes::Bytes; -use futures::{FutureExt, TryFutureExt}; +use futures::{channel::mpsc, FutureExt, SinkExt, TryFutureExt}; use hyper::{ service::{make_service_fn, service_fn}, Body, Request, Response, Server, }; use serde::Deserialize; use stream_cancel::{Trigger, Tripwire}; -use tokio::sync::mpsc; pub fn load_sink(config: &str) -> crate::Result<(T, SinkContext)> where diff --git a/src/sinks/util/udp.rs b/src/sinks/util/udp.rs index 0b740cfa75a44..62abf01abd1bf 100644 --- a/src/sinks/util/udp.rs +++ b/src/sinks/util/udp.rs @@ -1,5 +1,4 @@ use super::SinkBuildError; -#[cfg(unix)] use crate::udp; use crate::{ buffers::Acker, @@ -26,7 +25,7 @@ use std::{ task::{Context, Poll}, time::Duration, }; -use tokio::{net::UdpSocket, sync::oneshot, time::delay_for}; +use tokio::{net::UdpSocket, sync::oneshot, time::sleep}; #[derive(Debug, Snafu)] pub enum UdpError { @@ -124,9 +123,10 @@ impl UdpConnector { let socket = UdpSocket::bind(bind_address).await.context(BindError)?; - #[cfg(unix)] if let Some(send_buffer_bytes) = self.send_buffer_bytes { - udp::set_send_buffer_size(&socket, send_buffer_bytes); + if let Err(error) = udp::set_send_buffer_size(&socket, send_buffer_bytes) { + warn!(message = "Failed configuring send buffer size on UDP socket.", %error); + } } socket.connect(addr).await.context(ConnectError)?; @@ -144,7 +144,7 @@ impl UdpConnector { } Err(error) => { emit!(UdpSocketConnectionFailed { error }); - delay_for(backoff.next().unwrap()).await; + sleep(backoff.next().unwrap()).await; } } } diff --git a/src/sinks/util/unix.rs b/src/sinks/util/unix.rs index f2123af693964..1d4c12bc6bde1 100644 --- a/src/sinks/util/unix.rs +++ b/src/sinks/util/unix.rs @@ -22,7 +22,7 @@ use futures::{stream::BoxStream, SinkExt, StreamExt}; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; use std::{path::PathBuf, pin::Pin, sync::Arc, time::Duration}; -use tokio::{net::UnixStream, time::delay_for}; +use tokio::{net::UnixStream, time::sleep}; #[derive(Debug, Snafu)] pub enum UnixError { @@ -89,7 +89,7 @@ impl UnixConnector { error, path: &self.path }); - delay_for(backoff.next().unwrap()).await; + sleep(backoff.next().unwrap()).await; } } } diff --git a/src/sources/apache_metrics/mod.rs b/src/sources/apache_metrics/mod.rs index 2f8027c3374b5..31e90c5adcdc1 100644 --- a/src/sources/apache_metrics/mod.rs +++ b/src/sources/apache_metrics/mod.rs @@ -19,6 +19,7 @@ use std::{ future::ready, time::{Duration, Instant}, }; +use tokio_stream::wrappers::IntervalStream; mod parser; @@ -145,7 +146,7 @@ fn apache_metrics( let out = out.sink_map_err(|error| error!(message = "Error sending metric.", %error)); Box::pin( - tokio::time::interval(Duration::from_secs(interval)) + IntervalStream::new(tokio::time::interval(Duration::from_secs(interval))) .take_until(shutdown) .map(move |_| stream::iter(urls.clone())) .flatten() @@ -277,7 +278,7 @@ mod test { {Body, Response, Server}, }; use pretty_assertions::assert_eq; - use tokio::time::{delay_for, Duration}; + use tokio::time::{sleep, Duration}; #[test] fn generate_config() { @@ -358,7 +359,7 @@ Scoreboard: ____S_____I______R____I_______KK___D__C__G_L____________W___________ .unwrap(); tokio::spawn(source); - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let metrics = collect_ready(rx) .await @@ -424,7 +425,7 @@ Scoreboard: ____S_____I______R____I_______KK___D__C__G_L____________W___________ .unwrap(); tokio::spawn(source); - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let metrics = collect_ready(rx) .await @@ -463,7 +464,7 @@ Scoreboard: ____S_____I______R____I_______KK___D__C__G_L____________W___________ .unwrap(); tokio::spawn(source); - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let metrics = collect_ready(rx) .await diff --git a/src/sources/aws_ecs_metrics/mod.rs b/src/sources/aws_ecs_metrics/mod.rs index 9d298926d4371..86b5a62d5e9df 100644 --- a/src/sources/aws_ecs_metrics/mod.rs +++ b/src/sources/aws_ecs_metrics/mod.rs @@ -12,6 +12,7 @@ use hyper::{Body, Client, Request}; use serde::{Deserialize, Serialize}; use std::{env, time::Instant}; use tokio::time; +use tokio_stream::wrappers::IntervalStream; mod parser; @@ -128,7 +129,7 @@ async fn aws_ecs_metrics( let mut out = out.sink_map_err(|error| error!(message = "Error sending metric.", %error)); let interval = time::Duration::from_secs(interval); - let mut interval = time::interval(interval).take_until(shutdown); + let mut interval = IntervalStream::new(time::interval(interval)).take_until(shutdown); while interval.next().await.is_some() { let client = Client::new(); @@ -199,7 +200,7 @@ mod test { service::{make_service_fn, service_fn}, {Body, Response, Server}, }; - use tokio::time::{delay_for, Duration}; + use tokio::time::{sleep, Duration}; #[tokio::test] async fn test_aws_ecs_metrics_source() { @@ -529,7 +530,7 @@ mod test { .unwrap(); tokio::spawn(source); - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let metrics = collect_ready(rx) .await @@ -565,7 +566,7 @@ mod test { mod integration_tests { use super::*; use crate::test_util::collect_ready; - use tokio::time::{delay_for, Duration}; + use tokio::time::{sleep, Duration}; async fn scrape_metrics(endpoint: String, version: Version) { let (tx, rx) = Pipeline::new_test(); @@ -586,7 +587,7 @@ mod integration_tests { .unwrap(); tokio::spawn(source); - delay_for(Duration::from_secs(5)).await; + sleep(Duration::from_secs(5)).await; let metrics = collect_ready(rx).await; diff --git a/src/sources/aws_kinesis_firehose/errors.rs b/src/sources/aws_kinesis_firehose/errors.rs index 447035ef48e4d..7b663b3dd6938 100644 --- a/src/sources/aws_kinesis_firehose/errors.rs +++ b/src/sources/aws_kinesis_firehose/errors.rs @@ -82,9 +82,3 @@ impl RequestError { } } } - -impl From for warp::reject::Rejection { - fn from(error: RequestError) -> Self { - warp::reject::custom(error) - } -} diff --git a/src/sources/aws_kinesis_firehose/filters.rs b/src/sources/aws_kinesis_firehose/filters.rs index a21862536fd87..afbf1c78877bb 100644 --- a/src/sources/aws_kinesis_firehose/filters.rs +++ b/src/sources/aws_kinesis_firehose/filters.rs @@ -7,7 +7,7 @@ use crate::{ internal_events::{AwsKinesisFirehoseRequestError, AwsKinesisFirehoseRequestReceived}, Pipeline, }; -use bytes::{buf::BufExt, Bytes}; +use bytes::{Buf, Bytes}; use chrono::Utc; use flate2::read::GzDecoder; use snafu::ResultExt; diff --git a/src/sources/aws_kinesis_firehose/mod.rs b/src/sources/aws_kinesis_firehose/mod.rs index f2dd652c476b8..6a6a8fcef5c60 100644 --- a/src/sources/aws_kinesis_firehose/mod.rs +++ b/src/sources/aws_kinesis_firehose/mod.rs @@ -36,14 +36,12 @@ impl SourceConfig for AwsKinesisFirehoseConfig { let listener = tls.bind(&self.address).await?; Ok(Box::pin(async move { - let _ = warp::serve(svc) + warp::serve(svc) .serve_incoming_with_graceful_shutdown( listener.accept_stream(), - shutdown.clone().map(|_| ()), + shutdown.map(|_| ()), ) .await; - // We need to drop the last copy of ShutdownSignalToken only after server has shut down. - drop(shutdown); Ok(()) })) } @@ -86,12 +84,12 @@ mod tests { }; use chrono::{DateTime, SubsecRound, Utc}; use flate2::{read::GzEncoder, Compression}; + use futures::channel::mpsc; use pretty_assertions::assert_eq; use std::{ io::{Cursor, Read}, net::SocketAddr, }; - use tokio::sync::mpsc; #[test] fn generate_config() { diff --git a/src/sources/aws_s3/mod.rs b/src/sources/aws_s3/mod.rs index b2a0cfb7c29ea..dfc439ff6067a 100644 --- a/src/sources/aws_s3/mod.rs +++ b/src/sources/aws_s3/mod.rs @@ -160,7 +160,7 @@ fn s3_object_decoder( content_type: Option<&str>, body: rusoto_s3::StreamingBody, ) -> Box { - use async_compression::tokio_02::bufread; + use async_compression::tokio::bufread; let r = tokio::io::BufReader::new(body.into_async_read()); @@ -436,6 +436,7 @@ mod integration_tests { client .put_bucket_notification_configuration(PutBucketNotificationConfigurationRequest { bucket: bucket_name.clone(), + expected_bucket_owner: None, notification_configuration: NotificationConfiguration { queue_configurations: Some(vec![QueueConfiguration { events: vec!["s3:ObjectCreated:*".to_string()], diff --git a/src/sources/aws_s3/sqs.rs b/src/sources/aws_s3/sqs.rs index bd2974a2e8ec5..13f28d171c2ee 100644 --- a/src/sources/aws_s3/sqs.rs +++ b/src/sources/aws_s3/sqs.rs @@ -25,6 +25,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use snafu::{ResultExt, Snafu}; use std::{future::ready, time::Duration}; use tokio::time; +use tokio_stream::wrappers::IntervalStream; use tokio_util::codec::FramedRead; lazy_static! { @@ -156,7 +157,7 @@ impl Ingestor { } pub(super) async fn run(self, out: Pipeline, shutdown: ShutdownSignal) -> Result<(), ()> { - time::interval(self.poll_interval) + IntervalStream::new(time::interval(self.poll_interval)) .take_until(shutdown) .for_each(|_| self.run_once(&out)) .await; diff --git a/src/sources/datadog/logs.rs b/src/sources/datadog/logs.rs index 1431d60ff288f..bf78fb0ff2040 100644 --- a/src/sources/datadog/logs.rs +++ b/src/sources/datadog/logs.rs @@ -141,10 +141,10 @@ mod tests { test_util::{collect_n, next_addr, trace_init, wait_for_tcp}, Pipeline, }; + use futures::channel::mpsc; use http::HeaderMap; use pretty_assertions::assert_eq; use std::net::SocketAddr; - use tokio::sync::mpsc; #[test] fn generate_config() { diff --git a/src/sources/docker_logs.rs b/src/sources/docker_logs.rs index a72cc37eb34cb..3b718fa2c9889 100644 --- a/src/sources/docker_logs.rs +++ b/src/sources/docker_logs.rs @@ -434,7 +434,7 @@ impl DockerLogsSource { async fn run(mut self) { loop { tokio::select! { - value = self.main_recv.next() => { + value = self.main_recv.recv() => { match value { Some(message) => { match message { @@ -547,7 +547,7 @@ impl EventStreamBuilder { let this = self.clone(); tokio::spawn(async move { if let Some(duration) = backoff { - tokio::time::delay_for(duration).await; + tokio::time::sleep(duration).await; } match this .core @@ -605,13 +605,15 @@ impl EventStreamBuilder { // Create event streamer let mut partial_event_merge_state = None; + let core = Arc::clone(&self.core); + let events_stream = stream .map(|value| { match value { Ok(message) => Ok(info.new_event( message, - self.core.config.partial_event_marker_field.clone(), - self.core.config.auto_partial_merge, + core.config.partial_event_marker_field.clone(), + core.config.auto_partial_merge, &mut partial_event_merge_state, )), Err(error) => { @@ -630,7 +632,6 @@ impl EventStreamBuilder { container_id: Some(info.id.as_str()) }), }; - Err(()) } } @@ -640,7 +641,7 @@ impl EventStreamBuilder { .take_until(self.shutdown.clone()); let events_stream: Box + Unpin + Send> = - if let Some(ref line_agg_config) = self.core.line_agg_config { + if let Some(ref line_agg_config) = core.line_agg_config { Box::new(line_agg_adapter( events_stream, line_agg::Logic::new(line_agg_config.clone()), @@ -1026,8 +1027,18 @@ fn docker(host: Option, tls: Option) -> crate::Result Docker::connect_with_local_defaults().map_err(Into::into), - + None => { + // TODO: Use `connect_with_local_defaults` on all platforms. + // + // Using `connect_with_local_defaults` defers to `connect_with_named_pipe_defaults` on Windows. However, + // named pipes are currently disabled in Tokio. Tracking issue: + // https://github.com/fussybeaver/bollard/pull/138 + if cfg!(windows) { + Docker::connect_with_http_defaults().map_err(Into::into) + } else { + Docker::connect_with_local_defaults().map_err(Into::into) + } + } Some(host) => { let scheme = host .parse::() @@ -1056,8 +1067,17 @@ fn docker(host: Option, tls: Option) -> crate::Result { - Docker::connect_with_local(&host, DEFAULT_TIMEOUT, API_DEFAULT_VERSION) - .map_err(Into::into) + // TODO: Use `connect_with_local` on all platforms. + // + // Named pipes are currently disabled in Tokio. Tracking issue: + // https://github.com/fussybeaver/bollard/pull/138 + if cfg!(windows) { + warn!("Named pipes are currently not available on Windows, trying to connecting to Docker with default HTTP settings instead."); + Docker::connect_with_http_defaults().map_err(Into::into) + } else { + Docker::connect_with_local(&host, DEFAULT_TIMEOUT, API_DEFAULT_VERSION) + .map_err(Into::into) + } } Some(scheme) => Err(format!("Unknown scheme: {}", scheme).into()), } @@ -1129,8 +1149,7 @@ mod integration_tests { }, image::{CreateImageOptions, ListImagesOptions}, }; - use futures::stream::TryStreamExt; - use tokio::sync::mpsc; + use futures::{channel::mpsc, stream::TryStreamExt, FutureExt}; /// None if docker is not present on the system fn source_with<'a, L: Into>>( @@ -1351,12 +1370,8 @@ mod integration_tests { id } - async fn is_empty(mut rx: mpsc::Receiver) -> Result { - match rx.try_recv() { - Ok(_) => Ok(false), - Err(mpsc::error::TryRecvError::Empty) => Ok(true), - Err(mpsc::error::TryRecvError::Closed) => Err(()), - } + fn is_empty(mut rx: mpsc::Receiver) -> bool { + rx.next().now_or_never().is_none() } #[tokio::test] @@ -1459,7 +1474,7 @@ mod integration_tests { let id0 = container_log_n(1, &excluded0, None, "will not be read", &docker).await; let id1 = container_log_n(1, &included0, None, will_be_read, &docker).await; let id2 = container_log_n(1, &included1, None, will_be_read, &docker).await; - tokio::time::delay_for(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_secs(1)).await; let events = collect_ready(out).await; container_remove(&id0, &docker).await; container_remove(&id1, &docker).await; @@ -1575,7 +1590,7 @@ mod integration_tests { let id = container_log_n(1, name, None, message, &docker).await; container_remove(&id, &docker).await; - assert!(is_empty(exclude_out).await.unwrap()); + assert!(is_empty(exclude_out)); } #[tokio::test] @@ -1604,7 +1619,7 @@ mod integration_tests { let _ = container_kill(&id, &docker).await; container_remove(&id, &docker).await; - assert!(is_empty(exclude_out).await.unwrap()); + assert!(is_empty(exclude_out)); } #[tokio::test] diff --git a/src/sources/file.rs b/src/sources/file.rs index 46d82aff647cd..6d4cae60514eb 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -431,9 +431,8 @@ mod tests { future::Future, io::{Seek, Write}, }; - use tempfile::tempdir; - use tokio::time::{delay_for, timeout, Duration}; + use tokio::time::{sleep, timeout, Duration}; #[test] fn generate_config() { @@ -454,8 +453,8 @@ mod tests { async fn wait_with_timeout(future: F) -> R where - F: Future + Send + 'static, - R: Send + 'static, + F: Future + Send, + R: Send, { timeout(Duration::from_secs(5), future) .await @@ -465,7 +464,7 @@ mod tests { } async fn sleep_500_millis() { - delay_for(Duration::from_millis(500)).await; + sleep(Duration::from_millis(500)).await; } #[test] @@ -824,7 +823,7 @@ mod tests { { let (trigger_shutdown, shutdown, shutdown_done) = ShutdownSignal::new_wired(); - let (tx, rx) = Pipeline::new_test(); + let (tx, mut rx) = Pipeline::new_test(); let dir = tempdir().unwrap(); let config = file::FileConfig { include: vec![dir.path().join("*")], @@ -846,7 +845,7 @@ mod tests { drop(trigger_shutdown); shutdown_done.await; - let received = wait_with_timeout(rx.into_future()).await.0.unwrap(); + let received = wait_with_timeout(rx.next()).await.unwrap(); assert_eq!( received.as_log()["file"].to_string_lossy(), path.to_str().unwrap() @@ -857,7 +856,7 @@ mod tests { { let (trigger_shutdown, shutdown, shutdown_done) = ShutdownSignal::new_wired(); - let (tx, rx) = Pipeline::new_test(); + let (tx, mut rx) = Pipeline::new_test(); let dir = tempdir().unwrap(); let config = file::FileConfig { include: vec![dir.path().join("*")], @@ -880,7 +879,7 @@ mod tests { drop(trigger_shutdown); shutdown_done.await; - let received = wait_with_timeout(rx.into_future()).await.0.unwrap(); + let received = wait_with_timeout(rx.next()).await.unwrap(); assert_eq!( received.as_log()["source"].to_string_lossy(), path.to_str().unwrap() @@ -891,7 +890,7 @@ mod tests { { let (trigger_shutdown, shutdown, shutdown_done) = ShutdownSignal::new_wired(); - let (tx, rx) = Pipeline::new_test(); + let (tx, mut rx) = Pipeline::new_test(); let dir = tempdir().unwrap(); let config = file::FileConfig { include: vec![dir.path().join("*")], @@ -914,7 +913,7 @@ mod tests { drop(trigger_shutdown); shutdown_done.await; - let received = wait_with_timeout(rx.into_future()).await.0.unwrap(); + let received = wait_with_timeout(rx.next()).await.unwrap(); assert_eq!( received.as_log().keys().collect::>(), vec![ @@ -1711,7 +1710,7 @@ mod tests { for _ in 0..10 { // Wait for remove grace period to end. - delay_for(Duration::from_secs(remove_after_secs + 1)).await; + sleep(Duration::from_secs(remove_after_secs + 1)).await; if File::open(&path).is_err() { break; diff --git a/src/sources/generator.rs b/src/sources/generator.rs index 2ea3d82eeffd8..cfc77be76facb 100644 --- a/src/sources/generator.rs +++ b/src/sources/generator.rs @@ -6,7 +6,7 @@ use crate::{ Pipeline, }; use fakedata::logs::*; -use futures::{stream::StreamExt, SinkExt}; +use futures::SinkExt; use rand::seq::SliceRandom; use serde::{Deserialize, Serialize}; use snafu::Snafu; @@ -118,7 +118,7 @@ impl GeneratorConfig { } if let Some(interval) = &mut interval { - interval.next().await; + interval.tick().await; } let event = self.format.generate_event(n); @@ -167,8 +167,8 @@ impl SourceConfig for GeneratorConfig { mod tests { use super::*; use crate::{config::log_schema, shutdown::ShutdownSignal, Pipeline}; + use futures::{channel::mpsc, poll, StreamExt}; use std::time::{Duration, Instant}; - use tokio::sync::mpsc; #[test] fn generate_config() { @@ -213,13 +213,16 @@ mod tests { let lines = &["one", "two", "three", "four"]; for _ in 0..5 { - let event = rx.try_recv().unwrap(); + let event = match poll!(rx.next()) { + Poll::Ready(event) => event.unwrap(), + _ => unreachable!(), + }; let log = event.as_log(); let message = log[&message_key].to_string_lossy(); assert!(lines.contains(&&*message)); } - assert_eq!(rx.try_recv(), Err(mpsc::error::TryRecvError::Closed)); + assert_eq!(poll!(rx.next()), Poll::Ready(None)); } #[tokio::test] @@ -232,9 +235,9 @@ mod tests { .await; for _ in 0..5 { - assert!(matches!(rx.try_recv(), Ok(_))); + assert!(poll!(rx.next()).is_ready()); } - assert_eq!(rx.try_recv(), Err(mpsc::error::TryRecvError::Closed)); + assert_eq!(poll!(rx.next()), Poll::Ready(None)); } #[tokio::test] @@ -249,13 +252,16 @@ mod tests { .await; for n in 0..5 { - let event = rx.try_recv().unwrap(); + let event = match poll!(rx.next()) { + Poll::Ready(event) => event.unwrap(), + _ => unreachable!(), + }; let log = event.as_log(); let message = log[&message_key].to_string_lossy(); assert!(message.starts_with(&n.to_string())); } - assert_eq!(rx.try_recv(), Err(mpsc::error::TryRecvError::Closed)); + assert_eq!(poll!(rx.next()), Poll::Ready(None)); } #[tokio::test] @@ -270,9 +276,9 @@ mod tests { .await; for _ in 0..3 { - assert!(matches!(rx.try_recv(), Ok(_))); + assert!(poll!(rx.next()).is_ready()); } - assert_eq!(rx.try_recv(), Err(mpsc::error::TryRecvError::Closed)); + assert_eq!(poll!(rx.next()), Poll::Ready(None)); let duration = start.elapsed(); assert!(duration >= Duration::from_secs(2)); @@ -287,9 +293,9 @@ mod tests { .await; for _ in 0..5 { - assert!(matches!(rx.try_recv(), Ok(_))); + assert!(poll!(rx.next()).is_ready()); } - assert_eq!(rx.try_recv(), Err(mpsc::error::TryRecvError::Closed)); + assert_eq!(poll!(rx.next()), Poll::Ready(None)); } #[tokio::test] @@ -301,9 +307,9 @@ mod tests { .await; for _ in 0..5 { - assert!(matches!(rx.try_recv(), Ok(_))); + assert!(poll!(rx.next()).is_ready()); } - assert_eq!(rx.try_recv(), Err(mpsc::error::TryRecvError::Closed)); + assert_eq!(poll!(rx.next()), Poll::Ready(None)); } #[tokio::test] @@ -315,9 +321,9 @@ mod tests { .await; for _ in 0..5 { - assert!(matches!(rx.try_recv(), Ok(_))); + assert!(poll!(rx.next()).is_ready()); } - assert_eq!(rx.try_recv(), Err(mpsc::error::TryRecvError::Closed)); + assert_eq!(poll!(rx.next()), Poll::Ready(None)); } #[tokio::test] @@ -329,9 +335,9 @@ mod tests { .await; for _ in 0..5 { - assert!(matches!(rx.try_recv(), Ok(_))); + assert!(poll!(rx.next()).is_ready()); } - assert_eq!(rx.try_recv(), Err(mpsc::error::TryRecvError::Closed)); + assert_eq!(poll!(rx.next()), Poll::Ready(None)); } #[tokio::test] @@ -344,11 +350,14 @@ mod tests { .await; for _ in 0..5 { - let event = rx.try_recv().unwrap(); + let event = match poll!(rx.next()) { + Poll::Ready(event) => event.unwrap(), + _ => unreachable!(), + }; let log = event.as_log(); let message = log[&message_key].to_string_lossy(); assert!(serde_json::from_str::(&message).is_ok()); } - assert_eq!(rx.try_recv(), Err(mpsc::error::TryRecvError::Closed)); + assert_eq!(poll!(rx.next()), Poll::Ready(None)); } } diff --git a/src/sources/heroku_logs.rs b/src/sources/heroku_logs.rs index f45e4eb077585..36aee13fe2eaa 100644 --- a/src/sources/heroku_logs.rs +++ b/src/sources/heroku_logs.rs @@ -10,7 +10,7 @@ use crate::{ tls::TlsConfig, Pipeline, }; -use bytes::{buf::BufExt, Bytes}; +use bytes::{Buf, Bytes}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::{ @@ -250,9 +250,9 @@ mod tests { Pipeline, }; use chrono::{DateTime, Utc}; + use futures::channel::mpsc; use pretty_assertions::assert_eq; use std::net::SocketAddr; - use tokio::sync::mpsc; #[test] fn generate_config() { diff --git a/src/sources/host_metrics.rs b/src/sources/host_metrics.rs index 2f043c8495311..74e53a5ca1595 100644 --- a/src/sources/host_metrics.rs +++ b/src/sources/host_metrics.rs @@ -36,6 +36,7 @@ use std::collections::BTreeMap; use std::fmt; use std::path::Path; use tokio::time; +use tokio_stream::wrappers::IntervalStream; #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] @@ -145,7 +146,7 @@ impl HostMetricsConfig { out.sink_map_err(|error| error!(message = "Error sending host metrics.", %error)); let duration = time::Duration::from_secs(self.scrape_interval_secs); - let mut interval = time::interval(duration).take_until(shutdown); + let mut interval = IntervalStream::new(time::interval(duration)).take_until(shutdown); while interval.next().await.is_some() { let metrics = self.capture_metrics().await; out.send_all(&mut stream::iter(metrics).map(Ok)).await?; diff --git a/src/sources/http.rs b/src/sources/http.rs index 725006e7b8dd8..2bbfab6e32dd3 100644 --- a/src/sources/http.rs +++ b/src/sources/http.rs @@ -168,7 +168,6 @@ fn add_headers( #[cfg(test)] mod tests { use super::{Encoding, SimpleHttpConfig}; - use crate::shutdown::ShutdownSignal; use crate::{ config::{log_schema, GlobalOptions, SourceConfig}, @@ -180,12 +179,12 @@ mod tests { write::{DeflateEncoder, GzEncoder}, Compression, }; + use futures::channel::mpsc; use http::HeaderMap; use pretty_assertions::assert_eq; use std::collections::BTreeMap; use std::io::Write; use std::net::SocketAddr; - use tokio::sync::mpsc; #[test] fn generate_config() { diff --git a/src/sources/internal_logs.rs b/src/sources/internal_logs.rs index 14b6d7a1f01ef..9ca99522a007b 100644 --- a/src/sources/internal_logs.rs +++ b/src/sources/internal_logs.rs @@ -5,7 +5,7 @@ use crate::{ }; use futures::{stream, SinkExt, StreamExt}; use serde::{Deserialize, Serialize}; -use tokio::sync::broadcast::RecvError; +use tokio::sync::broadcast::error::RecvError; #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -39,10 +39,10 @@ impl SourceConfig for InternalLogsConfig { } } -async fn run(out: Pipeline, shutdown: ShutdownSignal) -> Result<(), ()> { +async fn run(out: Pipeline, mut shutdown: ShutdownSignal) -> Result<(), ()> { let mut out = out.sink_map_err(|error| error!(message = "Error sending log.", %error)); let subscription = trace::subscribe(); - let mut subscriber = subscription.receiver.take_until(shutdown); + let mut rx = subscription.receiver; out.send_all(&mut stream::iter(subscription.buffer).map(Ok)) .await?; @@ -50,14 +50,19 @@ async fn run(out: Pipeline, shutdown: ShutdownSignal) -> Result<(), ()> { // Note: This loop, or anything called within it, MUST NOT generate // any logs that don't break the loop, as that could cause an // infinite loop since it receives all such logs. - - while let Some(receive) = subscriber.next().await { - match receive { - Ok(event) => out.send(event).await?, - Err(RecvError::Lagged(_)) => (), - Err(RecvError::Closed) => break, + loop { + tokio::select! { + receive = rx.recv() => { + match receive { + Ok(event) => out.send(event).await?, + Err(RecvError::Lagged(_)) => (), + Err(RecvError::Closed) => break, + } + } + _ = &mut shutdown => break, } } + Ok(()) } @@ -65,10 +70,8 @@ async fn run(out: Pipeline, shutdown: ShutdownSignal) -> Result<(), ()> { mod tests { use super::*; use crate::{config::GlobalOptions, test_util::collect_ready, trace, Event}; - use tokio::{ - sync::mpsc::Receiver, - time::{delay_for, Duration}, - }; + use futures::channel::mpsc; + use tokio::time::{sleep, Duration}; #[test] fn generates_config() { @@ -102,7 +105,7 @@ mod tests { check_events(logs, start); } - async fn start_source() -> Receiver { + async fn start_source() -> mpsc::Receiver { let (tx, rx) = Pipeline::new_test(); let source = InternalLogsConfig {} @@ -115,13 +118,13 @@ mod tests { .await .unwrap(); tokio::spawn(source); - delay_for(Duration::from_millis(1)).await; + sleep(Duration::from_millis(1)).await; trace::stop_buffering(); rx } - async fn collect_output(rx: Receiver) -> Vec { - delay_for(Duration::from_millis(1)).await; + async fn collect_output(rx: mpsc::Receiver) -> Vec { + sleep(Duration::from_millis(1)).await; collect_ready(rx).await } diff --git a/src/sources/internal_metrics.rs b/src/sources/internal_metrics.rs index f6093bfea5a69..4bc60bf343148 100644 --- a/src/sources/internal_metrics.rs +++ b/src/sources/internal_metrics.rs @@ -8,6 +8,7 @@ use crate::{ use futures::{stream, SinkExt, StreamExt}; use serde::{Deserialize, Serialize}; use tokio::time; +use tokio_stream::wrappers::IntervalStream; #[derive(Deserialize, Serialize, Debug, Clone, Derivative)] #[derivative(Default)] @@ -60,7 +61,7 @@ async fn run( let mut out = out.sink_map_err(|error| error!(message = "Error sending internal metrics.", %error)); - let mut interval = time::interval(interval).take_until(shutdown); + let mut interval = IntervalStream::new(time::interval(interval)).take_until(shutdown); while interval.next().await.is_some() { let metrics = capture_metrics(controller); out.send_all(&mut stream::iter(metrics).map(Ok)).await?; diff --git a/src/sources/journald.rs b/src/sources/journald.rs index 71266df5d6193..eb355f8b78631 100644 --- a/src/sources/journald.rs +++ b/src/sources/journald.rs @@ -30,9 +30,9 @@ use tokio_util::codec::FramedRead; use tokio::{ fs::{File, OpenOptions}, - io::{self, AsyncReadExt, AsyncWriteExt}, + io::{self, AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, process::Command, - time::delay_for, + time::sleep, }; use tracing_futures::Instrument; @@ -242,7 +242,7 @@ impl JournaldSource { // journalctl process should never stop, // so it is an error if we reach here. - delay_for(BACKOFF_DURATION).await; + sleep(BACKOFF_DURATION).await; } } @@ -373,7 +373,7 @@ fn start_journalctl( ) .boxed(); - let pid = Pid::from_raw(child.id() as i32); + let pid = Pid::from_raw(child.id().unwrap() as _); let stop = Box::new(move || { let _ = kill(pid, Signal::SIGTERM); }); @@ -586,7 +586,7 @@ mod tests { use tempfile::tempdir; use tokio::{ io, - time::{delay_for, timeout, Duration}, + time::{sleep, timeout, Duration}, }; const FAKE_JOURNAL: &str = r#"{"_SYSTEMD_UNIT":"sysinit.target","MESSAGE":"System Initialization","__CURSOR":"1","_SOURCE_REALTIME_TIMESTAMP":"1578529839140001","PRIORITY":"6"} @@ -680,7 +680,7 @@ mod tests { ); tokio::spawn(source); - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; drop(trigger); timeout(Duration::from_secs(1), rx.collect()).await.unwrap() diff --git a/src/sources/kafka.rs b/src/sources/kafka.rs index d38eae84702f3..9da58665368f0 100644 --- a/src/sources/kafka.rs +++ b/src/sources/kafka.rs @@ -131,9 +131,11 @@ fn kafka_source( let consumer = Arc::new(create_consumer(config)?); Ok(Box::pin(async move { + let shutdown = shutdown; + Arc::clone(&consumer) - .start() - .take_until(shutdown.clone()) + .stream() + .take_until(shutdown) .then(move |message| { let key_field = key_field.clone(); let topic_key = topic_key.clone(); @@ -278,14 +280,14 @@ mod test { } } - #[test] - fn kafka_source_create_ok() { + #[tokio::test] + async fn kafka_source_create_ok() { let config = make_config(); assert!(kafka_source(&config, ShutdownSignal::noop(), Pipeline::new_test().0).is_ok()); } - #[test] - fn kafka_source_create_incorrect_auto_offset_reset() { + #[tokio::test] + async fn kafka_source_create_incorrect_auto_offset_reset() { let config = KafkaSourceConfig { auto_offset_reset: "incorrect-auto-offset-reset".to_string(), ..make_config() diff --git a/src/sources/kubernetes_logs/util.rs b/src/sources/kubernetes_logs/util.rs index c4d34856bef69..ab5a992c5596f 100644 --- a/src/sources/kubernetes_logs/util.rs +++ b/src/sources/kubernetes_logs/util.rs @@ -58,7 +58,7 @@ pub async fn complete_with_deadline_on_signal( future: F, signal: S, deadline: Duration, -) -> Result<::Output, tokio::time::Elapsed> +) -> Result<::Output, tokio::time::error::Elapsed> where F: Future, S: Future, diff --git a/src/sources/mongodb_metrics/mod.rs b/src/sources/mongodb_metrics/mod.rs index 24d221d201fe4..58d8124a2e7a2 100644 --- a/src/sources/mongodb_metrics/mod.rs +++ b/src/sources/mongodb_metrics/mod.rs @@ -23,6 +23,7 @@ use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; use std::{collections::BTreeMap, time::Instant}; use tokio::time; +use tokio_stream::wrappers::IntervalStream; mod types; use types::{CommandBuildInfo, CommandIsMaster, CommandServerStatus, NodeType}; @@ -126,7 +127,7 @@ impl SourceConfig for MongoDbMetricsConfig { let duration = time::Duration::from_secs(self.scrape_interval_secs); Ok(Box::pin(async move { - let mut interval = time::interval(duration).take_until(shutdown); + let mut interval = IntervalStream::new(time::interval(duration)).take_until(shutdown); while interval.next().await.is_some() { let start = Instant::now(); let metrics = join_all(sources.iter().map(|mongodb| mongodb.collect())).await; diff --git a/src/sources/nginx_metrics/mod.rs b/src/sources/nginx_metrics/mod.rs index a9e90bddfb9dc..6c74a9b6c3b1c 100644 --- a/src/sources/nginx_metrics/mod.rs +++ b/src/sources/nginx_metrics/mod.rs @@ -19,6 +19,7 @@ use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; use std::{collections::BTreeMap, convert::TryFrom, time::Instant}; use tokio::time; +use tokio_stream::wrappers::IntervalStream; pub mod parser; use parser::NginxStubStatus; @@ -106,7 +107,7 @@ impl SourceConfig for NginxMetricsConfig { let duration = time::Duration::from_secs(self.scrape_interval_secs); Ok(Box::pin(async move { - let mut interval = time::interval(duration).take_until(shutdown); + let mut interval = IntervalStream::new(time::interval(duration)).take_until(shutdown); while interval.next().await.is_some() { let start = Instant::now(); let metrics = join_all(sources.iter().map(|nginx| nginx.collect())).await; diff --git a/src/sources/postgresql_metrics.rs b/src/sources/postgresql_metrics.rs index e3f60d06c9efb..fe250112f9b3e 100644 --- a/src/sources/postgresql_metrics.rs +++ b/src/sources/postgresql_metrics.rs @@ -29,6 +29,7 @@ use tokio_postgres::{ types::FromSql, Client, Config, Error as PgError, NoTls, Row, }; +use tokio_stream::wrappers::IntervalStream; macro_rules! tags { ($tags:expr) => { $tags.clone() }; @@ -156,7 +157,7 @@ impl SourceConfig for PostgresqlMetricsConfig { let duration = time::Duration::from_secs(self.scrape_interval_secs); Ok(Box::pin(async move { - let mut interval = time::interval(duration).take_until(shutdown); + let mut interval = IntervalStream::new(time::interval(duration)).take_until(shutdown); while interval.next().await.is_some() { let start = Instant::now(); let metrics = join_all(sources.iter_mut().map(|source| source.collect())).await; diff --git a/src/sources/prometheus/remote_write.rs b/src/sources/prometheus/remote_write.rs index 5c40b523fb680..bf138262f854b 100644 --- a/src/sources/prometheus/remote_write.rs +++ b/src/sources/prometheus/remote_write.rs @@ -255,7 +255,7 @@ mod integration_tests { tokio::spawn(source); - tokio::time::delay_for(Duration::from_secs(2)).await; + tokio::time::sleep(Duration::from_secs(2)).await; let events = test_util::collect_ready(rx).await; assert!(!events.is_empty()); diff --git a/src/sources/prometheus/scrape.rs b/src/sources/prometheus/scrape.rs index abe47f72b9882..12fba1fb77038 100644 --- a/src/sources/prometheus/scrape.rs +++ b/src/sources/prometheus/scrape.rs @@ -20,6 +20,7 @@ use std::{ future::ready, time::{Duration, Instant}, }; +use tokio_stream::wrappers::IntervalStream; #[derive(Debug, Snafu)] enum ConfigError { @@ -156,7 +157,7 @@ fn prometheus( ) -> sources::Source { let out = out.sink_map_err(|error| error!(message = "Error sending metric.", %error)); - Box::pin(tokio::time::interval(Duration::from_secs(interval)) + Box::pin(IntervalStream::new(tokio::time::interval(Duration::from_secs(interval))) .take_until(shutdown) .map(move |_| stream::iter(urls.clone())) .flatten() @@ -261,7 +262,7 @@ mod test { {Body, Client, Response, Server}, }; use pretty_assertions::assert_eq; - use tokio::time::{delay_for, Duration}; + use tokio::time::{sleep, Duration}; #[test] fn genreate_config() { @@ -338,7 +339,7 @@ mod test { ); let (topology, _crash) = start_topology(config.build().unwrap(), false).await; - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let response = Client::new() .get(format!("http://{}/metrics", out_addr).parse().unwrap()) @@ -416,7 +417,7 @@ mod integration_tests { .unwrap(); tokio::spawn(source); - tokio::time::delay_for(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_secs(1)).await; let events = test_util::collect_ready(rx).await; assert!(!events.is_empty()); diff --git a/src/sources/socket/mod.rs b/src/sources/socket/mod.rs index cf63043398794..254c3c691ab93 100644 --- a/src/sources/socket/mod.rs +++ b/src/sources/socket/mod.rs @@ -111,7 +111,6 @@ impl SourceConfig for SocketConfig { config.address(), config.max_length(), host_key, - #[cfg(unix)] config.receive_buffer_bytes(), shutdown, out, @@ -189,7 +188,8 @@ mod test { }, thread, }; - + #[cfg(unix)] + use tokio::io::AsyncWriteExt; use tokio::{ task::JoinHandle, time::{Duration, Instant}, @@ -233,7 +233,7 @@ mod test { .await .unwrap(); - let event = rx.recv().await.unwrap(); + let event = rx.next().await.unwrap(); assert_eq!(event.as_log()[log_schema().host_key()], "127.0.0.1".into()); } @@ -258,7 +258,7 @@ mod test { .await .unwrap(); - let event = rx.recv().await.unwrap(); + let event = rx.next().await.unwrap(); assert_eq!( event.as_log()[log_schema().source_type_key()], "socket".into() @@ -422,7 +422,7 @@ mod test { .await .unwrap(); - let event = rx.recv().await.unwrap(); + let event = rx.next().await.unwrap(); assert_eq!(event.as_log()[log_schema().message_key()], "test".into()); // Now signal to the Source to shut down. @@ -556,7 +556,7 @@ mod test { let source_handle = tokio::spawn(server); // Wait for UDP to start listening - tokio::time::delay_for(tokio::time::Duration::from_millis(100)).await; + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; (address, source_handle) } @@ -820,7 +820,7 @@ mod test { ////////////// UNIX DATAGRAM TESTS ////////////// #[cfg(unix)] async fn send_lines_unix_datagram(path: PathBuf, lines: &[&str]) { - let mut socket = UnixDatagram::unbound().unwrap(); + let socket = UnixDatagram::unbound().unwrap(); socket.connect(path).unwrap(); for line in lines { @@ -864,8 +864,8 @@ mod test { let lines = lines.collect::>(); sink.send_all(&mut stream::iter(lines)).await.unwrap(); - let socket = sink.into_inner(); - socket.shutdown(std::net::Shutdown::Both).unwrap(); + let mut socket = sink.into_inner(); + socket.shutdown().await.unwrap(); } #[cfg(unix)] diff --git a/src/sources/socket/udp.rs b/src/sources/socket/udp.rs index bcdf3f13681c7..c74094362fd61 100644 --- a/src/sources/socket/udp.rs +++ b/src/sources/socket/udp.rs @@ -1,4 +1,3 @@ -#[cfg(unix)] use crate::udp; use crate::{ event::Event, @@ -27,7 +26,6 @@ pub struct UdpConfig { max_length: usize, #[get = "pub"] host_key: Option, - #[cfg(unix)] #[get_copy = "pub"] receive_buffer_bytes: Option, } @@ -42,7 +40,6 @@ impl UdpConfig { address, max_length: default_max_length(), host_key: None, - #[cfg(unix)] receive_buffer_bytes: None, } } @@ -52,23 +49,23 @@ pub fn udp( address: SocketAddr, max_length: usize, host_key: String, - #[cfg(unix)] receive_buffer_bytes: Option, + receive_buffer_bytes: Option, mut shutdown: ShutdownSignal, out: Pipeline, ) -> Source { let mut out = out.sink_map_err(|error| error!(message = "Error sending event.", %error)); Box::pin(async move { - let mut socket = UdpSocket::bind(&address) + let socket = UdpSocket::bind(&address) .await .expect("Failed to bind to udp listener socket"); - #[cfg(unix)] if let Some(receive_buffer_bytes) = receive_buffer_bytes { - udp::set_receive_buffer_size(&socket, receive_buffer_bytes); + if let Err(error) = udp::set_receive_buffer_size(&socket, receive_buffer_bytes) { + warn!(message = "Failed configuring receive buffer size on UDP socket.", %error); + } } - #[cfg(unix)] let max_length = if let Some(receive_buffer_bytes) = receive_buffer_bytes { std::cmp::min(max_length, receive_buffer_bytes) } else { diff --git a/src/sources/splunk_hec.rs b/src/sources/splunk_hec.rs index a6c0694cd4c60..bd3f39651cc51 100644 --- a/src/sources/splunk_hec.rs +++ b/src/sources/splunk_hec.rs @@ -9,7 +9,7 @@ use crate::{ tls::{MaybeTlsSettings, TlsConfig}, Pipeline, }; -use bytes::{buf::BufExt, Bytes}; +use bytes::{Buf, Bytes}; use chrono::{DateTime, TimeZone, Utc}; use flate2::read::GzDecoder; use futures::{FutureExt, SinkExt, StreamExt, TryFutureExt}; @@ -116,14 +116,13 @@ impl SourceConfig for SplunkConfig { let listener = tls.bind(&self.address).await?; Ok(Box::pin(async move { - let _ = warp::serve(services) + warp::serve(services) .serve_incoming_with_graceful_shutdown( listener.accept_stream(), - shutdown.clone().map(|_| ()), + shutdown.map(|_| ()), ) .await; - // We need to drop the last copy of ShutdownSignalToken only after server has shut down. - drop(shutdown); + Ok(()) })) } @@ -632,12 +631,6 @@ pub(crate) enum ApiError { BadRequest, } -impl From for Rejection { - fn from(error: ApiError) -> Self { - warp::reject::custom(error) - } -} - impl warp::reject::Reject for ApiError {} /// Cached bodies for common responses @@ -762,9 +755,8 @@ mod tests { Pipeline, }; use chrono::{TimeZone, Utc}; - use futures::{stream, StreamExt}; + use futures::{channel::mpsc, stream, StreamExt}; use std::{future::ready, net::SocketAddr}; - use tokio::sync::mpsc; #[test] fn generate_config() { diff --git a/src/sources/statsd/mod.rs b/src/sources/statsd/mod.rs index 95d79fc2b7eba..87f3f318d644c 100644 --- a/src/sources/statsd/mod.rs +++ b/src/sources/statsd/mod.rs @@ -1,4 +1,3 @@ -#[cfg(unix)] use crate::udp; use crate::{ config::{self, GenerateConfig, GlobalOptions, Resource, SourceConfig, SourceDescription}, @@ -37,7 +36,6 @@ enum StatsdConfig { #[derive(Deserialize, Serialize, Debug, Clone)] pub struct UdpConfig { address: SocketAddr, - #[cfg(unix)] receive_buffer_bytes: Option, } @@ -45,7 +43,6 @@ impl UdpConfig { pub fn from_address(address: SocketAddr) -> Self { Self { address, - #[cfg(unix)] receive_buffer_bytes: None, } } @@ -163,9 +160,10 @@ async fn statsd_udp( .map_err(|error| emit!(StatsdSocketError::bind(error))) .await?; - #[cfg(unix)] if let Some(receive_buffer_bytes) = config.receive_buffer_bytes { - udp::set_receive_buffer_size(&socket, receive_buffer_bytes); + if let Err(error) = udp::set_receive_buffer_size(&socket, receive_buffer_bytes) { + warn!(message = "Failed configuring receive buffer size on UDP socket.", %error); + } } info!( @@ -224,10 +222,10 @@ mod test { sinks::prometheus::exporter::PrometheusExporterConfig, test_util::{next_addr, start_topology}, }; + use futures::channel::mpsc; use hyper::body::to_bytes as body_to_bytes; use tokio::io::AsyncWriteExt; - use tokio::sync::mpsc; - use tokio::time::{delay_for, Duration}; + use tokio::time::{sleep, Duration}; #[test] fn generate_config() { @@ -251,9 +249,9 @@ mod test { let (sender, mut receiver) = mpsc::channel(200); tokio::spawn(async move { let bind_addr = next_addr(); - let mut socket = UdpSocket::bind(bind_addr).await.unwrap(); + let socket = UdpSocket::bind(bind_addr).await.unwrap(); socket.connect(in_addr).await.unwrap(); - while let Some(bytes) = receiver.recv().await { + while let Some(bytes) = receiver.next().await { socket.send(bytes).await.unwrap(); } }); @@ -266,7 +264,7 @@ mod test { let config = StatsdConfig::Tcp(TcpConfig::from_address(in_addr.into())); let (sender, mut receiver) = mpsc::channel(200); tokio::spawn(async move { - while let Some(bytes) = receiver.recv().await { + while let Some(bytes) = receiver.next().await { tokio::net::TcpStream::connect(in_addr) .await .unwrap() @@ -287,7 +285,7 @@ mod test { }); let (sender, mut receiver) = mpsc::channel(200); tokio::spawn(async move { - while let Some(bytes) = receiver.recv().await { + while let Some(bytes) = receiver.next().await { tokio::net::UnixStream::connect(&in_path) .await .unwrap() @@ -325,18 +323,18 @@ mod test { let (topology, _crash) = start_topology(config.build().unwrap(), false).await; // Give some time for the topology to start - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; for _ in 0..100 { sender.send( b"foo:1|c|#a,b:b\nbar:42|g\nfoo:1|c|#a,b:c\nglork:3|h|@0.1\nmilliglork:3000|ms|@0.1\nset:0|s\nset:1|s\n" ).await.unwrap(); // Space things out slightly to try to avoid dropped packets - delay_for(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; } // Give packets some time to flow through - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; let client = hyper::Client::new(); let response = client @@ -384,7 +382,7 @@ mod test { // Flush test { // Wait for flush to happen - delay_for(Duration::from_millis(2000)).await; + sleep(Duration::from_millis(2000)).await; let response = client .get(format!("http://{}/metrics", out_addr).parse().unwrap()) @@ -405,7 +403,7 @@ mod test { sender.send(b"set:0|s\nset:1|s\n").await.unwrap(); // Give packets some time to flow through - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; let response = client .get(format!("http://{}/metrics", out_addr).parse().unwrap()) diff --git a/src/sources/stdin.rs b/src/sources/stdin.rs index 1f130f92f92ec..a930970e84d04 100644 --- a/src/sources/stdin.rs +++ b/src/sources/stdin.rs @@ -6,10 +6,9 @@ use crate::{ Pipeline, }; use bytes::Bytes; -use futures::{executor, FutureExt, SinkExt, StreamExt, TryStreamExt}; +use futures::{channel::mpsc, executor, FutureExt, SinkExt, StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use std::{io, thread}; -use tokio::sync::mpsc::channel; #[derive(Deserialize, Serialize, Debug, Clone)] #[serde(deny_unknown_fields, default)] @@ -78,7 +77,7 @@ where .unwrap_or_else(|| log_schema().host_key().to_string()); let hostname = crate::get_hostname().ok(); - let (mut sender, receiver) = channel(1024); + let (mut sender, receiver) = mpsc::channel(1024); // Start the background thread thread::spawn(move || { @@ -135,7 +134,6 @@ mod tests { use super::*; use crate::{test_util::trace_init, Pipeline}; use std::io::Cursor; - use tokio::sync::mpsc; #[test] fn generate_config() { @@ -160,7 +158,7 @@ mod tests { async fn stdin_decodes_line() { trace_init(); - let (tx, mut rx) = Pipeline::new_test(); + let (tx, rx) = Pipeline::new_test(); let config = StdinConfig::default(); let buf = Cursor::new("hello world\nhello world again"); @@ -169,23 +167,21 @@ mod tests { .await .unwrap(); - let event = rx.try_recv(); + let mut stream = rx; - assert!(event.is_ok()); + let event = stream.next().await; assert_eq!( - Ok("hello world".into()), + Some("hello world".into()), event.map(|event| event.as_log()[log_schema().message_key()].to_string_lossy()) ); - let event = rx.try_recv(); - assert!(event.is_ok()); + let event = stream.next().await; assert_eq!( - Ok("hello world again".into()), + Some("hello world again".into()), event.map(|event| event.as_log()[log_schema().message_key()].to_string_lossy()) ); - let event = rx.try_recv(); - assert!(event.is_err()); - assert_eq!(Err(mpsc::error::TryRecvError::Closed), event); + let event = stream.next().await; + assert!(event.is_none()); } } diff --git a/src/sources/syslog.rs b/src/sources/syslog.rs index 99b763edf451b..6268444dc5d1f 100644 --- a/src/sources/syslog.rs +++ b/src/sources/syslog.rs @@ -1,7 +1,6 @@ use super::util::{SocketListenAddr, TcpSource}; #[cfg(unix)] use crate::sources::util::build_unix_stream_source; -#[cfg(unix)] use crate::udp; use crate::{ config::{ @@ -54,7 +53,6 @@ pub enum Mode { }, Udp { address: SocketAddr, - #[cfg(unix)] receive_buffer_bytes: Option, }, #[cfg(unix)] @@ -133,7 +131,6 @@ impl SourceConfig for SyslogConfig { out, ) } - #[cfg(unix)] Mode::Udp { address, receive_buffer_bytes, @@ -145,8 +142,6 @@ impl SourceConfig for SyslogConfig { shutdown, out, )), - #[cfg(not(unix))] - Mode::Udp { address } => Ok(udp(address, self.max_length, host_key, shutdown, out)), #[cfg(unix)] Mode::Unix { path } => Ok(build_unix_stream_source( path, @@ -408,7 +403,7 @@ pub fn udp( addr: SocketAddr, _max_length: usize, host_key: String, - #[cfg(unix)] receive_buffer_bytes: Option, + receive_buffer_bytes: Option, shutdown: ShutdownSignal, out: Pipeline, ) -> super::Source { @@ -419,9 +414,10 @@ pub fn udp( .await .expect("Failed to bind to UDP listener socket"); - #[cfg(unix)] if let Some(receive_buffer_bytes) = receive_buffer_bytes { - udp::set_receive_buffer_size(&socket, receive_buffer_bytes); + if let Err(error) = udp::set_receive_buffer_size(&socket, receive_buffer_bytes) { + warn!(message = "Failed configuring receive buffer size on UDP socket.", %error); + } } info!( @@ -654,7 +650,6 @@ mod test { assert!(config.mode.is_udp()); } - #[cfg(unix)] #[test] fn config_udp_with_receive_buffer_size() { let config: SyslogConfig = toml::from_str( diff --git a/src/sources/util/http.rs b/src/sources/util/http.rs index 3055193fbffab..b42bf515a4b03 100644 --- a/src/sources/util/http.rs +++ b/src/sources/util/http.rs @@ -6,7 +6,7 @@ use crate::{ Pipeline, }; use async_trait::async_trait; -use bytes::{buf::BufExt, Bytes}; +use bytes::{Buf, Bytes}; use flate2::read::{DeflateDecoder, GzDecoder}; use futures::{FutureExt, SinkExt, StreamExt, TryFutureExt}; use headers::{Authorization, HeaderMapExt}; @@ -290,14 +290,12 @@ pub trait HttpSource: Clone + Send + Sync + 'static { info!(message = "Building HTTP server.", address = %address); let listener = tls.bind(&address).await.unwrap(); - let _ = warp::serve(routes) + warp::serve(routes) .serve_incoming_with_graceful_shutdown( listener.accept_stream(), - shutdown.clone().map(|_| ()), + shutdown.map(|_| ()), ) .await; - // We need to drop the last copy of ShutdownSignalToken only after server has shut down. - drop(shutdown); Ok(()) })) } diff --git a/src/sources/util/tcp.rs b/src/sources/util/tcp.rs index d703b55ca892e..b82e295054358 100644 --- a/src/sources/util/tcp.rs +++ b/src/sources/util/tcp.rs @@ -10,10 +10,11 @@ use bytes::Bytes; use futures::{future::BoxFuture, stream, FutureExt, Sink, SinkExt, StreamExt, TryFutureExt}; use listenfd::ListenFd; use serde::{de, Deserialize, Deserializer, Serialize}; +use socket2::SockRef; use std::{fmt, future::ready, io, mem::drop, net::SocketAddr, task::Poll, time::Duration}; use tokio::{ net::{TcpListener, TcpStream}, - time::delay_for, + time::sleep, }; use tokio_util::codec::{Decoder, FramedRead, LinesCodecError}; use tracing_futures::Instrument; @@ -86,7 +87,7 @@ where shutdown_timeout_secs: u64, tls: MaybeTlsSettings, receive_buffer_bytes: Option, - shutdown: ShutdownSignal, + shutdown_signal: ShutdownSignal, out: Pipeline, ) -> crate::Result { let out = out.sink_map_err(|error| error!(message = "Error sending event.", %error)); @@ -107,20 +108,21 @@ where .unwrap_or(addr) ); - let tripwire = shutdown.clone(); + let tripwire = shutdown_signal.clone(); let tripwire = async move { let _ = tripwire.await; - delay_for(Duration::from_secs(shutdown_timeout_secs)).await; + sleep(Duration::from_secs(shutdown_timeout_secs)).await; } .shared(); let connection_gauge = OpenGauge::new(); + let shutdown_clone = shutdown_signal.clone(); listener .accept_stream() - .take_until(shutdown.clone()) - .for_each(|connection| { - let shutdown = shutdown.clone(); + .take_until(shutdown_clone) + .for_each(move |connection| { + let shutdown_signal = shutdown_signal.clone(); let tripwire = tripwire.clone(); let source = self.clone(); let out = out.clone(); @@ -159,7 +161,7 @@ where connection_gauge.open(|count| emit!(ConnectionOpen { count })); let fut = handle_stream( - shutdown, + shutdown_signal, socket, keepalive, receive_buffer_bytes, @@ -182,7 +184,7 @@ where } async fn handle_stream( - mut shutdown: ShutdownSignal, + mut shutdown_signal: ShutdownSignal, mut socket: MaybeTlsIncomingStream, keepalive: Option, receive_buffer_bytes: Option, @@ -201,7 +203,7 @@ async fn handle_stream( return; } }, - _ = &mut shutdown => { + _ = &mut shutdown_signal => { return; } }; @@ -218,18 +220,19 @@ async fn handle_stream( } } - let mut _token = None; - let mut shutdown = Some(shutdown); + let mut shutdown_token = None; let mut reader = FramedRead::new(socket, source.decoder()); - stream::poll_fn(move |cx| { - if let Some(fut) = shutdown.as_mut() { - match fut.poll_unpin(cx) { + + stream::poll_fn(|cx| { + if shutdown_token.is_none() { + match shutdown_signal.poll_unpin(cx) { Poll::Ready(token) => { debug!("Start graceful shutdown."); // Close our write part of TCP socket to signal the other side // that it should stop writing and close the channel. - let socket: Option<&TcpStream> = reader.get_ref().get_ref(); - if let Some(socket) = socket { + let socket = reader.get_ref().get_ref(); + if let Some(stream) = socket { + let socket = SockRef::from(stream); if let Err(error) = socket.shutdown(std::net::Shutdown::Write) { warn!(message = "Failed in signalling to the other side to close the TCP channel.", %error); } @@ -239,8 +242,7 @@ async fn handle_stream( return Poll::Ready(None); } - _token = Some(token); - shutdown = None; + shutdown_token = Some(token); } Poll::Pending => {} } diff --git a/src/sources/util/unix_datagram.rs b/src/sources/util/unix_datagram.rs index 36d01060f7732..f4784a675e066 100644 --- a/src/sources/util/unix_datagram.rs +++ b/src/sources/util/unix_datagram.rs @@ -33,8 +33,7 @@ where let mut out = out.sink_map_err(|error| error!(message = "Error sending line.", %error)); Box::pin(async move { - let mut socket = - UnixDatagram::bind(&listen_path).expect("Failed to bind to datagram socket"); + let socket = UnixDatagram::bind(&listen_path).expect("Failed to bind to datagram socket"); info!(message = "Listening.", path = ?listen_path, r#type = "unix_datagram"); let mut buf = BytesMut::with_capacity(max_length); diff --git a/src/sources/util/unix_stream.rs b/src/sources/util/unix_stream.rs index fc9f4af6addb5..dc7fac79085ad 100644 --- a/src/sources/util/unix_stream.rs +++ b/src/sources/util/unix_stream.rs @@ -10,7 +10,9 @@ use crate::{ use bytes::Bytes; use futures::{FutureExt, SinkExt, StreamExt}; use std::{future::ready, path::PathBuf}; +use tokio::io::AsyncWriteExt; use tokio::net::{UnixListener, UnixStream}; +use tokio_stream::wrappers::UnixListenerStream; use tokio_util::codec::{Decoder, FramedRead}; use tracing::field; use tracing_futures::Instrument; @@ -34,12 +36,12 @@ where let out = out.sink_map_err(|error| error!(message = "Error sending line.", %error)); Box::pin(async move { - let mut listener = - UnixListener::bind(&listen_path).expect("Failed to bind to listener socket"); + let listener = UnixListener::bind(&listen_path).expect("Failed to bind to listener socket"); info!(message = "Listening.", path = ?listen_path, r#type = "unix"); let connection_open = OpenGauge::new(); - let mut stream = listener.incoming().take_until(shutdown.clone()); + let stream = UnixListenerStream::new(listener).take_until(shutdown.clone()); + tokio::pin!(stream); while let Some(socket) = stream.next().await { let socket = match socket { Err(error) => { @@ -90,8 +92,10 @@ where let _ = out.send_all(&mut stream).await; info!("Finished sending."); - let socket: &UnixStream = stream.get_ref().get_ref().get_ref(); - let _ = socket.shutdown(std::net::Shutdown::Both); + let socket: &mut UnixStream = stream.get_mut().get_mut().get_mut(); + if let Err(error) = socket.shutdown().await { + error!(message = "Failed shutting down socket.", %error); + } } .instrument(span), ); diff --git a/src/sources/vector.rs b/src/sources/vector.rs index f2d01d2fd2d80..8e9c77e316ab1 100644 --- a/src/sources/vector.rs +++ b/src/sources/vector.rs @@ -134,14 +134,11 @@ mod test { Event, Pipeline, }; use futures::stream; - use std::{ - net::{Shutdown, SocketAddr}, - thread, - }; + use std::net::SocketAddr; use tokio::{ io::AsyncWriteExt, net::TcpStream, - time::{delay_for, Duration}, + time::{sleep, Duration}, }; #[cfg(not(target_os = "windows"))] @@ -194,7 +191,7 @@ mod test { sink.run(stream::iter(events.clone())).await.unwrap(); - delay_for(Duration::from_millis(50)).await; + sleep(Duration::from_millis(50)).await; let output = collect_ready(rx).await; assert_eq!(events, output); @@ -258,8 +255,8 @@ mod test { let mut stream = TcpStream::connect(&addr).await.unwrap(); stream.write(b"hello world \n").await.unwrap(); - thread::sleep(Duration::from_secs(2)); - stream.shutdown(Shutdown::Both).unwrap(); + tokio::time::sleep(Duration::from_secs(2)).await; + stream.shutdown().await.unwrap(); drop(trigger_shutdown); shutdown_down.await; @@ -298,13 +295,13 @@ mod test { let mut sink = FramedWrite::new(stream, encoder); sink.send(out.into()).await.unwrap(); - let stream = sink.into_inner(); - thread::sleep(Duration::from_secs(2)); - stream.shutdown(Shutdown::Both).unwrap(); + let mut stream = sink.into_inner(); + tokio::time::sleep(Duration::from_secs(2)).await; + stream.shutdown().await.unwrap(); drop(trigger_shutdown); shutdown_down.await; let output = collect_ready(rx).await; - assert_eq!(Event::from(event), output[0]); + assert_eq!(vec![Event::from(event)], output); } } diff --git a/src/tcp.rs b/src/tcp.rs index ec313b52efb05..098905d729a38 100644 --- a/src/tcp.rs +++ b/src/tcp.rs @@ -1,4 +1,6 @@ use serde::{Deserialize, Serialize}; +use socket2::SockRef; +use tokio::net::TcpStream; /// Configuration for keepalive probes in a TCP stream. /// @@ -9,3 +11,21 @@ use serde::{Deserialize, Serialize}; pub struct TcpKeepaliveConfig { pub time_secs: Option, } + +// This function will be obsolete after tokio/mio internally use `socket2` and expose the methods to +// apply options to a socket. +pub fn set_keepalive(socket: &TcpStream, params: &socket2::TcpKeepalive) -> std::io::Result<()> { + SockRef::from(socket).set_tcp_keepalive(params) +} + +// This function will be obsolete after tokio/mio internally use `socket2` and expose the methods to +// apply options to a socket. +pub fn set_receive_buffer_size(socket: &TcpStream, size: usize) -> std::io::Result<()> { + SockRef::from(socket).set_recv_buffer_size(size) +} + +// This function will be obsolete after tokio/mio internally use `socket2` and expose the methods to +// apply options to a socket. +pub fn set_send_buffer_size(socket: &TcpStream, size: usize) -> std::io::Result<()> { + SockRef::from(socket).set_send_buffer_size(size) +} diff --git a/src/test_util/mod.rs b/src/test_util/mod.rs index 527a8164a3476..8c0ea84e9a7ee 100644 --- a/src/test_util/mod.rs +++ b/src/test_util/mod.rs @@ -18,7 +18,7 @@ use std::{ future::{ready, Future}, io::Read, iter, - net::{IpAddr, Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, path::{Path, PathBuf}, pin::Pin, sync::{ @@ -28,13 +28,16 @@ use std::{ task::{Context, Poll}, }; use tokio::{ - io::{AsyncRead, AsyncWrite, Result as IoResult}, + io::{AsyncRead, AsyncWrite, AsyncWriteExt, Result as IoResult}, net::{TcpListener, TcpStream}, runtime, - sync::{mpsc, oneshot}, + sync::oneshot, task::JoinHandle, - time::{delay_for, Duration, Instant}, + time::{sleep, Duration, Instant}, }; +use tokio_stream::wrappers::TcpListenerStream; +#[cfg(unix)] +use tokio_stream::wrappers::UnixListenerStream; use tokio_util::codec::{Encoder, FramedRead, FramedWrite, LinesCodec}; const WAIT_FOR_SECS: u64 = 5; // The default time to wait in `wait_for` @@ -127,7 +130,7 @@ pub async fn send_encodable + std::fmt::Debug>( sink.send_all(&mut lines).await.unwrap(); let stream = sink.get_mut(); - stream.shutdown(Shutdown::Both).unwrap(); + stream.shutdown().await.unwrap(); Ok(()) } @@ -147,16 +150,22 @@ pub async fn send_lines_tls( connector.set_verify(SslVerifyMode::NONE); } - let config = connector.build().configure().unwrap(); + let ssl = connector + .build() + .configure() + .unwrap() + .into_ssl(&host) + .unwrap(); - let stream = tokio_openssl::connect(config, &host, stream).await.unwrap(); + let mut stream = tokio_openssl::SslStream::new(ssl, stream).unwrap(); + Pin::new(&mut stream).connect().await.unwrap(); let mut sink = FramedWrite::new(stream, LinesCodec::new()); let mut lines = stream::iter(lines).map(Ok); sink.send_all(&mut lines).await.unwrap(); let stream = sink.get_mut().get_mut(); - stream.shutdown(Shutdown::Both).unwrap(); + stream.shutdown().await.unwrap(); Ok(()) } @@ -228,7 +237,10 @@ pub fn random_maps( iter::repeat(()).map(move |_| random_map(max_size, field_len)) } -pub async fn collect_n(rx: mpsc::Receiver, n: usize) -> Vec { +pub async fn collect_n(rx: S, n: usize) -> Vec +where + S: Stream + Unpin, +{ rx.take(n).collect().await } @@ -269,8 +281,7 @@ pub fn lines_from_gzip_file>(path: P) -> Vec { } pub fn runtime() -> runtime::Runtime { - runtime::Builder::new() - .threaded_scheduler() + runtime::Builder::new_multi_thread() .enable_all() .build() .unwrap() @@ -285,7 +296,7 @@ where let started = Instant::now(); let mut delay = WAIT_FOR_MIN_MILLIS; while !f().await { - delay_for(Duration::from_millis(delay)).await; + sleep(Duration::from_millis(delay)).await; if started.elapsed() > duration { panic!("Timed out while waiting"); } @@ -336,7 +347,7 @@ where while started.elapsed() < until { match f().await { Ok(res) => return res, - Err(_) => tokio::time::delay_for(retry).await, + Err(_) => tokio::time::sleep(retry).await, } } panic!("Timeout") @@ -426,9 +437,9 @@ impl Future for CountReceiver { impl CountReceiver { pub fn receive_lines(addr: SocketAddr) -> CountReceiver { CountReceiver::new(|count, tripwire, connected| async move { - let mut listener = TcpListener::bind(addr).await.unwrap(); + let listener = TcpListener::bind(addr).await.unwrap(); CountReceiver::receive_lines_stream( - listener.incoming(), + TcpListenerStream::new(listener), count, tripwire, Some(connected), @@ -443,9 +454,9 @@ impl CountReceiver { P: AsRef + Send + 'static, { CountReceiver::new(|count, tripwire, connected| async move { - let mut listener = tokio::net::UnixListener::bind(path).unwrap(); + let listener = tokio::net::UnixListener::bind(path).unwrap(); CountReceiver::receive_lines_stream( - listener.incoming(), + UnixListenerStream::new(listener), count, tripwire, Some(connected), diff --git a/src/tls/incoming.rs b/src/tls/incoming.rs index 3598aee81d803..fa7a2130b18e4 100644 --- a/src/tls/incoming.rs +++ b/src/tls/incoming.rs @@ -1,27 +1,25 @@ -#[cfg(feature = "listenfd")] -use super::Handshake; use super::{ - CreateAcceptor, IncomingListener, MaybeTlsSettings, MaybeTlsStream, TcpBind, TlsError, - TlsSettings, + CreateAcceptor, Handshake, IncomingListener, MaybeTlsSettings, MaybeTlsStream, SslBuildError, + TcpBind, TlsError, TlsSettings, }; +#[cfg(feature = "sources-utils-tcp-socket")] +use crate::tcp; #[cfg(feature = "sources-utils-tcp-keepalive")] use crate::tcp::TcpKeepaliveConfig; -use bytes::{Buf, BufMut}; use futures::{future::BoxFuture, stream, FutureExt, Stream}; -use openssl::ssl::{SslAcceptor, SslMethod}; +use openssl::ssl::{Ssl, SslAcceptor, SslMethod}; use snafu::ResultExt; use std::{ future::Future, - mem::MaybeUninit, net::SocketAddr, pin::Pin, task::{Context, Poll}, }; use tokio::{ - io::{self, AsyncRead, AsyncWrite}, + io::{self, AsyncRead, AsyncWrite, ReadBuf}, net::{TcpListener, TcpStream}, }; -use tokio_openssl::{HandshakeError, SslStream}; +use tokio_openssl::SslStream; impl TlsSettings { pub(crate) fn acceptor(&self) -> crate::tls::Result { @@ -110,8 +108,9 @@ pub struct MaybeTlsIncomingStream { enum StreamState { Accepted(MaybeTlsStream), - Accepting(BoxFuture<'static, Result, HandshakeError>>), + Accepting(BoxFuture<'static, Result, TlsError>>), AcceptError(String), + Closed, } impl MaybeTlsIncomingStream { @@ -136,6 +135,27 @@ impl MaybeTlsIncomingStream { }), StreamState::Accepting(_) => None, StreamState::AcceptError(_) => None, + StreamState::Closed => None, + } + } + + #[cfg(all( + test, + feature = "sinks-socket", + feature = "sources-utils-tls", + feature = "listenfd" + ))] + pub fn get_mut(&mut self) -> Option<&mut S> { + use super::MaybeTls; + + match &mut self.state { + StreamState::Accepted(ref mut stream) => Some(match stream { + MaybeTls::Raw(ref mut s) => s, + MaybeTls::Tls(s) => s.get_mut(), + }), + StreamState::Accepting(_) => None, + StreamState::AcceptError(_) => None, + StreamState::Closed => None, } } } @@ -148,7 +168,13 @@ impl MaybeTlsIncomingStream { ) -> Self { let state = match acceptor { Some(acceptor) => StreamState::Accepting( - async move { tokio_openssl::accept(&acceptor, stream).await }.boxed(), + async move { + let ssl = Ssl::new(acceptor.context()).context(SslBuildError)?; + let mut stream = SslStream::new(ssl, stream).context(SslBuildError)?; + Pin::new(&mut stream).accept().await.context(Handshake)?; + Ok(stream) + } + .boxed(), ), None => StreamState::Accepted(MaybeTlsStream::Raw(stream)), }; @@ -159,7 +185,7 @@ impl MaybeTlsIncomingStream { #[cfg(feature = "listenfd")] pub(crate) async fn handshake(&mut self) -> crate::tls::Result<()> { if let StreamState::Accepting(fut) = &mut self.state { - let stream = fut.await.context(Handshake)?; + let stream = fut.await?; self.state = StreamState::Accepted(MaybeTlsStream::Tls(stream)); } @@ -175,7 +201,12 @@ impl MaybeTlsIncomingStream { ) })?; - stream.set_keepalive(keepalive.time_secs.map(std::time::Duration::from_secs))?; + if let Some(time_secs) = keepalive.time_secs { + let config = + socket2::TcpKeepalive::new().with_time(std::time::Duration::from_secs(time_secs)); + + tcp::set_keepalive(stream, &config)?; + } Ok(()) } @@ -189,9 +220,7 @@ impl MaybeTlsIncomingStream { ) })?; - stream.set_recv_buffer_size(bytes)?; - - Ok(()) + tcp::set_receive_buffer_size(stream, bytes) } fn poll_io(self: Pin<&mut Self>, cx: &mut Context, poll_fn: F) -> Poll> @@ -216,6 +245,7 @@ impl MaybeTlsIncomingStream { StreamState::AcceptError(error) => { Poll::Ready(Err(io::Error::new(io::ErrorKind::Other, error.to_owned()))) } + StreamState::Closed => Poll::Ready(Err(io::ErrorKind::BrokenPipe.into())), }; } } @@ -225,24 +255,10 @@ impl AsyncRead for MaybeTlsIncomingStream { fn poll_read( self: Pin<&mut Self>, cx: &mut Context, - buf: &mut [u8], - ) -> Poll> { + buf: &mut ReadBuf<'_>, + ) -> Poll> { self.poll_io(cx, |s, cx| s.poll_read(cx, buf)) } - - unsafe fn prepare_uninitialized_buffer(&self, _buf: &mut [MaybeUninit]) -> bool { - // Both, TcpStream & SslStream return false - // We can not use `poll_io` here, because need Context for polling handshake - false - } - - fn poll_read_buf( - self: Pin<&mut Self>, - cx: &mut Context, - buf: &mut B, - ) -> Poll> { - self.poll_io(cx, |s, cx| s.poll_read_buf(cx, buf)) - } } impl AsyncWrite for MaybeTlsIncomingStream { @@ -255,14 +271,30 @@ impl AsyncWrite for MaybeTlsIncomingStream { } fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - self.poll_io(cx, |s, cx| s.poll_shutdown(cx)) - } - - fn poll_write_buf( - self: Pin<&mut Self>, - cx: &mut Context, - buf: &mut B, - ) -> Poll> { - self.poll_io(cx, |s, cx| s.poll_write_buf(cx, buf)) + let mut this = self.get_mut(); + match &mut this.state { + StreamState::Accepted(stream) => match Pin::new(stream).poll_shutdown(cx) { + Poll::Ready(Ok(())) => { + this.state = StreamState::Closed; + Poll::Ready(Ok(())) + } + poll_result => poll_result, + }, + StreamState::Accepting(fut) => match futures::ready!(fut.as_mut().poll(cx)) { + Ok(stream) => { + this.state = StreamState::Accepted(MaybeTlsStream::Tls(stream)); + Poll::Pending + } + Err(error) => { + let error = io::Error::new(io::ErrorKind::Other, error); + this.state = StreamState::AcceptError(error.to_string()); + Poll::Ready(Err(error)) + } + }, + StreamState::AcceptError(error) => { + Poll::Ready(Err(io::Error::new(io::ErrorKind::Other, error.to_owned()))) + } + StreamState::Closed => Poll::Ready(Ok(())), + } } } diff --git a/src/tls/maybe_tls.rs b/src/tls/maybe_tls.rs index 238a1bc17d403..696e40f83b3a4 100644 --- a/src/tls/maybe_tls.rs +++ b/src/tls/maybe_tls.rs @@ -1,12 +1,10 @@ -use bytes::{Buf, BufMut}; use pin_project::pin_project; use std::{ fmt, - mem::MaybeUninit, pin::Pin, task::{Context, Poll}, }; -use tokio::io::{self, AsyncRead, AsyncWrite}; +use tokio::io::{self, AsyncRead, AsyncWrite, ReadBuf}; /// A type wrapper for objects that can exist in either a raw state or /// wrapped by TLS handling. @@ -73,31 +71,13 @@ impl AsyncRead for MaybeTls { fn poll_read( self: Pin<&mut Self>, cx: &mut Context, - buf: &mut [u8], - ) -> Poll> { + buf: &mut ReadBuf<'_>, + ) -> Poll> { match self.project() { MaybeTlsProj::Tls(s) => s.poll_read(cx, buf), MaybeTlsProj::Raw(s) => s.poll_read(cx, buf), } } - - unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [MaybeUninit]) -> bool { - match self { - MaybeTls::Tls(s) => s.prepare_uninitialized_buffer(buf), - MaybeTls::Raw(s) => s.prepare_uninitialized_buffer(buf), - } - } - - fn poll_read_buf( - self: Pin<&mut Self>, - cx: &mut Context, - buf: &mut B, - ) -> Poll> { - match self.project() { - MaybeTlsProj::Tls(s) => s.poll_read_buf(cx, buf), - MaybeTlsProj::Raw(s) => s.poll_read_buf(cx, buf), - } - } } impl AsyncWrite for MaybeTls { @@ -121,15 +101,4 @@ impl AsyncWrite for MaybeTls { MaybeTlsProj::Raw(s) => s.poll_shutdown(cx), } } - - fn poll_write_buf( - self: Pin<&mut Self>, - cx: &mut Context, - buf: &mut B, - ) -> Poll> { - match self.project() { - MaybeTlsProj::Tls(s) => s.poll_write_buf(cx, buf), - MaybeTlsProj::Raw(s) => s.poll_write_buf(cx, buf), - } - } } diff --git a/src/tls/mod.rs b/src/tls/mod.rs index cf6fd1b39a027..f3dc4dc016668 100644 --- a/src/tls/mod.rs +++ b/src/tls/mod.rs @@ -1,4 +1,4 @@ -use crate::tcp::TcpKeepaliveConfig; +use crate::tcp::{self, TcpKeepaliveConfig}; use openssl::{ error::ErrorStack, ssl::{ConnectConfiguration, SslConnector, SslConnectorBuilder, SslMethod}, @@ -6,7 +6,7 @@ use openssl::{ use snafu::{ResultExt, Snafu}; use std::{fmt::Debug, net::SocketAddr, path::PathBuf, time::Duration}; use tokio::net::TcpStream; -use tokio_openssl::{HandshakeError, SslStream}; +use tokio_openssl::SslStream; #[cfg(feature = "sources-utils-tls")] mod incoming; @@ -76,11 +76,13 @@ pub enum TlsError { #[snafu(display("TLS configuration requires a certificate when enabled"))] MissingRequiredIdentity, #[snafu(display("TLS handshake failed: {}", source))] - Handshake { source: HandshakeError }, + Handshake { source: openssl::ssl::Error }, #[snafu(display("Incoming listener failed: {}", source))] IncomingListener { source: tokio::io::Error }, #[snafu(display("Creating the TLS acceptor failed: {}", source))] CreateAcceptor { source: ErrorStack }, + #[snafu(display("Error building SSL context: {}", source))] + SslBuildError { source: openssl::error::ErrorStack }, #[snafu(display("Error setting up the TLS certificate: {}", source))] SetCertificate { source: ErrorStack }, #[snafu(display("Error setting up the TLS private key: {}", source))] @@ -132,7 +134,11 @@ impl MaybeTlsStream { Self::Tls(tls) => tls.get_ref(), }; - stream.set_keepalive(keepalive.time_secs.map(Duration::from_secs))?; + if let Some(time_secs) = keepalive.time_secs { + let config = socket2::TcpKeepalive::new().with_time(Duration::from_secs(time_secs)); + + tcp::set_keepalive(stream, &config)?; + } Ok(()) } @@ -143,9 +149,7 @@ impl MaybeTlsStream { Self::Tls(tls) => tls.get_ref(), }; - stream.set_send_buffer_size(bytes)?; - - Ok(()) + tcp::set_send_buffer_size(stream, bytes) } pub fn set_receive_buffer_bytes(&mut self, bytes: usize) -> std::io::Result<()> { @@ -154,9 +158,7 @@ impl MaybeTlsStream { Self::Tls(tls) => tls.get_ref(), }; - stream.set_recv_buffer_size(bytes)?; - - Ok(()) + tcp::set_receive_buffer_size(stream, bytes) } } diff --git a/src/tls/outgoing.rs b/src/tls/outgoing.rs index d062aa0c35789..ec750ef921a0c 100644 --- a/src/tls/outgoing.rs +++ b/src/tls/outgoing.rs @@ -1,7 +1,8 @@ -use super::{tls_connector, Connect, Handshake, MaybeTlsSettings, MaybeTlsStream}; +use super::{tls_connector, Connect, Handshake, MaybeTlsSettings, MaybeTlsStream, SslBuildError}; use snafu::ResultExt; -use std::net::SocketAddr; +use std::{net::SocketAddr, pin::Pin}; use tokio::net::TcpStream; +use tokio_openssl::SslStream; impl MaybeTlsSettings { pub(crate) async fn connect( @@ -15,9 +16,10 @@ impl MaybeTlsSettings { MaybeTlsSettings::Raw(()) => Ok(MaybeTlsStream::Raw(stream)), MaybeTlsSettings::Tls(_) => { let config = tls_connector(self)?; - let stream = tokio_openssl::connect(config, host, stream) - .await - .context(Handshake)?; + let ssl = config.into_ssl(host).context(SslBuildError)?; + + let mut stream = SslStream::new(ssl, stream).context(SslBuildError)?; + Pin::new(&mut stream).connect().await.context(Handshake)?; debug!(message = "Negotiated TLS."); diff --git a/src/top/metrics.rs b/src/top/metrics.rs index 845fddfa79262..4c3b859f9e13a 100644 --- a/src/top/metrics.rs +++ b/src/top/metrics.rs @@ -1,13 +1,13 @@ use super::state; use std::sync::Arc; -use tokio::stream::StreamExt; +use tokio_stream::StreamExt; use vector_api_client::{ gql::{ComponentsQueryExt, ComponentsSubscriptionExt, MetricsSubscriptionExt}, Client, SubscriptionClient, }; /// Components that have been added -async fn component_added(client: Arc, mut tx: state::EventTx) { +async fn component_added(client: Arc, tx: state::EventTx) { let res = client.component_added(); tokio::pin! { @@ -34,7 +34,7 @@ async fn component_added(client: Arc, mut tx: state::EventTx } /// Components that have been removed -async fn component_removed(client: Arc, mut tx: state::EventTx) { +async fn component_removed(client: Arc, tx: state::EventTx) { let res = client.component_removed(); tokio::pin! { @@ -51,7 +51,7 @@ async fn component_removed(client: Arc, mut tx: state::Event async fn processed_events_totals( client: Arc, - mut tx: state::EventTx, + tx: state::EventTx, interval: i64, ) { let res = client.component_processed_events_totals_subscription(interval); @@ -76,7 +76,7 @@ async fn processed_events_totals( async fn processed_events_throughputs( client: Arc, - mut tx: state::EventTx, + tx: state::EventTx, interval: i64, ) { let res = client.component_processed_events_throughputs_subscription(interval); @@ -100,7 +100,7 @@ async fn processed_events_throughputs( async fn processed_bytes_totals( client: Arc, - mut tx: state::EventTx, + tx: state::EventTx, interval: i64, ) { let res = client.component_processed_bytes_totals_subscription(interval); @@ -125,7 +125,7 @@ async fn processed_bytes_totals( async fn processed_bytes_throughputs( client: Arc, - mut tx: state::EventTx, + tx: state::EventTx, interval: i64, ) { let res = client.component_processed_bytes_throughputs_subscription(interval); diff --git a/src/top/state.rs b/src/top/state.rs index ddd57b455a422..dd96ff2038ca9 100644 --- a/src/top/state.rs +++ b/src/top/state.rs @@ -36,7 +36,7 @@ pub struct ComponentRow { /// represents the single destination for handling subscriptions and returning 'immutable' state /// for re-rendering the dashboard. This approach uses channels vs. mutexes. pub async fn updater(mut state: State, mut event_rx: EventRx) -> StateRx { - let (mut tx, rx) = mpsc::channel(20); + let (tx, rx) = mpsc::channel(20); // Prime the receiver with the initial state let _ = tx.send(state.clone()).await; diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 6fead95a056c9..d83d836259ba9 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -54,7 +54,7 @@ pub async fn build_pieces( .iter() .filter(|(name, _)| diff.sources.contains_new(&name)) { - let (tx, rx) = tokio::sync::mpsc::channel(1000); + let (tx, rx) = futures::channel::mpsc::channel(1000); let pipeline = Pipeline::from_sender(tx, vec![]); let typetag = source.source_type(); diff --git a/src/topology/fanout.rs b/src/topology/fanout.rs index dc02491dadf78..c6bd8a5bbc5ad 100644 --- a/src/topology/fanout.rs +++ b/src/topology/fanout.rs @@ -1,11 +1,10 @@ use crate::Event; -use futures::{future, Sink, Stream}; +use futures::{channel::mpsc, future, stream::Fuse, Sink, Stream, StreamExt}; use std::{ fmt, pin::Pin, task::{Context, Poll}, }; -use tokio::sync::mpsc; type RouterSink = Box + 'static + Send>; @@ -32,17 +31,17 @@ pub type ControlChannel = mpsc::UnboundedSender; pub struct Fanout { sinks: Vec<(String, Option>)>, i: usize, - control_channel: mpsc::UnboundedReceiver, + control_channel: Fuse>, } impl Fanout { pub fn new() -> (Self, ControlChannel) { - let (control_tx, control_rx) = mpsc::unbounded_channel(); + let (control_tx, control_rx) = mpsc::unbounded(); let fanout = Self { sinks: vec![], i: 0, - control_channel: control_rx, + control_channel: control_rx.fuse(), }; (fanout, control_tx) @@ -192,23 +191,22 @@ impl Sink for Fanout { #[cfg(test)] mod tests { use super::{ControlMessage, Fanout}; - use crate::{sink::BoundedSink, test_util::collect_ready, Event}; - use futures::{stream, Sink, SinkExt, StreamExt}; + use crate::{test_util::collect_ready, Event}; + use futures::{channel::mpsc, stream, FutureExt, Sink, SinkExt, StreamExt}; use std::{ pin::Pin, task::{Context, Poll}, }; - use tokio::sync::mpsc; - use tokio::time::{delay_for, Duration}; + use tokio::time::{sleep, Duration}; #[tokio::test] async fn fanout_writes_to_all() { - let (tx_a, rx_a) = unbounded_channel(); + let (tx_a, rx_a) = mpsc::unbounded(); let tx_a = Box::new(tx_a.sink_map_err(|_| unreachable!())); - let (tx_b, rx_b) = unbounded_channel(); + let (tx_b, rx_b) = mpsc::unbounded(); let tx_b = Box::new(tx_b.sink_map_err(|_| unreachable!())); - let mut fanout = Fanout::new().0; + let (mut fanout, _fanout_control) = Fanout::new(); fanout.add("a".to_string(), tx_a); fanout.add("b".to_string(), tx_b); @@ -223,14 +221,14 @@ mod tests { #[tokio::test] async fn fanout_notready() { - let (tx_a, rx_a) = channel(2); + let (tx_a, rx_a) = mpsc::channel(1); let tx_a = Box::new(tx_a.sink_map_err(|_| unreachable!())); - let (tx_b, rx_b) = channel(1); + let (tx_b, rx_b) = mpsc::channel(0); let tx_b = Box::new(tx_b.sink_map_err(|_| unreachable!())); - let (tx_c, rx_c) = channel(2); + let (tx_c, rx_c) = mpsc::channel(1); let tx_c = Box::new(tx_c.sink_map_err(|_| unreachable!())); - let mut fanout = Fanout::new().0; + let (mut fanout, _fanout_control) = Fanout::new(); fanout.add("a".to_string(), tx_a); fanout.add("b".to_string(), tx_b); @@ -240,7 +238,7 @@ mod tests { let send = stream::iter(recs.clone()).map(Ok).forward(fanout); tokio::spawn(send); - delay_for(Duration::from_millis(50)).await; + sleep(Duration::from_millis(50)).await; // The send_all task will be blocked on sending rec1 because of b right now. let collect_a = tokio::spawn(rx_a.collect::>()); @@ -254,12 +252,12 @@ mod tests { #[tokio::test] async fn fanout_grow() { - let (tx_a, rx_a) = unbounded_channel(); + let (tx_a, rx_a) = mpsc::unbounded(); let tx_a = Box::new(tx_a.sink_map_err(|_| unreachable!())); - let (tx_b, rx_b) = unbounded_channel(); + let (tx_b, rx_b) = mpsc::unbounded(); let tx_b = Box::new(tx_b.sink_map_err(|_| unreachable!())); - let mut fanout = Fanout::new().0; + let (mut fanout, _fanout_control) = Fanout::new(); fanout.add("a".to_string(), tx_a); fanout.add("b".to_string(), tx_b); @@ -269,7 +267,7 @@ mod tests { fanout.send(recs[0].clone()).await.unwrap(); fanout.send(recs[1].clone()).await.unwrap(); - let (tx_c, rx_c) = unbounded_channel(); + let (tx_c, rx_c) = mpsc::unbounded(); let tx_c = Box::new(tx_c.sink_map_err(|_| unreachable!())); fanout.add("c".to_string(), tx_c); @@ -282,12 +280,12 @@ mod tests { #[tokio::test] async fn fanout_shrink() { - let (tx_a, rx_a) = unbounded_channel(); + let (tx_a, rx_a) = mpsc::unbounded(); let tx_a = Box::new(tx_a.sink_map_err(|_| unreachable!())); - let (tx_b, rx_b) = unbounded_channel(); + let (tx_b, rx_b) = mpsc::unbounded(); let tx_b = Box::new(tx_b.sink_map_err(|_| unreachable!())); - let (mut fanout, fanout_control) = Fanout::new(); + let (mut fanout, mut fanout_control) = Fanout::new(); fanout.add("a".to_string(), tx_a); fanout.add("b".to_string(), tx_b); @@ -299,6 +297,7 @@ mod tests { fanout_control .send(ControlMessage::Remove("b".to_string())) + .await .unwrap(); fanout.send(recs[2].clone()).await.unwrap(); @@ -309,14 +308,14 @@ mod tests { #[tokio::test] async fn fanout_shrink_after_notready() { - let (tx_a, rx_a) = channel(2); + let (tx_a, rx_a) = mpsc::channel(1); let tx_a = Box::new(tx_a.sink_map_err(|_| unreachable!())); - let (tx_b, rx_b) = channel(1); + let (tx_b, rx_b) = mpsc::channel(0); let tx_b = Box::new(tx_b.sink_map_err(|_| unreachable!())); - let (tx_c, rx_c) = channel(2); + let (tx_c, rx_c) = mpsc::channel(1); let tx_c = Box::new(tx_c.sink_map_err(|_| unreachable!())); - let (mut fanout, fanout_control) = Fanout::new(); + let (mut fanout, mut fanout_control) = Fanout::new(); fanout.add("a".to_string(), tx_a); fanout.add("b".to_string(), tx_b); @@ -326,10 +325,11 @@ mod tests { let send = stream::iter(recs.clone()).map(Ok).forward(fanout); tokio::spawn(send); - delay_for(Duration::from_millis(50)).await; + sleep(Duration::from_millis(50)).await; // The send_all task will be blocked on sending rec1 because of b right now. fanout_control .send(ControlMessage::Remove("c".to_string())) + .await .unwrap(); let collect_a = tokio::spawn(rx_a.collect::>()); @@ -343,14 +343,14 @@ mod tests { #[tokio::test] async fn fanout_shrink_at_notready() { - let (tx_a, rx_a) = channel(2); + let (tx_a, rx_a) = mpsc::channel(1); let tx_a = Box::new(tx_a.sink_map_err(|_| unreachable!())); - let (tx_b, rx_b) = channel(1); + let (tx_b, rx_b) = mpsc::channel(0); let tx_b = Box::new(tx_b.sink_map_err(|_| unreachable!())); - let (tx_c, rx_c) = channel(2); + let (tx_c, rx_c) = mpsc::channel(1); let tx_c = Box::new(tx_c.sink_map_err(|_| unreachable!())); - let (mut fanout, fanout_control) = Fanout::new(); + let (mut fanout, mut fanout_control) = Fanout::new(); fanout.add("a".to_string(), tx_a); fanout.add("b".to_string(), tx_b); @@ -360,10 +360,11 @@ mod tests { let send = stream::iter(recs.clone()).map(Ok).forward(fanout); tokio::spawn(send); - delay_for(Duration::from_millis(50)).await; + sleep(Duration::from_millis(50)).await; // The send_all task will be blocked on sending rec1 because of b right now. fanout_control .send(ControlMessage::Remove("b".to_string())) + .await .unwrap(); let collect_a = tokio::spawn(rx_a.collect::>()); @@ -377,14 +378,14 @@ mod tests { #[tokio::test] async fn fanout_shrink_before_notready() { - let (tx_a, rx_a) = channel(2); + let (tx_a, rx_a) = mpsc::channel(1); let tx_a = Box::new(tx_a.sink_map_err(|_| unreachable!())); - let (tx_b, rx_b) = channel(1); + let (tx_b, rx_b) = mpsc::channel(0); let tx_b = Box::new(tx_b.sink_map_err(|_| unreachable!())); - let (tx_c, rx_c) = channel(2); + let (tx_c, rx_c) = mpsc::channel(1); let tx_c = Box::new(tx_c.sink_map_err(|_| unreachable!())); - let (mut fanout, fanout_control) = Fanout::new(); + let (mut fanout, mut fanout_control) = Fanout::new(); fanout.add("a".to_string(), tx_a); fanout.add("b".to_string(), tx_b); @@ -394,11 +395,12 @@ mod tests { let send = stream::iter(recs.clone()).map(Ok).forward(fanout); tokio::spawn(send); - delay_for(Duration::from_millis(50)).await; + sleep(Duration::from_millis(50)).await; // The send_all task will be blocked on sending rec1 because of b right now. fanout_control .send(ControlMessage::Remove("a".to_string())) + .await .unwrap(); let collect_a = tokio::spawn(rx_a.collect::>()); @@ -412,7 +414,7 @@ mod tests { #[tokio::test] async fn fanout_no_sinks() { - let mut fanout = Fanout::new().0; + let (mut fanout, _fanout_control) = Fanout::new(); let recs = make_events(2); @@ -422,12 +424,12 @@ mod tests { #[tokio::test] async fn fanout_replace() { - let (tx_a1, rx_a1) = unbounded_channel(); + let (tx_a1, rx_a1) = mpsc::unbounded(); let tx_a1 = Box::new(tx_a1.sink_map_err(|_| unreachable!())); - let (tx_b, rx_b) = unbounded_channel(); + let (tx_b, rx_b) = mpsc::unbounded(); let tx_b = Box::new(tx_b.sink_map_err(|_| unreachable!())); - let mut fanout = Fanout::new().0; + let (mut fanout, _fanout_control) = Fanout::new(); fanout.add("a".to_string(), tx_a1); fanout.add("b".to_string(), tx_b); @@ -437,7 +439,7 @@ mod tests { fanout.send(recs[0].clone()).await.unwrap(); fanout.send(recs[1].clone()).await.unwrap(); - let (tx_a2, rx_a2) = unbounded_channel(); + let (tx_a2, rx_a2) = mpsc::unbounded(); let tx_a2 = Box::new(tx_a2.sink_map_err(|_| unreachable!())); fanout.replace("a".to_string(), Some(tx_a2)); @@ -450,12 +452,12 @@ mod tests { #[tokio::test] async fn fanout_wait() { - let (tx_a1, rx_a1) = unbounded_channel(); + let (tx_a1, rx_a1) = mpsc::unbounded(); let tx_a1 = Box::new(tx_a1.sink_map_err(|_| unreachable!())); - let (tx_b, rx_b) = unbounded_channel(); + let (tx_b, rx_b) = mpsc::unbounded(); let tx_b = Box::new(tx_b.sink_map_err(|_| unreachable!())); - let (mut fanout, cc) = Fanout::new(); + let (mut fanout, mut fanout_control) = Fanout::new(); fanout.add("a".to_string(), tx_a1); fanout.add("b".to_string(), tx_b); @@ -465,17 +467,20 @@ mod tests { fanout.send(recs[0].clone()).await.unwrap(); fanout.send(recs[1].clone()).await.unwrap(); - let (tx_a2, rx_a2) = unbounded_channel(); + let (tx_a2, rx_a2) = mpsc::unbounded(); let tx_a2 = Box::new(tx_a2.sink_map_err(|_| unreachable!())); fanout.replace("a".to_string(), None); - tokio::spawn(async move { - delay_for(Duration::from_millis(100)).await; - cc.send(ControlMessage::Replace("a".to_string(), Some(tx_a2))) - .unwrap(); - }); - - fanout.send(recs[2].clone()).await.unwrap(); + futures::join!( + async { + sleep(Duration::from_millis(100)).await; + fanout_control + .send(ControlMessage::Replace("a".to_string(), Some(tx_a2))) + .await + .unwrap(); + }, + fanout.send(recs[2].clone()).map(|_| ()) + ); assert_eq!(collect_ready(rx_a1).await, &recs[..2]); assert_eq!(collect_ready(rx_b).await, recs); @@ -523,7 +528,7 @@ mod tests { } async fn fanout_error(modes: &[Option]) { - let mut fanout = Fanout::new().0; + let (mut fanout, _fanout_control) = Fanout::new(); let mut rx_channels = vec![]; for (i, mode) in modes.iter().enumerate() { @@ -535,7 +540,7 @@ mod tests { fanout.add(name, tx); } None => { - let (tx, rx) = channel(1); + let (tx, rx) = mpsc::channel(0); let tx = Box::new(tx.sink_map_err(|_| unreachable!())); fanout.add(name, tx); rx_channels.push(rx); @@ -547,13 +552,14 @@ mod tests { let send = stream::iter(recs.clone()).map(Ok).forward(fanout); tokio::spawn(send); - delay_for(Duration::from_millis(50)).await; + sleep(Duration::from_millis(50)).await; // Start collecting from all at once let collectors = rx_channels .into_iter() .map(|rx| tokio::spawn(rx.collect::>())) .collect::>(); + for collect in collectors { assert_eq!(collect.await.unwrap(), recs); } @@ -606,34 +612,4 @@ mod tests { .map(|i| Event::from(format!("line {}", i))) .collect() } - - fn unbounded_channel() -> (UnboundedSink, mpsc::UnboundedReceiver) { - let (sender, recv) = mpsc::unbounded_channel(); - (UnboundedSink { sender }, recv) - } - - fn channel(capacity: usize) -> (BoundedSink, mpsc::Receiver) { - let (sender, recv) = mpsc::channel(capacity); - (BoundedSink::new(sender), recv) - } - - struct UnboundedSink { - sender: mpsc::UnboundedSender, - } - - impl Sink for UnboundedSink { - type Error = mpsc::error::SendError; - fn poll_ready(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - fn start_send(self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> { - self.sender.send(item) - } - fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - fn poll_close(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - } } diff --git a/src/topology/mod.rs b/src/topology/mod.rs index 10635c28ea68d..ff09256a2cacd 100644 --- a/src/topology/mod.rs +++ b/src/topology/mod.rs @@ -21,7 +21,7 @@ use crate::{ }, trigger::DisabledTrigger, }; -use futures::{future, Future, FutureExt, Stream}; +use futures::{future, Future, FutureExt, SinkExt, Stream}; use std::{ collections::{HashMap, HashSet}, panic::AssertUnwindSafe, @@ -30,7 +30,7 @@ use std::{ }; use tokio::{ sync::mpsc, - time::{delay_until, interval, Duration, Instant}, + time::{interval, sleep_until, Duration, Instant}, }; use tracing_futures::Instrument; @@ -149,7 +149,7 @@ impl RunningTopology { // gracefully shutdown since we will start to forcefully shutdown the sources. let mut check_handles2 = check_handles.clone(); let timeout = async move { - delay_until(deadline).await; + sleep_until(deadline).await; // Remove all tasks that have shutdown. check_handles2.retain(|_name, handles| { retain(handles, |handle| handle.peek().is_none()); @@ -225,7 +225,7 @@ impl RunningTopology { // Issue: https://github.com/timberio/vector/issues/3035 if cfg!(windows) { // This value is guess work. - tokio::time::delay_for(Duration::from_millis(200)).await; + tokio::time::sleep(Duration::from_millis(200)).await; } // Now let's actually build the new pieces. @@ -360,7 +360,7 @@ impl RunningTopology { let previous = self.tasks.remove(name).unwrap(); drop(previous); // detach and forget - self.remove_inputs(&name); + self.remove_inputs(&name).await; self.remove_outputs(&name); } @@ -418,7 +418,7 @@ impl RunningTopology { // Detach removed sinks for name in &diff.sinks.to_remove { info!(message = "Removing sink.", name = ?name); - self.remove_inputs(&name); + self.remove_inputs(&name).await; } // Detach changed sinks @@ -430,7 +430,7 @@ impl RunningTopology { .into_inner() .cancel(); } else if wait_for_sinks.contains(name) { - self.detach_inputs(name); + self.detach_inputs(name).await; } } @@ -474,31 +474,31 @@ impl RunningTopology { async fn connect_diff(&mut self, diff: &ConfigDiff, new_pieces: &mut Pieces) { // Sources for name in diff.sources.changed_and_added() { - self.setup_outputs(&name, new_pieces); + self.setup_outputs(&name, new_pieces).await; } // Transforms // Make sure all transform outputs are set up before another transform might try use // it as an input for name in diff.transforms.changed_and_added() { - self.setup_outputs(&name, new_pieces); + self.setup_outputs(&name, new_pieces).await; } for name in &diff.transforms.to_change { - self.replace_inputs(&name, new_pieces); + self.replace_inputs(&name, new_pieces).await; } for name in &diff.transforms.to_add { - self.setup_inputs(&name, new_pieces); + self.setup_inputs(&name, new_pieces).await; } // Sinks for name in &diff.sinks.to_change { - self.replace_inputs(&name, new_pieces); + self.replace_inputs(&name, new_pieces).await; } for name in &diff.sinks.to_add { - self.setup_inputs(&name, new_pieces); + self.setup_inputs(&name, new_pieces).await; } } @@ -595,7 +595,7 @@ impl RunningTopology { self.outputs.remove(name); } - fn remove_inputs(&mut self, name: &str) { + async fn remove_inputs(&mut self, name: &str) { self.inputs.remove(name); self.detach_triggers.remove(name); @@ -606,24 +606,26 @@ impl RunningTopology { if let Some(inputs) = inputs { for input in inputs { - if let Some(output) = self.outputs.get(input) { + if let Some(output) = self.outputs.get_mut(input) { // This can only fail if we are disconnected, which is a valid situation. - let _ = output.send(fanout::ControlMessage::Remove(name.to_string())); + let _ = output + .send(fanout::ControlMessage::Remove(name.to_string())) + .await; } } } } - fn setup_outputs(&mut self, name: &str, new_pieces: &mut builder::Pieces) { - let output = new_pieces.outputs.remove(name).unwrap(); + async fn setup_outputs(&mut self, name: &str, new_pieces: &mut builder::Pieces) { + let mut output = new_pieces.outputs.remove(name).unwrap(); for (sink_name, sink) in &self.config.sinks { if sink.inputs.iter().any(|i| i == name) { // Sink may have been removed with the new config so it may not be present. if let Some(input) = self.inputs.get(sink_name) { - output + let _ = output .send(fanout::ControlMessage::Add(sink_name.clone(), input.get())) - .expect("Components shouldn't be spawned before connecting them together."); + .await; } } } @@ -631,12 +633,12 @@ impl RunningTopology { if transform.inputs.iter().any(|i| i == name) { // Transform may have been removed with the new config so it may not be present. if let Some(input) = self.inputs.get(transform_name) { - output + let _ = output .send(fanout::ControlMessage::Add( transform_name.clone(), input.get(), )) - .expect("Components shouldn't be spawned before connecting them together."); + .await; } } } @@ -644,13 +646,17 @@ impl RunningTopology { self.outputs.insert(name.to_string(), output); } - fn setup_inputs(&mut self, name: &str, new_pieces: &mut builder::Pieces) { + async fn setup_inputs(&mut self, name: &str, new_pieces: &mut builder::Pieces) { let (tx, inputs) = new_pieces.inputs.remove(name).unwrap(); for input in inputs { // This can only fail if we are disconnected, which is a valid situation. - let _ = - self.outputs[&input].send(fanout::ControlMessage::Add(name.to_string(), tx.get())); + let _ = self + .outputs + .get_mut(&input) + .unwrap() + .send(fanout::ControlMessage::Add(name.to_string(), tx.get())) + .await; } self.inputs.insert(name.to_string(), tx); @@ -660,7 +666,7 @@ impl RunningTopology { }); } - fn replace_inputs(&mut self, name: &str, new_pieces: &mut builder::Pieces) { + async fn replace_inputs(&mut self, name: &str, new_pieces: &mut builder::Pieces) { let (tx, inputs) = new_pieces.inputs.remove(name).unwrap(); let sink_inputs = self.config.sinks.get(name).map(|s| &s.inputs); @@ -678,24 +684,35 @@ impl RunningTopology { let inputs_to_replace = old_inputs.intersection(&new_inputs); for input in inputs_to_remove { - if let Some(output) = self.outputs.get(input) { + if let Some(output) = self.outputs.get_mut(input) { // This can only fail if we are disconnected, which is a valid situation. - let _ = output.send(fanout::ControlMessage::Remove(name.to_string())); + let _ = output + .send(fanout::ControlMessage::Remove(name.to_string())) + .await; } } for input in inputs_to_add { // This can only fail if we are disconnected, which is a valid situation. - let _ = - self.outputs[input].send(fanout::ControlMessage::Add(name.to_string(), tx.get())); + let _ = self + .outputs + .get_mut(input) + .unwrap() + .send(fanout::ControlMessage::Add(name.to_string(), tx.get())) + .await; } for &input in inputs_to_replace { // This can only fail if we are disconnected, which is a valid situation. - let _ = self.outputs[input].send(fanout::ControlMessage::Replace( - name.to_string(), - Some(tx.get()), - )); + let _ = self + .outputs + .get_mut(input) + .unwrap() + .send(fanout::ControlMessage::Replace( + name.to_string(), + Some(tx.get()), + )) + .await; } self.inputs.insert(name.to_string(), tx); @@ -705,7 +722,7 @@ impl RunningTopology { }); } - fn detach_inputs(&mut self, name: &str) { + async fn detach_inputs(&mut self, name: &str) { self.inputs.remove(name); self.detach_triggers.remove(name); @@ -715,8 +732,12 @@ impl RunningTopology { for input in old_inputs { // This can only fail if we are disconnected, which is a valid situation. - let _ = - self.outputs[input].send(fanout::ControlMessage::Replace(name.to_string(), None)); + let _ = self + .outputs + .get_mut(input) + .unwrap() + .send(fanout::ControlMessage::Replace(name.to_string(), None)) + .await; } } @@ -816,7 +837,8 @@ mod reload_tests { use futures::StreamExt; use std::net::{SocketAddr, TcpListener}; use std::time::Duration; - use tokio::time::delay_for; + use tokio::time::sleep; + use tokio_stream::wrappers::UnboundedReceiverStream; #[tokio::test] async fn topology_reuse_old_port() { @@ -1013,7 +1035,7 @@ mod reload_tests { .await; } - #[tokio::test(core_threads = 2)] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn topology_disk_buffer_conflict() { let address_0 = next_addr(); let address_1 = next_addr(); @@ -1077,13 +1099,14 @@ mod reload_tests { old_address: SocketAddr, new_address: SocketAddr, ) { - let (mut topology, mut crash) = start_topology(old_config, false).await; + let (mut topology, crash) = start_topology(old_config, false).await; + let mut crash_stream = UnboundedReceiverStream::new(crash); // Wait for sink to come online wait_for_tcp(old_address).await; // Give topology some time to run - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; assert!(topology .reload_config_and_respawn(new_config) @@ -1091,11 +1114,11 @@ mod reload_tests { .unwrap()); // Give old time to shutdown if it didn't, and new one to come online. - delay_for(Duration::from_secs(2)).await; + sleep(Duration::from_secs(2)).await; tokio::select! { _ = wait_for_tcp(new_address) => {}//Success - _ = crash.next() => panic!(), + _ = crash_stream.next() => panic!(), } } } @@ -1151,12 +1174,14 @@ mod transient_state_tests { }; use futures::{future, FutureExt}; use serde::{Deserialize, Serialize}; + use std::sync::Arc; use stream_cancel::{Trigger, Tripwire}; + use tokio::sync::Mutex; #[derive(Debug, Deserialize, Serialize)] pub struct MockSourceConfig { #[serde(skip)] - tripwire: Option, + tripwire: Arc>>, } impl MockSourceConfig { @@ -1165,7 +1190,7 @@ mod transient_state_tests { ( trigger, Self { - tripwire: Some(tripwire), + tripwire: Arc::new(Mutex::new(Some(tripwire))), }, ) } @@ -1181,10 +1206,12 @@ mod transient_state_tests { shutdown: ShutdownSignal, out: Pipeline, ) -> Result { + let tripwire = self.tripwire.lock().await; + Ok(Box::pin( future::select( shutdown.map(|_| ()).boxed(), - self.tripwire + tripwire .clone() .unwrap() .then(crate::stream::tripwire_handler) diff --git a/src/transforms/aws_ec2_metadata.rs b/src/transforms/aws_ec2_metadata.rs index c364cdd3c7075..d1c7e4b6ef970 100644 --- a/src/transforms/aws_ec2_metadata.rs +++ b/src/transforms/aws_ec2_metadata.rs @@ -17,7 +17,7 @@ use std::{ future::ready, pin::Pin, }; -use tokio::time::{delay_for, Duration, Instant}; +use tokio::time::{sleep, Duration, Instant}; use tracing_futures::Instrument; type WriteHandle = evmap::WriteHandle; @@ -254,7 +254,7 @@ impl MetadataClient { } } - delay_for(self.refresh_interval).await; + sleep(self.refresh_interval).await; } } @@ -536,15 +536,15 @@ mod integration_tests { .into_task(); let (mut tx, rx) = futures::channel::mpsc::channel(100); - let mut rx = transform.transform(Box::pin(rx)); + let mut stream = transform.transform(Box::pin(rx)); // We need to sleep to let the background task fetch the data. - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let event = Event::new_empty_log(); tx.send(event).await.unwrap(); - let event = rx.next().await.unwrap(); + let event = stream.next().await.unwrap(); let log = event.as_log(); assert_eq!(log.get("availability-zone"), Some(&"ww-region-1a".into())); @@ -578,15 +578,15 @@ mod integration_tests { .into_task(); let (mut tx, rx) = futures::channel::mpsc::channel(100); - let mut rx = transform.transform(Box::pin(rx)); + let mut stream = transform.transform(Box::pin(rx)); // We need to sleep to let the background task fetch the data. - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let event = Event::new_empty_log(); tx.send(event).await.unwrap(); - let event = rx.next().await.unwrap(); + let event = stream.next().await.unwrap(); let log = event.as_log(); assert_eq!(log.get("availability-zone"), None); @@ -615,15 +615,15 @@ mod integration_tests { .into_task(); let (mut tx, rx) = futures::channel::mpsc::channel(100); - let mut rx = transform.transform(Box::pin(rx)); + let mut stream = transform.transform(Box::pin(rx)); // We need to sleep to let the background task fetch the data. - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let event = Event::new_empty_log(); tx.send(event).await.unwrap(); - let event = rx.next().await.unwrap(); + let event = stream.next().await.unwrap(); let log = event.as_log(); assert_eq!( @@ -650,15 +650,15 @@ mod integration_tests { .into_task(); let (mut tx, rx) = futures::channel::mpsc::channel(100); - let mut rx = transform.transform(Box::pin(rx)); + let mut stream = transform.transform(Box::pin(rx)); // We need to sleep to let the background task fetch the data. - delay_for(Duration::from_secs(1)).await; + sleep(Duration::from_secs(1)).await; let event = Event::new_empty_log(); tx.send(event).await.unwrap(); - let event = rx.next().await.unwrap(); + let event = stream.next().await.unwrap(); let log = event.as_log(); assert_eq!(log.get("availability-zone"), Some(&"ww-region-1a".into())); diff --git a/src/transforms/reduce/mod.rs b/src/transforms/reduce/mod.rs index 6a727003ce42a..5640622175fac 100644 --- a/src/transforms/reduce/mod.rs +++ b/src/transforms/reduce/mod.rs @@ -260,7 +260,7 @@ impl TaskTransform for Reduce { loop { let mut output = Vec::new(); let done = tokio::select! { - _ = flush_stream.next() => { + _ = flush_stream.tick() => { me.flush_into(&mut output); false } diff --git a/src/transforms/util/runtime_transform.rs b/src/transforms/util/runtime_transform.rs index 442cb0eccbd74..1f7ce9169ca10 100644 --- a/src/transforms/util/runtime_transform.rs +++ b/src/transforms/util/runtime_transform.rs @@ -5,6 +5,7 @@ use futures::{ }; use std::{future::ready, pin::Pin}; use tokio::time; +use tokio_stream::wrappers::IntervalStream; /// A structure representing user-defined timer. #[derive(Clone, Copy, Debug)] @@ -138,7 +139,7 @@ where fn make_timer_msgs_stream(timers: Vec) -> BoxStream<'static, Message> { let streams = timers.into_iter().map(|timer| { let period = time::Duration::from_secs(timer.interval_seconds); - time::interval(period).map(move |_| Message::Timer(timer)) + IntervalStream::new(time::interval(period)).map(move |_| Message::Timer(timer)) }); stream::select_all(streams).boxed() } diff --git a/src/udp.rs b/src/udp.rs index 1f978131443ed..b9bfeddd38eb1 100644 --- a/src/udp.rs +++ b/src/udp.rs @@ -1,40 +1,14 @@ -#[cfg(unix)] -use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd}; -#[cfg(unix)] +use socket2::SockRef; use tokio::net::UdpSocket; -#[cfg(unix)] // This function will be obsolete after tokio/mio internally use `socket2` and expose the methods to -// apply options to a socket. Until then, use of `unsafe` is necessary here. -pub fn set_receive_buffer_size(socket: &UdpSocket, size: usize) { - // SAFETY: We create a socket from an existing file descriptor without destructing the previous - // owner and therefore temporarily have two objects that own the same socket. - // - // This is safe since we make sure that the new socket owner does not call its destructor by - // giving up its ownership at the end of this scope. - let socket = unsafe { socket2::Socket::from_raw_fd(socket.as_raw_fd()) }; - - if let Err(error) = socket.set_recv_buffer_size(size) { - warn!(message = "Failed configuring receive buffer size on UDP socket.", %error); - } - - socket.into_raw_fd(); +// apply options to a socket. +pub fn set_receive_buffer_size(socket: &UdpSocket, size: usize) -> std::io::Result<()> { + SockRef::from(socket).set_recv_buffer_size(size) } -#[cfg(unix)] // This function will be obsolete after tokio/mio internally use `socket2` and expose the methods to -// apply options to a socket. Until then, use of `unsafe` is necessary here. -pub fn set_send_buffer_size(socket: &UdpSocket, size: usize) { - // SAFETY: We create a socket from an existing file descriptor without destructing the previous - // owner and therefore temporarily have two objects that own the same socket. - // - // This is safe since we make sure that the new socket owner does not call its destructor by - // giving up its ownership at the end of this scope. - let socket = unsafe { socket2::Socket::from_raw_fd(socket.as_raw_fd()) }; - - if let Err(error) = socket.set_send_buffer_size(size) { - warn!(message = "Failed configuring send buffer size on UDP socket.", %error); - } - - socket.into_raw_fd(); +// apply options to a socket. +pub fn set_send_buffer_size(socket: &UdpSocket, size: usize) -> std::io::Result<()> { + SockRef::from(socket).set_send_buffer_size(size) } diff --git a/src/vector_windows.rs b/src/vector_windows.rs index d8008ae6fa22d..1fcca5a9cf133 100644 --- a/src/vector_windows.rs +++ b/src/vector_windows.rs @@ -392,7 +392,7 @@ fn run_service(_arguments: Vec) -> Result<()> { process_id: None, })?; - let mut rt = app.runtime; + let rt = app.runtime; let topology = app.config.topology; rt.block_on(async move { diff --git a/tests/api.rs b/tests/api.rs index c6bbe02d9159f..d99ccb9c87474 100644 --- a/tests/api.rs +++ b/tests/api.rs @@ -1,9 +1,48 @@ +// These tests have been (inconsistently) hanging after the Tokio 1.x upgrade, most likely due to +// some interaction between the Tokio runtime and the rusty_fork library. +// For an attempt to fix these tests, see https://github.com/timberio/vector/pull/6926, which has +// been blocked on several changes that would be required to upstream crates. +/* + #[cfg(feature = "api")] #[macro_use] extern crate matches; mod support; +/// Takes a test name and a future, and uses `rusty_fork` to perform a cross-platform +/// process fork. This allows us to test functionality without conflicting with global +/// state that may have been set/mutated from previous tests +fn fork_test>(test_name: &'static str, fut: T) { + let fork_id = rusty_fork::rusty_fork_id!(); + + rusty_fork::fork( + test_name, + fork_id, + |_| {}, + |child, f| { + let status = child.wait().expect("Couldn't wait for child process"); + + // Copy all output + let mut stdout = io::stdout(); + io::copy(f, &mut stdout).expect("Couldn't write to stdout"); + + // If the test failed, panic on the parent thread + if !status.success() { + panic!("Test failed"); + } + }, + || { + // Since we are spawning the runtime from within a forked process, use one worker less + // to account for the additional process. + // This adjustment mainly serves to not overload CI workers with low resources. + let rt = runtime_constrained(std::cmp::max(1, num_cpus::get() - 1)); + rt.block_on(fut); + }, + ) + .expect("Couldn't fork test"); +} + #[cfg(all(feature = "api", feature = "vector-api-client"))] mod tests { use crate::support::{fork_test, sink, source_with_event_counter, transform}; @@ -16,6 +55,7 @@ mod tests { time::{Duration, Instant}, }; use tokio::sync::oneshot; + use tokio_stream::wrappers::IntervalStream; use url::Url; use vector::{ self, @@ -42,7 +82,7 @@ mod tests { let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); tokio::spawn(async move { let since = Instant::now(); - tokio::time::interval(Duration::from_secs(1)) + IntervalStream::new(tokio::time::interval(Duration::from_secs(1))) .take_until(shutdown_rx) .for_each(|_| async move { emit(Heartbeat { since }) }) .await @@ -134,7 +174,7 @@ mod tests { fn emit_fake_generator_events() -> oneshot::Sender<()> { let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); tokio::spawn(async move { - tokio::time::interval(Duration::from_millis(10)) + IntervalStream::new(tokio::time::interval(Duration::from_millis(10))) .take_until(shutdown_rx) .for_each(|_| async { emit(GeneratorEventProcessed) }) .await @@ -470,7 +510,7 @@ mod tests { let topology = from_str_config(conf).await; - tokio::time::delay_for(tokio::time::Duration::from_millis(500)).await; + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; let server = api::Server::start(topology.config()); let client = new_subscription_client(server.addr()).await; @@ -591,7 +631,7 @@ mod tests { }); // After a short delay, update the config to include `gen2` - tokio::time::delay_for(tokio::time::Duration::from_millis(200)).await; + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; let conf = r#" [api] @@ -681,7 +721,7 @@ mod tests { }); // After a short delay, update the config to remove `gen2` - tokio::time::delay_for(tokio::time::Duration::from_millis(200)).await; + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; // New configuration that will be reloaded conf = r#" @@ -862,7 +902,7 @@ mod tests { let server = api::Server::start(topology.config()); // Short delay to ensure logs are picked up - tokio::time::delay_for(tokio::time::Duration::from_millis(200)).await; + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; let client = make_client(server.addr()); let res = client @@ -1042,3 +1082,4 @@ mod tests { }); } } +*/ diff --git a/tests/buffering.rs b/tests/buffering.rs index f63b979d5d619..4f00bee8a4f60 100644 --- a/tests/buffering.rs +++ b/tests/buffering.rs @@ -56,7 +56,7 @@ fn test_buffering() { config.build().unwrap() }; - let mut rt = runtime(); + let rt = runtime(); let (topology, input_events) = rt.block_on(async move { let (topology, _crash) = start_topology(config, false).await; let (input_events, input_events_stream) = @@ -105,7 +105,7 @@ fn test_buffering() { config.build().unwrap() }; - let mut rt = runtime(); + let rt = runtime(); rt.block_on(async move { let (topology, _crash) = start_topology(config, false).await; diff --git a/tests/crash.rs b/tests/crash.rs index b655c3432df3a..9bf9377eac480 100644 --- a/tests/crash.rs +++ b/tests/crash.rs @@ -8,7 +8,8 @@ use std::{ pin::Pin, task::{Context, Poll}, }; -use tokio::time::{delay_for, Duration}; +use tokio::time::{sleep, Duration}; +use tokio_stream::wrappers::UnboundedReceiverStream; use vector::{ config::{self, GlobalOptions, SinkConfig, SinkContext, SourceConfig}, shutdown::ShutdownSignal, @@ -84,22 +85,22 @@ async fn test_sink_panic() { let mut output_lines = CountReceiver::receive_lines(out_addr); std::panic::set_hook(Box::new(|_| {})); // Suppress panic print on background thread - let (topology, mut crash) = start_topology(config.build().unwrap(), false).await; + let (topology, crash) = start_topology(config.build().unwrap(), false).await; // Wait for server to accept traffic wait_for_tcp(in_addr).await; - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; // Wait for output to connect output_lines.connected().await; let input_lines = random_lines(100).take(num_lines).collect::>(); send_lines(in_addr, input_lines.clone()).await.unwrap(); - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; let _ = std::panic::take_hook(); - assert!(crash.next().await.is_some()); + assert!(UnboundedReceiverStream::new(crash).next().await.is_some()); topology.stop().await; - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; let output_lines = output_lines.await; assert_eq!(num_lines, output_lines.len()); @@ -169,21 +170,21 @@ async fn test_sink_error() { let mut output_lines = CountReceiver::receive_lines(out_addr); - let (topology, mut crash) = start_topology(config.build().unwrap(), false).await; + let (topology, crash) = start_topology(config.build().unwrap(), false).await; // Wait for server to accept traffic wait_for_tcp(in_addr).await; - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; // Wait for output to connect output_lines.connected().await; let input_lines = random_lines(100).take(num_lines).collect::>(); send_lines(in_addr, input_lines.clone()).await.unwrap(); - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; - assert!(crash.next().await.is_some()); + assert!(UnboundedReceiverStream::new(crash).next().await.is_some()); topology.stop().await; - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; let output_lines = output_lines.await; assert_eq!(num_lines, output_lines.len()); @@ -236,21 +237,21 @@ async fn test_source_error() { let mut output_lines = CountReceiver::receive_lines(out_addr); - let (topology, mut crash) = start_topology(config.build().unwrap(), false).await; + let (topology, crash) = start_topology(config.build().unwrap(), false).await; // Wait for server to accept traffic wait_for_tcp(in_addr).await; - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; // Wait for output to connect output_lines.connected().await; let input_lines = random_lines(100).take(num_lines).collect::>(); send_lines(in_addr, input_lines.clone()).await.unwrap(); - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; - assert!(crash.next().await.is_some()); + assert!(UnboundedReceiverStream::new(crash).next().await.is_some()); topology.stop().await; - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; let output_lines = output_lines.await; assert_eq!(num_lines, output_lines.len()); @@ -304,22 +305,22 @@ async fn test_source_panic() { let mut output_lines = CountReceiver::receive_lines(out_addr); std::panic::set_hook(Box::new(|_| {})); // Suppress panic print on background thread - let (topology, mut crash) = start_topology(config.build().unwrap(), false).await; + let (topology, crash) = start_topology(config.build().unwrap(), false).await; // Wait for server to accept traffic wait_for_tcp(in_addr).await; - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; // Wait for output to connect output_lines.connected().await; let input_lines = random_lines(100).take(num_lines).collect::>(); send_lines(in_addr, input_lines.clone()).await.unwrap(); - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; let _ = std::panic::take_hook(); - assert!(crash.next().await.is_some()); + assert!(UnboundedReceiverStream::new(crash).next().await.is_some()); topology.stop().await; - delay_for(Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; let output_lines = output_lines.await; assert_eq!(num_lines, output_lines.len()); diff --git a/tests/data/adaptive-concurrency/defers-at-high-concurrency.toml b/tests/data/adaptive-concurrency/defers-at-high-concurrency.toml index 84363ad26ccca..843e230b0a320 100644 --- a/tests/data/adaptive-concurrency/defers-at-high-concurrency.toml +++ b/tests/data/adaptive-concurrency/defers-at-high-concurrency.toml @@ -15,8 +15,8 @@ mode = [4, 6] mean = [3.5, 5.0] [stats.rate] -max = [55, 55] -mean = [42, 43] +max = [52, 52] +mean = [43, 44] [controller.in_flight] max = [5, 7] diff --git a/tests/data/adaptive-concurrency/defers-at-high-rate.toml b/tests/data/adaptive-concurrency/defers-at-high-rate.toml index c7de1191b2104..9ca3b8f4fca48 100644 --- a/tests/data/adaptive-concurrency/defers-at-high-rate.toml +++ b/tests/data/adaptive-concurrency/defers-at-high-rate.toml @@ -9,20 +9,20 @@ rate.action = "defer" # drop down repeatedly. [stats.in_flight] -max = [15, 15] +max = [16, 16] mean = [8.0, 9.0] [stats.rate] -max = [81, 110] -mean = [73, 75] +max = [90, 120] +mean = [72, 75] [controller.in_flight] -max = [15, 15] -mean = [8.0, 9.0] +max = [16, 16] +mean = [8.0, 10.0] [controller.concurrency_limit] -max = [15, 15] -mean = [8.0, 9.0] +max = [16, 16] +mean = [8.0, 10.0] [controller.observed_rtt] min = [0.100, 0.102] diff --git a/tests/data/adaptive-concurrency/fixed-concurrency-jitter.toml b/tests/data/adaptive-concurrency/fixed-concurrency-jitter.toml index 546933465b71c..b42bda7243eeb 100644 --- a/tests/data/adaptive-concurrency/fixed-concurrency-jitter.toml +++ b/tests/data/adaptive-concurrency/fixed-concurrency-jitter.toml @@ -13,7 +13,7 @@ mode = [10, 10] mean = [8.0, 10.0] [stats.rate] -max = [65, 82] +max = [65, 84] mean = [55, 66] [controller.in_flight] diff --git a/tests/data/adaptive-concurrency/jittery-link-small.toml b/tests/data/adaptive-concurrency/jittery-link-small.toml index 83ca5fd8f2148..cf27404b6cab4 100644 --- a/tests/data/adaptive-concurrency/jittery-link-small.toml +++ b/tests/data/adaptive-concurrency/jittery-link-small.toml @@ -7,7 +7,7 @@ jitter = 0.1 # will typically reach high values of requests in flight. [stats.in_flight] -max = [20, 37] +max = [20, 40] mean = [8.0, 20.0] [stats.rate] @@ -15,11 +15,11 @@ max = [175, 350] mean = [70, 150] [controller.in_flight] -max = [20, 37] +max = [20, 40] mean = [8.0, 20.0] [controller.concurrency_limit] -max = [20, 37] +max = [20, 40] mean = [9.0, 20.0] [controller.observed_rtt] diff --git a/tests/data/wasm/protobuf/Cargo.toml b/tests/data/wasm/protobuf/Cargo.toml index 425ee55f0eb09..12baf55415b69 100644 --- a/tests/data/wasm/protobuf/Cargo.toml +++ b/tests/data/wasm/protobuf/Cargo.toml @@ -10,7 +10,7 @@ build = "build.rs" crate-type = ["cdylib"] [dependencies] -prost = "0.6" +prost = "0.7" serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } vector-wasm = { version = "0.1", path = "../../../../lib/vector-wasm"} @@ -20,4 +20,4 @@ anyhow = "1.0.28" [build-dependencies] prost-build = "0.6" -[workspace] \ No newline at end of file +[workspace] diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 4d8766904881a..e269c38f2361c 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -6,6 +6,7 @@ use async_trait::async_trait; use futures::{ + channel::mpsc, future, stream::{self, BoxStream}, task::Poll, @@ -15,7 +16,7 @@ use serde::{Deserialize, Serialize}; use snafu::Snafu; use std::{ fs::{create_dir, OpenOptions}, - io::{self, Write}, + io::Write, path::PathBuf, pin::Pin, sync::{ @@ -24,7 +25,6 @@ use std::{ }, task::Context, }; -use tokio::sync::mpsc; use tracing::{error, info}; use vector::{ buffers::Acker, @@ -36,7 +36,7 @@ use vector::{ shutdown::ShutdownSignal, sinks::{util::StreamSink, Healthcheck, VectorSink}, sources::Source, - test_util::{runtime, temp_dir, temp_file}, + test_util::{temp_dir, temp_file}, transforms::{FunctionTransform, Transform}, Event, Pipeline, }; @@ -402,33 +402,3 @@ impl Sink for DeadSink { Err("never ready") } } - -/// Takes a test name and a future, and uses `rusty_fork` to perform a cross-platform -/// process fork. This allows us to test functionality without conflicting with global -/// state that may have been set/mutated from previous tests -pub fn fork_test>(test_name: &'static str, fut: T) { - let fork_id = rusty_fork::rusty_fork_id!(); - - rusty_fork::fork( - test_name, - fork_id, - |_| {}, - |child, f| { - let status = child.wait().expect("Couldn't wait for child process"); - - // Copy all output - let mut stdout = io::stdout(); - io::copy(f, &mut stdout).expect("Couldn't write to stdout"); - - // If the test failed, panic on the parent thread - if !status.success() { - panic!("Test failed"); - } - }, - || { - let mut rt = runtime(); - rt.block_on(fut); - }, - ) - .expect("Couldn't fork test"); -} diff --git a/tests/syslog.rs b/tests/syslog.rs index d0b810e99c468..5cb6086951cba 100644 --- a/tests/syslog.rs +++ b/tests/syslog.rs @@ -7,6 +7,8 @@ use serde_json::Value; use sinks::socket::{self, SocketSinkConfig}; use sinks::util::{encoding::EncodingConfig, tcp::TcpSinkConfig, Encoding}; use std::{collections::HashMap, fmt, str::FromStr}; +#[cfg(unix)] +use tokio::io::AsyncWriteExt; use tokio_util::codec::BytesCodec; use vector::{ config, sinks, @@ -110,10 +112,10 @@ async fn test_unix_stream_syslog() { sink.send_all(&mut lines).await.unwrap(); let stream = sink.get_mut(); - stream.shutdown(std::net::Shutdown::Both).unwrap(); + stream.shutdown().await.unwrap(); // Otherwise some lines will be lost - tokio::time::delay_for(std::time::Duration::from_millis(1000)).await; + tokio::time::sleep(std::time::Duration::from_millis(1000)).await; // Shut down server topology.stop().await; diff --git a/tests/tcp.rs b/tests/tcp.rs index 2ad323deb5479..c02f7c9ff3733 100644 --- a/tests/tcp.rs +++ b/tests/tcp.rs @@ -161,7 +161,7 @@ async fn fork() { // are needed to finish processing all the events before // sources are forcefully shutted down. // Although that's still not a guarantee. -#[tokio::test(core_threads = 2)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn merge_and_fork() { trace_init(); diff --git a/tests/topology.rs b/tests/topology.rs index 4265297aed7a9..7d603fb833973 100644 --- a/tests/topology.rs +++ b/tests/topology.rs @@ -10,8 +10,7 @@ use std::{ Arc, }, }; - -use tokio::time::{delay_for, Duration}; +use tokio::time::{sleep, Duration}; use vector::{config::Config, event::Event, test_util::start_topology, topology}; fn basic_config() -> Config { @@ -64,7 +63,7 @@ async fn topology_shutdown_while_active() { // Wait until at least 100 events have been seen by the source so we know the pump is running // and pushing events through the pipeline. while source_event_total.load(Ordering::SeqCst) < 100 { - delay_for(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; } // Now shut down the RunningTopology while Events are still being processed. @@ -116,7 +115,7 @@ async fn topology_source_and_sink() { async fn topology_multiple_sources() { let (mut in1, source1) = source(); let (mut in2, source2) = source(); - let (out1, sink1) = sink(10); + let (mut out1, sink1) = sink(10); let mut config = Config::builder(); config.add_source("in1", source1); @@ -130,11 +129,11 @@ async fn topology_multiple_sources() { in1.send(event1.clone()).await.unwrap(); - let (out_event1, out1) = out1.into_future().await; + let out_event1 = out1.next().await; in2.send(event2.clone()).await.unwrap(); - let (out_event2, _out1) = out1.into_future().await; + let out_event2 = out1.next().await; topology.stop().await; @@ -452,7 +451,7 @@ async fn topology_swap_transform_is_atomic() { config.add_sink("out1", &["t1"], sink1); let (mut topology, _crash) = start_topology(config.build().unwrap(), false).await; - delay_for(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; let transform1v2 = transform(" replaced", 0.0); @@ -465,7 +464,7 @@ async fn topology_swap_transform_is_atomic() { .reload_config_and_respawn(config.build().unwrap()) .await .unwrap()); - delay_for(Duration::from_millis(10)).await; + sleep(Duration::from_millis(10)).await; run_control.store(false, Ordering::Release); h_in.await.unwrap();