diff --git a/COPYRIGHT b/COPYRIGHT index 522b0c779..7c192ffa5 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -14,6 +14,6 @@ operating system images. =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= -Contains modified aws-smithy-experimental file(s) [hyper_1_0.rs] from -https://github.com/smithy-lang/smithy-rs/tree/release-2024-10-09. +Contains modified aws-smithy-http-client file(s) [rustls_provider.rs] from +https://github.com/awslabs/aws-sdk-rust/tree/release-2026-07-10. Licensed under the Apache-2.0 License. diff --git a/packages/os/Cargo.toml b/packages/os/Cargo.toml index 50645364e..c922465c6 100644 --- a/packages/os/Cargo.toml +++ b/packages/os/Cargo.toml @@ -10,7 +10,6 @@ source-groups = [ "api", "constants", "generate-readme", - "aws-smithy-experimental", "bottlerocket-release", "metricdog", "parse-datetime", diff --git a/sources/Cargo.lock b/sources/Cargo.lock index d100af3ad..abd60b1ea 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -997,9 +997,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.14" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" dependencies = [ "futures-util", "pin-project-lite", @@ -1038,30 +1038,6 @@ dependencies = [ "crc32fast", ] -[[package]] -name = "aws-smithy-experimental" -version = "0.1.4" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "h2 0.4.15", - "headers", - "http 1.4.2", - "hyper 1.10.1", - "hyper-http-proxy", - "hyper-rustls 0.27.9", - "hyper-util", - "once_cell", - "pin-project-lite", - "rustls 0.23.40", - "tokio", - "tower 0.4.13", - "tracing", - "url", -] - [[package]] name = "aws-smithy-http" version = "0.63.6" @@ -1089,11 +1065,40 @@ name = "aws-smithy-http-client" version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.15", + "http 0.2.12", + "http 1.4.2", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.10.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.9", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.40", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.2.0" dependencies = [ "aws-smithy-async", "aws-smithy-protocol-test", "aws-smithy-runtime-api", "aws-smithy-types", + "base64", "bytes", "h2 0.3.27", "h2 0.4.15", @@ -1101,6 +1106,7 @@ dependencies = [ "http 1.4.2", "http-body 0.4.6", "http-body 1.0.1", + "http-body-util", "hyper 0.14.32", "hyper 1.10.1", "hyper-rustls 0.24.2", @@ -1110,13 +1116,18 @@ dependencies = [ "pin-project-lite", "rustls 0.21.12", "rustls 0.23.40", - "rustls-native-certs 0.8.4", + "rustls-native-certs", + "rustls-pemfile", "rustls-pki-types", + "s2n-tls", + "s2n-tls-hyper", + "s2n-tls-tokio", "serde", "serde_json", + "serial_test", "tokio", "tokio-rustls 0.26.4", - "tower 0.5.3", + "tower", "tracing", ] @@ -1142,9 +1153,9 @@ dependencies = [ [[package]] name = "aws-smithy-protocol-test" -version = "0.63.14" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b227aa94af99a8e5ee52551cc7e3ee30a217019ef99207b6f0b7a1527685941" +checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a" dependencies = [ "assert-json-diff", "aws-smithy-runtime-api", @@ -1177,7 +1188,7 @@ checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" dependencies = [ "aws-smithy-async", "aws-smithy-http", - "aws-smithy-http-client", + "aws-smithy-http-client 1.1.13", "aws-smithy-observability", "aws-smithy-runtime-api", "aws-smithy-schema", @@ -1193,14 +1204,13 @@ dependencies = [ "pin-utils", "tokio", "tracing", - "tracing-subscriber", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.12.3" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -1216,9 +1226,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api-macros" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" dependencies = [ "proc-macro2", "quote", @@ -1238,9 +1248,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.5.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" +checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" dependencies = [ "base64-simd", "bytes", @@ -1957,7 +1967,7 @@ dependencies = [ "aws-config", "aws-lc-rs", "aws-sdk-cloudformation", - "aws-smithy-experimental", + "aws-smithy-http-client 1.2.0", "aws-types", "generate-readme", "imdsclient", @@ -3180,30 +3190,6 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -[[package]] -name = "headers" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" -dependencies = [ - "base64", - "bytes", - "headers-core", - "http 1.4.2", - "httpdate", - "mime", - "sha1 0.10.6", -] - -[[package]] -name = "headers-core" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" -dependencies = [ - "http 1.4.2", -] - [[package]] name = "heck" version = "0.5.0" @@ -3417,6 +3403,7 @@ dependencies = [ "http 1.4.2", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -3424,26 +3411,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-http-proxy" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d06dbdfbacf34d996c6fb540a71a684a7aae9056c71951163af8a8a4c07b9a4" -dependencies = [ - "bytes", - "futures-util", - "headers", - "http 1.4.2", - "hyper 1.10.1", - "hyper-rustls 0.27.9", - "hyper-util", - "pin-project-lite", - "rustls-native-certs 0.7.3", - "tokio", - "tokio-rustls 0.26.4", - "tower-service", -] - [[package]] name = "hyper-rustls" version = "0.24.2" @@ -3469,7 +3436,7 @@ dependencies = [ "hyper 1.10.1", "hyper-util", "rustls 0.23.40", - "rustls-native-certs 0.8.4", + "rustls-native-certs", "tokio", "tokio-rustls 0.26.4", "tower-service", @@ -3506,9 +3473,12 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.6.4", + "system-configuration", "tokio", + "tower-layer", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -4010,15 +3980,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata 0.4.14", -] - [[package]] name = "md-5" version = "0.11.0" @@ -4205,15 +4166,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "num" version = "0.4.3" @@ -4364,12 +4316,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - [[package]] name = "openssl-probe" version = "0.2.1" @@ -4598,7 +4544,7 @@ dependencies = [ "aws-lc-rs", "aws-sdk-ec2", "aws-sdk-eks", - "aws-smithy-experimental", + "aws-smithy-http-client 1.2.0", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", @@ -5030,7 +4976,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.40", - "rustls-native-certs 0.8.4", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -5038,7 +4984,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls 0.26.4", - "tower 0.5.3", + "tower", "tower-http", "tower-service", "url", @@ -5074,7 +5020,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tokio-util", - "tower 0.5.3", + "tower", "tower-http", "tower-service", "url", @@ -5272,29 +5218,16 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" -dependencies = [ - "openssl-probe 0.1.6", - "rustls-pemfile", - "rustls-pki-types", - "schannel", - "security-framework 2.11.1", -] - [[package]] name = "rustls-native-certs" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ - "openssl-probe 0.2.1", + "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.7.0", + "security-framework", ] [[package]] @@ -5328,10 +5261,10 @@ dependencies = [ "log", "once_cell", "rustls 0.23.40", - "rustls-native-certs 0.8.4", + "rustls-native-certs", "rustls-platform-verifier-android", "rustls-webpki 0.103.13", - "security-framework 3.7.0", + "security-framework", "security-framework-sys", "webpki-root-certs", "windows-sys 0.61.2", @@ -5377,6 +5310,57 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "s2n-tls" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ac76702594ee0cea5d029b6df75830bd8b0a83eeca496f341f556afb456adf" +dependencies = [ + "errno", + "hex", + "libc", + "pin-project-lite", + "s2n-tls-sys", +] + +[[package]] +name = "s2n-tls-hyper" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f48b9922eed6521a7f9d5b444bcc36a5df988161b67579112cc5377ca021ed78" +dependencies = [ + "http 1.4.2", + "hyper 1.10.1", + "hyper-util", + "s2n-tls", + "s2n-tls-tokio", + "tower-service", +] + +[[package]] +name = "s2n-tls-sys" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47aa68e32808cd3f0a4c15b306ca77fed4379b4b8746031bcaea9728a2f0bda5" +dependencies = [ + "aws-lc-rs", + "cc", + "libc", +] + +[[package]] +name = "s2n-tls-tokio" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ccc16dc6d5307e9ad2ab6a6eee8deeb5cbb6f8b5b8820610639f2acf8767a2f" +dependencies = [ + "errno", + "libc", + "pin-project-lite", + "s2n-tls", + "tokio", +] + [[package]] name = "same-file" version = "1.0.6" @@ -5484,19 +5468,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.13.0", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - [[package]] name = "security-framework" version = "3.7.0" @@ -5646,6 +5617,31 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "serial_test" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "699f4197115b8a7e7ff19c9a315a4bd6fffec26cc4626ef45ecaea389e081c6d" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "settings-committer" version = "0.1.0" @@ -6038,15 +6034,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - [[package]] name = "shell-words" version = "1.1.1" @@ -6350,6 +6337,27 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "systemd-derive" version = "0.1.0" @@ -6536,15 +6544,6 @@ dependencies = [ "syn 2.0.118", ] -[[package]] -name = "thread_local" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" -dependencies = [ - "cfg-if", -] - [[package]] name = "time" version = "0.3.49" @@ -6818,17 +6817,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tower" version = "0.5.3" @@ -6856,7 +6844,7 @@ dependencies = [ "http 1.4.2", "http-body 1.0.1", "pin-project-lite", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "url", @@ -6904,49 +6892,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-serde" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" -dependencies = [ - "serde", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata 0.4.14", - "serde", - "serde_json", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", - "tracing-serde", ] [[package]] @@ -7182,12 +7127,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - [[package]] name = "version_check" version = "0.9.5" @@ -7476,6 +7415,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index f5157473f..7b43dddc0 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -21,7 +21,6 @@ members = [ "api/shibaken", "api/prairiedog", "api/simple-settings-plugin", - "aws-smithy-experimental", "bloodhound", @@ -90,7 +89,7 @@ members = [ [workspace.dependencies] apiclient = { version = "0.1", path = "api/apiclient", default-features = false } -aws-smithy-experimental = { version = "0.1", path = "aws-smithy-experimental" } +aws-smithy-http-client = { version = "1", path = "aws-smithy-http-client" } block-party = { version = "0.1", path = "updater/block-party" } bottlerocket-image-features = { version = "0.1", path = "bottlerocket-image-features" } bottlerocket-release = { version = "0.1", path = "bottlerocket-release" } @@ -160,9 +159,16 @@ hex = "0.4" hex-literal = "0.4" hkdf = { version = "0.12", default-features = false } http = "0.2" +# http-02x is an alias for the same http 0.2 crate above. It exists because +# aws-smithy-http-client imports http under the renamed name "http-02x" to +# coexist with http 1.x. This does not introduce a duplicate both resolve +# to the same http 0.2.x in Cargo.lock. +http-02x = { package = "http", version = "0.2" } +http-body-util = "0.1" httparse = "1" httptest = "0.15" hyper = { version = "1", default-features = false } +hyper-0-14 = { package = "hyper", version = "0.14", default-features = false } hyper-http-proxy = "~1.0" hyper-rustls = { version = "0.27", default-features = false } hyper-util = { version = "0.1", default-features = false } @@ -194,7 +200,8 @@ quote = "1" rand = { version = "0.8", default-features = false } regex = "1" reqwest = { version = "0.12", default-features = false } -rustls = "0.23" +rustls = { version = "0.23", default-features = false } +rustls-pki-types = { version = "1", features = ["std"] } semver = "1" serde = "1" serde-xml-rs = "0.6" @@ -214,12 +221,13 @@ tempfile = "3" test-case = "3" tokio = { version = "~1.51", default-features = false } # LTS tokio-retry = "0.3" +tokio-rustls = { version = "0.26", default-features = false } tokio-test = "0.4" tokio-tungstenite = { version = "0.20", default-features = false } tokio-util = "0.7" toml = "0.8" tough = "0.24" -tower = "0.4" +tower = "0.5" tracing = "0.1" typed-path = "0.9" unindent = "0.2" diff --git a/sources/api/apiclient/Cargo.toml b/sources/api/apiclient/Cargo.toml index 5e7e37a62..72f15e5dc 100644 --- a/sources/api/apiclient/Cargo.toml +++ b/sources/api/apiclient/Cargo.toml @@ -33,7 +33,7 @@ futures-channel.workspace = true http.workspace = true httparse.workspace = true # apiclient exec requires an older version of hyper -hyper = { version = "0.14", features = ["client", "http1", "http2", "tcp"] } +hyper-0-14 = { workspace = true, features = ["client", "http1", "http2", "tcp"] } hyper-unix-connector.workspace = true libc.workspace = true log.workspace = true diff --git a/sources/api/apiclient/src/lib.rs b/sources/api/apiclient/src/lib.rs index 1c8b27a72..6ebc2a85c 100644 --- a/sources/api/apiclient/src/lib.rs +++ b/sources/api/apiclient/src/lib.rs @@ -13,6 +13,8 @@ // of hyper, but it lacks Unix-domain socket support: // https://github.com/seanmonstar/reqwest/issues/39 +extern crate hyper_0_14 as hyper; + use hyper::{body, header, Body, Client, Request}; use hyper_unix_connector::{UnixClient, Uri}; use rand::{distributions::Alphanumeric, thread_rng, Rng}; diff --git a/sources/api/pluto/Cargo.toml b/sources/api/pluto/Cargo.toml index 614700dd6..c71f7c062 100644 --- a/sources/api/pluto/Cargo.toml +++ b/sources/api/pluto/Cargo.toml @@ -10,7 +10,7 @@ build = "build.rs" exclude = ["README.md"] [features] -fips = ["aws-lc-rs/fips", "aws-smithy-experimental/crypto-aws-lc-fips", "rustls/fips"] +fips = ["aws-lc-rs/fips", "aws-smithy-http-client/rustls-aws-lc-fips", "rustls/fips"] [dependencies] argh.workspace = true @@ -18,7 +18,7 @@ aws-config.workspace = true aws-lc-rs = { workspace = true, features = ["bindgen"] } aws-sdk-eks.workspace = true aws-sdk-ec2.workspace = true -aws-smithy-experimental = {workspace = true, features = ["crypto-aws-lc"]} +aws-smithy-http-client = { workspace = true, features = ["rustls-aws-lc"] } aws-smithy-runtime-api.workspace = true aws-smithy-types.workspace = true aws-types.workspace = true diff --git a/sources/api/pluto/src/ec2.rs b/sources/api/pluto/src/ec2.rs index ba32ae3e5..93dce35f0 100644 --- a/sources/api/pluto/src/ec2.rs +++ b/sources/api/pluto/src/ec2.rs @@ -1,6 +1,6 @@ use crate::aws::sdk_config; use crate::PROVIDER; -use aws_smithy_experimental::hyper_1_0::HyperClientBuilder; +use aws_smithy_http_client::{proxy::ProxyConfig, tls, Builder as HttpClientBuilder, Connector}; use aws_smithy_types::error::display::DisplayErrorContext; use aws_smithy_types::error::metadata::ProvideErrorMetadata; use snafu::{OptionExt, ResultExt, Snafu}; @@ -37,6 +37,11 @@ pub(super) enum Error { #[snafu(display("Missing field '{}' in EC2 response", field))] Missing { field: &'static str }, + + #[snafu(display("Invalid proxy URL: {}", source))] + ProxyConfig { + source: aws_smithy_http_client::proxy::ProxyError, + }, } type Result = std::result::Result; @@ -113,12 +118,21 @@ where { let http_client = if let Some(https_proxy) = https_proxy { let https_proxy = https_proxy.as_ref().to_string(); - HyperClientBuilder::new() - .crypto_mode(PROVIDER) - .build_with_proxy(https_proxy, no_proxy) + let mut proxy = ProxyConfig::https(&https_proxy).context(ProxyConfigSnafu)?; + if let Some(no_proxy) = no_proxy { + let no_proxy_str: Vec<&str> = no_proxy.iter().map(|s| s.as_ref()).collect(); + proxy = proxy.no_proxy(no_proxy_str.join(",")); + } + HttpClientBuilder::new().build_with_connector_fn(move |settings, _runtime_components| { + let mut builder = Connector::builder() + .proxy_config(proxy.clone()) + .tls_provider(tls::Provider::Rustls(PROVIDER.clone())); + builder.set_connector_settings(settings.cloned()); + builder.build() + }) } else { - HyperClientBuilder::new() - .crypto_mode(PROVIDER) + HttpClientBuilder::new() + .tls_provider(tls::Provider::Rustls(PROVIDER.clone())) .build_https() }; let ec2_config = aws_sdk_ec2::config::Builder::from(&config) diff --git a/sources/api/pluto/src/eks.rs b/sources/api/pluto/src/eks.rs index dc36f17a5..a36c337f0 100644 --- a/sources/api/pluto/src/eks.rs +++ b/sources/api/pluto/src/eks.rs @@ -1,7 +1,7 @@ use crate::aws::sdk_config; use crate::PROVIDER; use aws_sdk_eks::types::KubernetesNetworkConfigResponse; -use aws_smithy_experimental::hyper_1_0::HyperClientBuilder; +use aws_smithy_http_client::{proxy::ProxyConfig, tls, Builder as HttpClientBuilder, Connector}; use aws_smithy_types::error::display::DisplayErrorContext; use aws_smithy_types::error::metadata::ProvideErrorMetadata; use snafu::{OptionExt, ResultExt, Snafu}; @@ -34,6 +34,11 @@ pub(super) enum Error { #[snafu(display("Missing field '{}' in EKS response", field))] Missing { field: &'static str }, + + #[snafu(display("Invalid proxy URL: {}", source))] + ProxyConfig { + source: aws_smithy_http_client::proxy::ProxyError, + }, } type Result = std::result::Result; @@ -95,12 +100,21 @@ where { let http_client = if let Some(https_proxy) = https_proxy { let https_proxy = https_proxy.as_ref().to_string(); - HyperClientBuilder::new() - .crypto_mode(PROVIDER) - .build_with_proxy(https_proxy, no_proxy) + let mut proxy = ProxyConfig::https(&https_proxy).context(ProxyConfigSnafu)?; + if let Some(no_proxy) = no_proxy { + let no_proxy_str: Vec<&str> = no_proxy.iter().map(|s| s.as_ref()).collect(); + proxy = proxy.no_proxy(no_proxy_str.join(",")); + } + HttpClientBuilder::new().build_with_connector_fn(move |settings, _runtime_components| { + let mut builder = Connector::builder() + .proxy_config(proxy.clone()) + .tls_provider(tls::Provider::Rustls(PROVIDER.clone())); + builder.set_connector_settings(settings.cloned()); + builder.build() + }) } else { - HyperClientBuilder::new() - .crypto_mode(PROVIDER) + HttpClientBuilder::new() + .tls_provider(tls::Provider::Rustls(PROVIDER.clone())) .build_https() }; let eks_config = aws_sdk_eks::config::Builder::from(&config) diff --git a/sources/api/pluto/src/main.rs b/sources/api/pluto/src/main.rs index 92e986c09..e97186a91 100644 --- a/sources/api/pluto/src/main.rs +++ b/sources/api/pluto/src/main.rs @@ -29,7 +29,7 @@ mod eks; use api::{settings_view_get, settings_view_set, SettingsViewDelta}; use argh::FromArgs; use aws_sdk_eks::types::IpFamily; -use aws_smithy_experimental::hyper_1_0::CryptoMode; +use aws_smithy_http_client::tls::rustls_provider::CryptoMode; use base64::Engine; use bottlerocket_modeled_types::{KubernetesClusterDnsIp, KubernetesHostnameOverrideSource}; use imdsclient::ImdsClient; diff --git a/sources/aws-smithy-experimental/Cargo.toml b/sources/aws-smithy-experimental/Cargo.toml deleted file mode 100644 index 2d504606e..000000000 --- a/sources/aws-smithy-experimental/Cargo.toml +++ /dev/null @@ -1,41 +0,0 @@ -[package] -name = "aws-smithy-experimental" -version = "0.1.4" -authors = ["AWS Rust SDK Team "] -description = "Experiments for the smithy-rs ecosystem" -edition = "2021" -license = "Apache-2.0" -publish = false - -[features] -crypto-ring = ["rustls/ring"] -crypto-aws-lc = ["rustls/aws_lc_rs"] -crypto-aws-lc-fips = ["rustls/fips"] - -[lints.rust] -unexpected_cfgs = { level = "warn", check-cfg = ['cfg(crypto_unstable)'] } - -[dependencies] -aws-smithy-types = { workspace = true, features = ["http-body-1-x"] } -aws-smithy-runtime-api = { workspace = true, features = ["client", "http-1x"] } -aws-smithy-runtime = { workspace = true, features = ["client"] } -aws-smithy-async.workspace = true -h2.workspace = true -headers.workspace = true -hyper-util.workspace = true -once_cell.workspace = true -pin-project-lite.workspace = true -rustls.workspace = true -tracing.workspace = true -tokio.workspace = true -tower.workspace = true -hyper = { workspace = true, features = ["client", "http1", "http2"] } -hyper-http-proxy.workspace = true -hyper-rustls = { workspace = true, features = ["http2", "http1", "native-tokio", "tls12"], default-features = false } -http = "1" -url.workspace = true - -[dev-dependencies] -aws-smithy-async = { workspace = true, features = ["rt-tokio", "test-util"] } -aws-smithy-runtime = { workspace = true, features = ["client", "test-util", "connector-hyper-0-14-x"]} -tokio = { workspace = true, features = ["full", "test-util"]} diff --git a/sources/aws-smithy-experimental/README.md b/sources/aws-smithy-experimental/README.md deleted file mode 100644 index a537e7ee4..000000000 --- a/sources/aws-smithy-experimental/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# aws-smithy-experimental - -See [`aws-smithy-experimental`](https://github.com/smithy-lang/smithy-rs/tree/42751e5dbf4d51c06c085e4193bf013a7333a6f5/rust-runtime/aws-smithy-experimental) - - -## Changes -- Remove `examples` and `tests` directories -- Remove `external-types.toml` -- Remove `examples` section in `Cargo.toml` -- Remove `package.metadata` section in `Cargo.toml` -- Remove `package.repository` in `Cargo.toml` -- Prevent crate from being published with `publish = false` in `Cargo.toml` -- Use workspace dependencies wherever possible -- Add linting rule to warn for missing `crypto_unstable` flag diff --git a/sources/aws-smithy-experimental/src/lib.rs b/sources/aws-smithy-experimental/src/lib.rs deleted file mode 100644 index fc0bf781e..000000000 --- a/sources/aws-smithy-experimental/src/lib.rs +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -/* Automatically managed default lints */ -#![cfg_attr(docsrs, feature(doc_auto_cfg))] -/* End of automatically managed default lints */ - -pub mod hyper_1_0; diff --git a/sources/aws-smithy-http-client/Cargo.toml b/sources/aws-smithy-http-client/Cargo.toml new file mode 100644 index 000000000..f7fcb9e07 --- /dev/null +++ b/sources/aws-smithy-http-client/Cargo.toml @@ -0,0 +1,225 @@ +# Code generated by software.amazon.smithy.rust.codegen.smithy-rs. DO NOT EDIT. +[[example]] +name = "client-ring" +required-features = ["rustls-ring"] +doc-scrape-examples = true + +[[example]] +name = "client-aws-lc" +required-features = ["rustls-aws-lc", "rustls-aws-lc-fips"] +doc-scrape-examples = true + +[[example]] +name = "client-s2n-tls" +required-features = ["s2n-tls"] +doc-scrape-examples = true + +[[example]] +name = "custom-dns" +required-features = ["rustls-ring"] +doc-scrape-examples = true + +[package] +name = "aws-smithy-http-client" +authors = ["AWS Rust SDK Team "] +description = "HTTP client abstractions for generated smithy clients" +version = "1.2.0" +license = "Apache-2.0" +edition = "2021" +repository = "https://github.com/smithy-lang/smithy-rs" +rust-version = "1.94.1" +[package.metadata.smithy-rs-release-tooling] +stable = true +[package.metadata.docs.rs] +all-features = false +features = ["default-client ", "wire-mock", "test-util", "rustls-ring", "rustls-aws-lc"] +targets = ["x86_64-unknown-linux-gnu"] +cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"] +rustdoc-args = ["--cfg", "docsrs"] + +[features] +hyper-014 = ["aws-smithy-runtime-api/http-02x", "aws-smithy-types/http-body-0-4-x", "dep:http-02x", "dep:http-body-04x", "dep:hyper-0-14", "dep:h2-0-3"] +default-client = ["aws-smithy-runtime-api/http-1x", "aws-smithy-types/http-body-1-x", "dep:hyper", "dep:hyper-util", "hyper-util?/client-legacy", "hyper-util?/client-proxy", "dep:http-1x", "dep:tower", "dep:rustls-pki-types", "dep:rustls-native-certs"] +wire-mock = ["test-util", "default-client", "hyper-util?/server", "hyper-util?/server-auto", "hyper-util?/service", "hyper-util?/server-graceful", "tokio/macros", "dep:http-body-util"] +test-util = ["dep:aws-smithy-protocol-test", "dep:serde", "dep:serde_json", "dep:indexmap", "dep:bytes", "dep:http-1x", "aws-smithy-runtime-api/http-1x", "dep:http-body-1x", "aws-smithy-types/http-body-1-x", "tokio/rt"] +legacy-test-util = ["test-util", "dep:http-02x", "aws-smithy-runtime-api/http-02x", "aws-smithy-types/http-body-0-4-x"] +legacy-rustls-ring = ["dep:legacy-hyper-rustls", "dep:legacy-rustls", "dep:rustls-native-certs", "hyper-014"] +__rustls = ["dep:rustls", "dep:hyper-rustls", "dep:tokio-rustls", "default-client"] +rustls-ring = ["__rustls", "rustls?/ring"] +rustls-aws-lc = ["__rustls", "rustls?/aws_lc_rs", "rustls?/prefer-post-quantum"] +rustls-aws-lc-fips = ["__rustls", "rustls?/fips", "rustls?/prefer-post-quantum"] +s2n-tls = ["dep:s2n-tls", "dep:s2n-tls-hyper", "dep:s2n-tls-tokio", "default-client"] + +[dependencies] +pin-project-lite = "0.2.14" +tracing.workspace = true + +[dependencies.aws-smithy-async] +workspace = true + +[dependencies.aws-smithy-runtime-api] +workspace = true +features = ["client"] + +[dependencies.aws-smithy-types] +workspace = true + +[dependencies.aws-smithy-protocol-test] +optional = true +version = "0.64.0" + +[dependencies.h2] +version = "0.4.11" +default-features = false + +[dependencies.tokio] +workspace = true +features = [] + +[dependencies.hyper] +workspace = true +features = ["client", "http1", "http2"] +optional = true + +[dependencies.hyper-util] +workspace = true +features = ["http1", "http2"] +optional = true + +[dependencies.http-1x] +package = "http" +version = "1.3.1" +optional = true + +[dependencies.http-body-1x] +package = "http-body" +version = "1.0.1" +optional = true + +[dependencies.hyper-rustls] +version = "0.27" +features = ["http2", "http1", "native-tokio", "tls12"] +default-features = false +optional = true + +[dependencies.rustls] +workspace = true +default-features = false +optional = true + +[dependencies.tokio-rustls] +workspace = true +optional = true + +[dependencies.s2n-tls-hyper] +version = "0.1.0" +optional = true + +[dependencies.s2n-tls] +version = "0.3.33" +optional = true + +[dependencies.s2n-tls-tokio] +version = "0.3.33" +optional = true + +[dependencies.tower] +workspace = true +optional = true + +[dependencies.rustls-pki-types] +workspace = true +optional = true + +[dependencies.rustls-native-certs] +version = "0.8.1" +optional = true + +[dependencies.http-02x] +workspace = true +optional = true + +[dependencies.http-body-04x] +package = "http-body" +version = "0.4.6" +optional = true + +[dependencies.hyper-0-14] +workspace = true +features = ["client", "http1", "http2", "tcp", "stream"] +optional = true + +[dependencies.legacy-hyper-rustls] +package = "hyper-rustls" +version = "0.24.2" +default-features = false +features = ["http1", "tls12", "logging", "acceptor", "tokio-runtime", "http2"] +optional = true + +[dependencies.legacy-rustls] +package = "rustls" +version = "0.21.8" +optional = true + +[dependencies.h2-0-3] +package = "h2" +version = "0.3.24" +optional = true + +[dependencies.bytes] +workspace = true +optional = true + +[dependencies.serde] +workspace = true +features = ["derive"] +optional = true + +[dependencies.serde_json] +workspace = true +features = ["preserve_order"] +optional = true + +[dependencies.indexmap] +workspace = true +features = ["serde"] +optional = true + +[dependencies.http-body-util] +workspace = true +optional = true + +[dev-dependencies] +serial_test = "3.2" +base64.workspace = true +rustls-pemfile = "2.2.0" +tokio-rustls.workspace = true + +[dev-dependencies.aws-smithy-async] +workspace = true +features = ["rt-tokio", "test-util"] + +[dev-dependencies.aws-smithy-runtime-api] +workspace = true +features = ["test-util"] + +[dev-dependencies.aws-smithy-types] +workspace = true +features = ["http-body-0-4-x", "test-util"] + +[dev-dependencies.http-body-util] +workspace = true + +[dev-dependencies.hyper-util] +workspace = true +features = ["full"] + +[dev-dependencies.rustls-pki-types] +workspace = true + +[dev-dependencies.tokio] +workspace = true +features = ["macros", "rt", "rt-multi-thread", "test-util", "full"] +[lints.rust.unexpected_cfgs] +level = "warn" +check-cfg = ["cfg(aws_sdk_unstable)"] diff --git a/sources/aws-smithy-experimental/LICENSE b/sources/aws-smithy-http-client/LICENSE similarity index 100% rename from sources/aws-smithy-experimental/LICENSE rename to sources/aws-smithy-http-client/LICENSE diff --git a/sources/aws-smithy-http-client/README.md b/sources/aws-smithy-http-client/README.md new file mode 100644 index 000000000..261a84b86 --- /dev/null +++ b/sources/aws-smithy-http-client/README.md @@ -0,0 +1,7 @@ +# aws-smithy-http-client + +HTTP client abstractions for generated smithy clients. + + +This crate is part of the [AWS SDK for Rust](https://awslabs.github.io/aws-sdk-rust/) and the [smithy-rs](https://github.com/smithy-lang/smithy-rs) code generator. In most cases, it should not be used directly. + diff --git a/sources/aws-smithy-http-client/additional-ci b/sources/aws-smithy-http-client/additional-ci new file mode 100755 index 000000000..21cc04a5f --- /dev/null +++ b/sources/aws-smithy-http-client/additional-ci @@ -0,0 +1,13 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +# This script contains additional CI checks to run for this specific package + +set -e + +echo "### Testing unstable custom rustls crypto provider" +# Enabling ring because the tests use it as the "custom" provider, and it transitively enables __rustls +RUSTFLAGS="--cfg aws_sdk_unstable" cargo test --features rustls-ring diff --git a/sources/aws-smithy-http-client/examples/client-aws-lc.rs b/sources/aws-smithy-http-client/examples/client-aws-lc.rs new file mode 100644 index 000000000..462ab5e88 --- /dev/null +++ b/sources/aws-smithy-http-client/examples/client-aws-lc.rs @@ -0,0 +1,23 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_smithy_http_client::{ + tls::{self, rustls_provider::CryptoMode}, + Builder, +}; + +#[tokio::main] +async fn main() { + // feature = rustls-aws-lc + let _client = Builder::new() + .tls_provider(tls::Provider::Rustls(CryptoMode::AwsLc)) + .build_https(); + + // feature = rustls-aws-lc-fips + // A FIPS client can also be created. Note that this has a more complex build environment required. + let _client = Builder::new() + .tls_provider(tls::Provider::Rustls(CryptoMode::AwsLcFips)) + .build_https(); +} diff --git a/sources/aws-smithy-http-client/examples/client-ring.rs b/sources/aws-smithy-http-client/examples/client-ring.rs new file mode 100644 index 000000000..790446972 --- /dev/null +++ b/sources/aws-smithy-http-client/examples/client-ring.rs @@ -0,0 +1,15 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_smithy_http_client::{ + tls::{self, rustls_provider::CryptoMode}, + Builder, +}; + +fn main() { + let _client = Builder::new() + .tls_provider(tls::Provider::Rustls(CryptoMode::Ring)) + .build_https(); +} diff --git a/sources/aws-smithy-http-client/examples/client-s2n-tls.rs b/sources/aws-smithy-http-client/examples/client-s2n-tls.rs new file mode 100644 index 000000000..e9991db7d --- /dev/null +++ b/sources/aws-smithy-http-client/examples/client-s2n-tls.rs @@ -0,0 +1,12 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_smithy_http_client::{tls, Builder}; + +fn main() { + let _client = Builder::new() + .tls_provider(tls::Provider::S2nTls) + .build_https(); +} diff --git a/sources/aws-smithy-http-client/examples/custom-dns.rs b/sources/aws-smithy-http-client/examples/custom-dns.rs new file mode 100644 index 000000000..627bf5c39 --- /dev/null +++ b/sources/aws-smithy-http-client/examples/custom-dns.rs @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_smithy_http_client::{ + tls::{self, rustls_provider::CryptoMode}, + Builder, +}; +use aws_smithy_runtime_api::client::dns::{DnsFuture, ResolveDns}; +use std::net::{IpAddr, Ipv4Addr}; + +#[derive(Debug, Clone)] +struct StaticResolver; + +impl ResolveDns for StaticResolver { + fn resolve_dns<'a>(&'a self, _name: &'a str) -> DnsFuture<'a> { + DnsFuture::ready(Ok(vec![IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))])) + } +} + +fn main() { + let _client = Builder::new() + .tls_provider(tls::Provider::Rustls(CryptoMode::Ring)) + .build_with_resolver(StaticResolver); +} diff --git a/sources/aws-smithy-http-client/external-types.toml b/sources/aws-smithy-http-client/external-types.toml new file mode 100644 index 000000000..c52fbf753 --- /dev/null +++ b/sources/aws-smithy-http-client/external-types.toml @@ -0,0 +1,26 @@ +allowed_external_types = [ + "aws_smithy_runtime_api::*", + "aws_smithy_async::*", + "aws_smithy_types::*", + + # TODO(https://github.com/smithy-lang/smithy-rs/issues/1193): Once tooling permits it, only allow the following types based on the feature that exposes them + "hyper::client::client::Builder", + "hyper::client::connect::Connection", + "tokio::io::async_read::AsyncRead", + "tokio::io::async_write::AsyncWrite", + "http::uri::Uri", + # Used in trait bounds for legacy hyper connector build method + "tower_service::Service", + + "bytes::bytes::Bytes", + "serde::ser::Serialize", + "serde::de::Deserialize", + "serde_core::de::Deserialize", + "serde_core::ser::Serialize", + "hyper::client::connect::dns::Name", + "http::request::Request", + "http::response::Response", + "tokio::net::tcp::stream::TcpStream", + "hyper_util::rt::tokio::TokioIo", + +] diff --git a/sources/aws-smithy-http-client/src/client.rs b/sources/aws-smithy-http-client/src/client.rs new file mode 100644 index 000000000..cc808aa67 --- /dev/null +++ b/sources/aws-smithy-http-client/src/client.rs @@ -0,0 +1,1350 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +mod dns; +/// Proxy configuration +pub mod proxy; +mod timeout; +/// TLS connector(s) +pub mod tls; + +pub(crate) mod connect; + +use crate::cfg::cfg_tls; +use crate::tls::TlsContext; +use aws_smithy_async::future::timeout::TimedOutError; +use aws_smithy_async::rt::sleep::{default_async_sleep, AsyncSleep, SharedAsyncSleep}; +use aws_smithy_runtime_api::box_error::BoxError; +use aws_smithy_runtime_api::client::connection::CaptureSmithyConnection; +use aws_smithy_runtime_api::client::connection::ConnectionMetadata; +use aws_smithy_runtime_api::client::connector_metadata::ConnectorMetadata; +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpClient, + SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse}; +use aws_smithy_runtime_api::client::result::ConnectorError; +use aws_smithy_runtime_api::client::runtime_components::{ + RuntimeComponents, RuntimeComponentsBuilder, +}; +use aws_smithy_runtime_api::shared::IntoShared; +use aws_smithy_types::body::SdkBody; +use aws_smithy_types::config_bag::ConfigBag; +use aws_smithy_types::error::display::DisplayErrorContext; +use aws_smithy_types::retry::ErrorKind; +use client::connect::Connection; +use h2::Reason; +use http_1x::{Extensions, Uri}; +use hyper::rt::{Read, Write}; +use hyper_util::client::legacy as client; +use hyper_util::client::legacy::connect::dns::GaiResolver; +use hyper_util::client::legacy::connect::{ + capture_connection, CaptureConnection, Connect, HttpConnector as HyperHttpConnector, HttpInfo, +}; +use hyper_util::client::proxy::matcher::Matcher; +use hyper_util::rt::{TokioExecutor, TokioTimer}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::error::Error; +use std::fmt; +use std::sync::RwLock; +use std::time::Duration; + +/// Given `HttpConnectorSettings` and an `SharedAsyncSleep`, create a `SharedHttpConnector` from defaults depending on what cargo features are activated. +pub fn default_connector( + settings: &HttpConnectorSettings, + sleep: Option, +) -> Option { + #[cfg(feature = "rustls-aws-lc")] + { + tracing::trace!(settings = ?settings, sleep = ?sleep, "creating a new default connector"); + let mut conn_builder = Connector::builder().connector_settings(settings.clone()); + + if let Some(sleep) = sleep { + conn_builder = conn_builder.sleep_impl(sleep); + } + + let conn = conn_builder + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::AwsLc, + )) + .build(); + Some(SharedHttpConnector::new(conn)) + } + #[cfg(not(feature = "rustls-aws-lc"))] + { + tracing::trace!(settings = ?settings, sleep = ?sleep, "no default connector available"); + None + } +} + +/// [`HttpConnector`] used to make HTTP requests. +/// +/// This connector also implements socket connect and read timeouts. +/// +/// This shouldn't be used directly in most cases. +/// See the docs on [`Builder`] for examples of how to customize the HTTP client. +#[derive(Debug)] +pub struct Connector { + adapter: Box, +} + +impl Connector { + /// Builder for an HTTP connector. + pub fn builder() -> ConnectorBuilder { + ConnectorBuilder::default() + } +} + +impl HttpConnector for Connector { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + self.adapter.call(request) + } +} + +/// Builder for [`Connector`]. +#[derive(Debug, Clone)] +pub struct ConnectorBuilder { + connector_settings: Option, + sleep_impl: Option, + client_builder: Option, + pool_idle_timeout: Option>, + enable_tcp_nodelay: bool, + interface: Option, + proxy_config: Option, + #[allow(unused)] + tls: Tls, +} + +impl Default for ConnectorBuilder { + fn default() -> Self { + Self { + connector_settings: None, + sleep_impl: None, + client_builder: None, + pool_idle_timeout: None, + // Curated default: TCP_NODELAY on. Without it, Nagle's algorithm + // can hold a small write while earlier data is unacknowledged. On + // request shapes emitted as multiple sub-MSS writes, this can add + // an ACK wait, often RTT plus delayed-ACK time. Opt out with + // `enable_tcp_nodelay(false)`. + enable_tcp_nodelay: true, + interface: None, + proxy_config: None, + tls: Tls::default(), + } + } +} + +/// Initial builder state, `TlsProvider` choice required +#[derive(Default, Debug, Clone)] +#[non_exhaustive] +pub struct TlsUnset {} + +/// TLS implementation selected +#[derive(Debug, Clone)] +pub struct TlsProviderSelected { + #[allow(unused)] + provider: tls::Provider, + #[allow(unused)] + context: TlsContext, +} + +impl ConnectorBuilder { + /// Set the TLS implementation to use for this connector + pub fn tls_provider(self, provider: tls::Provider) -> ConnectorBuilder { + ConnectorBuilder { + connector_settings: self.connector_settings, + sleep_impl: self.sleep_impl, + client_builder: self.client_builder, + enable_tcp_nodelay: self.enable_tcp_nodelay, + interface: self.interface, + proxy_config: self.proxy_config, + pool_idle_timeout: self.pool_idle_timeout, + tls: TlsProviderSelected { + provider, + context: TlsContext::default(), + }, + } + } + + /// Build an HTTP connector sans TLS + #[doc(hidden)] + pub fn build_http(self) -> Connector { + if let Some(ref proxy_config) = self.proxy_config { + if proxy_config.requires_tls() { + tracing::warn!( + "HTTPS proxy configured but no TLS provider set. \ + Connections to HTTPS proxy servers will fail. \ + Consider configuring a TLS provider to enable TLS support." + ); + } + } + + let base = self.base_connector(); + + // Wrap with HTTP proxy support if proxy is configured + let proxy_config = self + .proxy_config + .clone() + .unwrap_or_else(proxy::ProxyConfig::disabled); + + if !proxy_config.is_disabled() { + let http_proxy_connector = connect::HttpProxyConnector::new(base, proxy_config); + self.wrap_connector(http_proxy_connector) + } else { + self.wrap_connector(base) + } + } +} + +impl ConnectorBuilder { + /// Create a [`Connector`] from this builder and a given connector. + pub(crate) fn wrap_connector(self, tcp_connector: C) -> Connector + where + C: Send + Sync + 'static, + C: Clone, + C: tower::Service, + C::Response: Read + Write + Connection + Send + Sync + Unpin, + C: Connect, + C::Future: Unpin + Send + 'static, + C::Error: Into, + { + let client_builder = self + .client_builder + .unwrap_or_else(|| new_tokio_hyper_builder(self.pool_idle_timeout)); + let sleep_impl = self.sleep_impl.or_else(default_async_sleep); + let (connect_timeout, read_timeout) = self + .connector_settings + .map(|c| (c.connect_timeout(), c.read_timeout())) + .unwrap_or((None, None)); + + let connector = match connect_timeout { + Some(duration) => timeout::ConnectTimeout::new( + tcp_connector, + sleep_impl + .clone() + .expect("a sleep impl must be provided in order to have a connect timeout"), + duration, + ), + None => timeout::ConnectTimeout::no_timeout(tcp_connector), + }; + let base = client_builder.build(connector); + let read_timeout = match read_timeout { + Some(duration) => timeout::HttpReadTimeout::new( + base, + sleep_impl.expect("a sleep impl must be provided in order to have a read timeout"), + duration, + ), + None => timeout::HttpReadTimeout::no_timeout(base), + }; + + let proxy_matcher = self + .proxy_config + .as_ref() + .map(|config| config.clone().into_hyper_util_matcher()); + + Connector { + adapter: Box::new(Adapter { + client: read_timeout, + proxy_matcher, + }), + } + } + + /// Get the base TCP connector by mapping our config to the underlying `HttpConnector` from hyper + /// (which is a base TCP connector with no TLS or any wrapping) + fn base_connector(&self) -> HyperHttpConnector { + self.base_connector_with_resolver(GaiResolver::new()) + } + + /// Get the base TCP connector by mapping our config to the underlying `HttpConnector` from hyper + /// using the given resolver `R` + fn base_connector_with_resolver(&self, resolver: R) -> HyperHttpConnector { + let mut conn = HyperHttpConnector::new_with_resolver(resolver); + conn.set_nodelay(self.enable_tcp_nodelay); + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + if let Some(interface) = &self.interface { + conn.set_interface(interface); + } + conn + } + + /// Set the async sleep implementation used for timeouts + /// + /// Calling this is only necessary for testing or to use something other than + /// [`default_async_sleep`]. + pub fn sleep_impl(mut self, sleep_impl: impl AsyncSleep + 'static) -> Self { + self.sleep_impl = Some(sleep_impl.into_shared()); + self + } + + /// Set the async sleep implementation used for timeouts + /// + /// Calling this is only necessary for testing or to use something other than + /// [`default_async_sleep`]. + pub fn set_sleep_impl(&mut self, sleep_impl: Option) -> &mut Self { + self.sleep_impl = sleep_impl; + self + } + + /// Configure the HTTP settings for the `HyperAdapter` + pub fn connector_settings(mut self, connector_settings: HttpConnectorSettings) -> Self { + self.connector_settings = Some(connector_settings); + self + } + + /// Configure the HTTP settings for the `HyperAdapter` + pub fn set_connector_settings( + &mut self, + connector_settings: Option, + ) -> &mut Self { + self.connector_settings = connector_settings; + self + } + + /// Configure `SO_NODELAY` for all sockets to the supplied value `nodelay` + pub fn enable_tcp_nodelay(mut self, nodelay: bool) -> Self { + self.enable_tcp_nodelay = nodelay; + self + } + + /// Configure `SO_NODELAY` for all sockets to the supplied value `nodelay` + pub fn set_enable_tcp_nodelay(&mut self, nodelay: bool) -> &mut Self { + self.enable_tcp_nodelay = nodelay; + self + } + + /// Sets the value for the `SO_BINDTODEVICE` option on this socket. + /// + /// If a socket is bound to an interface, only packets received from that particular + /// interface are processed by the socket. Note that this only works for some socket + /// types (e.g. `AF_INET` sockets). + /// + /// On Linux it can be used to specify a [VRF], but the binary needs to either have + /// `CAP_NET_RAW` capability set or be run as root. + /// + /// This function is only available on Android, Fuchsia, and Linux. + /// + /// [VRF]: https://www.kernel.org/doc/Documentation/networking/vrf.txt + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + pub fn set_interface>(&mut self, interface: S) -> &mut Self { + self.interface = Some(interface.into()); + self + } + + /// Configure proxy settings for this connector + /// + /// This method allows you to set explicit proxy configuration for the HTTP client. + /// The proxy configuration will be used to determine whether requests should be + /// routed through a proxy server or connect directly. + /// + /// # Examples + /// + /// ```rust + /// # #[cfg(feature = "rustls-aws-lc")] + /// # { + /// use aws_smithy_http_client::{Connector, proxy::ProxyConfig, tls}; + /// + /// let proxy_config = ProxyConfig::http("http://proxy.example.com:8080")?; + /// let connector = Connector::builder() + /// .proxy_config(proxy_config) + /// .tls_provider(tls::Provider::Rustls(tls::rustls_provider::CryptoMode::AwsLc)) + /// .build(); + /// # } + /// # Ok::<(), Box>(()) + /// ``` + pub fn proxy_config(mut self, config: proxy::ProxyConfig) -> Self { + self.proxy_config = Some(config); + self + } + + /// Configure proxy settings for this connector + /// + /// This is the mutable version of [`proxy_config`](Self::proxy_config). + pub fn set_proxy_config(&mut self, config: Option) -> &mut Self { + self.proxy_config = config; + self + } + + /// Set an optional timeout for idle sockets being kept-alive. + /// + /// Pass `None` to disable timeout. + /// + /// Defaults to Hyper's default timeout, which is currently 90 seconds - see + /// [hyper_util::client::legacy::Builder::pool_idle_timeout], + /// but unlike that function, there is no need to call `pool_timer` yourself. + /// + /// # Examples + /// + /// ```rust + /// # #[cfg(feature = "rustls-aws-lc")] + /// # { + /// use aws_smithy_http_client::{Connector, tls}; + /// use std::time::Duration; + /// + /// let connector = Connector::builder() + /// .pool_idle_timeout(Duration::from_secs(30)) + /// .tls_provider(tls::Provider::Rustls(tls::rustls_provider::CryptoMode::AwsLc)) + /// .build(); + /// # } + /// # Ok::<(), Box>(()) + /// ``` + pub fn pool_idle_timeout(mut self, val: D) -> Self + where + D: Into>, + { + self.pool_idle_timeout = Some(val.into()); + self + } + + /// Set an optional timeout for idle sockets being kept-alive. + /// + /// Pass `None` to use Hyper's default timeout, `Some(None)` to disable timeouts. + /// + /// This is the mutable version of [`pool_idle_timeout`](Self::pool_idle_timeout). + /// + /// # Examples + /// + /// ```rust + /// # #[cfg(feature = "rustls-aws-lc")] + /// # { + /// use aws_smithy_http_client::{Connector, tls}; + /// use std::time::Duration; + /// + /// let mut connector = Connector::builder(); + /// connector + /// .set_pool_idle_timeout(Some(Some(Duration::from_secs(30)))); + /// connector + /// .tls_provider(tls::Provider::Rustls(tls::rustls_provider::CryptoMode::AwsLc)) + /// .build(); + /// # } + /// # Ok::<(), Box>(()) + /// ``` + pub fn set_pool_idle_timeout(&mut self, val: Option>) -> &mut Self { + self.pool_idle_timeout = val; + self + } + + /// Override the Hyper client [`Builder`](hyper_util::client::legacy::Builder) used to construct this client. + /// + /// This enables changing settings like forcing HTTP2 and modifying other default client behavior. + pub(crate) fn hyper_builder( + mut self, + hyper_builder: hyper_util::client::legacy::Builder, + ) -> Self { + self.set_hyper_builder(Some(hyper_builder)); + self + } + + /// Override the Hyper client [`Builder`](hyper_util::client::legacy::Builder) used to construct this client. + /// + /// This enables changing settings like forcing HTTP2 and modifying other default client behavior. + pub(crate) fn set_hyper_builder( + &mut self, + hyper_builder: Option, + ) -> &mut Self { + self.client_builder = hyper_builder; + self + } +} + +/// Adapter to use a Hyper 1.0-based Client as an `HttpConnector` +/// +/// This adapter also enables TCP `CONNECT` and HTTP `READ` timeouts via [`Connector::builder`]. +struct Adapter { + client: timeout::HttpReadTimeout< + hyper_util::client::legacy::Client, SdkBody>, + >, + proxy_matcher: Option, +} + +impl fmt::Debug for Adapter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Adapter") + .field("client", &"** hyper client **") + .field("proxy_matcher", &self.proxy_matcher.is_some()) + .finish() + } +} + +/// Extract a smithy connection from a hyper CaptureConnection +fn extract_smithy_connection(capture_conn: &CaptureConnection) -> Option { + let capture_conn = capture_conn.clone(); + if let Some(conn) = capture_conn.clone().connection_metadata().as_ref() { + let mut extensions = Extensions::new(); + conn.get_extras(&mut extensions); + let http_info = extensions.get::(); + let mut builder = ConnectionMetadata::builder() + .proxied(conn.is_proxied()) + .poison_fn(move || match capture_conn.connection_metadata().as_ref() { + Some(conn) => conn.poison(), + None => tracing::trace!("no connection existed to poison"), + }); + + builder + .set_local_addr(http_info.map(|info| info.local_addr())) + .set_remote_addr(http_info.map(|info| info.remote_addr())); + + let smithy_connection = builder.build(); + + Some(smithy_connection) + } else { + None + } +} + +fn new_tokio_hyper_builder( + pool_idle_timeout: Option>, +) -> hyper_util::client::legacy::Builder { + let mut builder = hyper_util::client::legacy::Builder::new(TokioExecutor::new()); + // Explicitly setting the pool_timer is required for connection timeouts to work. + builder.pool_timer(TokioTimer::new()); + + if let Some(pool_idle_timeout) = pool_idle_timeout { + builder.pool_idle_timeout(pool_idle_timeout); + } + + builder +} + +impl Adapter { + /// Add proxy authentication header to the request if needed + fn add_proxy_auth_header(&self, request: &mut http_1x::Request) { + // Only add auth for HTTP requests (not HTTPS which uses CONNECT tunneling) + if request.uri().scheme() != Some(&http_1x::uri::Scheme::HTTP) { + return; + } + + // Don't override existing proxy authorization header + if request + .headers() + .contains_key(http_1x::header::PROXY_AUTHORIZATION) + { + return; + } + + if let Some(ref matcher) = self.proxy_matcher { + if let Some(intercept) = matcher.intercept(request.uri()) { + // Add basic auth header if available + if let Some(auth_header) = intercept.basic_auth() { + request + .headers_mut() + .insert(http_1x::header::PROXY_AUTHORIZATION, auth_header.clone()); + tracing::debug!("added proxy authentication header for {}", request.uri()); + } + } + } + } +} + +impl HttpConnector for Adapter +where + C: Clone + Send + Sync + 'static, + C: tower::Service, + C::Response: Connection + Read + Write + Unpin + 'static, + timeout::ConnectTimeout: Connect, + C::Future: Unpin + Send + 'static, + C::Error: Into, +{ + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let mut request = match request.try_into_http1x() { + Ok(request) => request, + Err(err) => { + return HttpConnectorFuture::ready(Err(ConnectorError::user(err.into()))); + } + }; + + self.add_proxy_auth_header(&mut request); + + let capture_connection = capture_connection(&mut request); + if let Some(capture_smithy_connection) = + request.extensions().get::() + { + capture_smithy_connection + .set_connection_retriever(move || extract_smithy_connection(&capture_connection)); + } + let mut client = self.client.clone(); + use tower::Service; + let fut = client.call(request); + HttpConnectorFuture::new(async move { + let response = fut + .await + .map_err(downcast_error)? + .map(SdkBody::from_body_1_x); + match HttpResponse::try_from(response) { + Ok(response) => Ok(response), + Err(err) => Err(ConnectorError::other(err.into(), None)), + } + }) + } +} + +/// Downcast errors coming out of hyper into an appropriate `ConnectorError` +fn downcast_error(err: BoxError) -> ConnectorError { + // is a `TimedOutError` (from aws_smithy_async::timeout) in the chain? if it is, this is a timeout + if find_source::(err.as_ref()).is_some() { + return ConnectorError::timeout(err); + } + // is the top of chain error actually already a `ConnectorError`? return that directly + let err = match err.downcast::() { + Ok(connector_error) => return *connector_error, + Err(box_error) => box_error, + }; + // generally, the top of chain will probably be a hyper error. Go through a set of hyper specific + // error classifications + let err = match find_source::(err.as_ref()) { + Some(hyper_error) => return to_connector_error(hyper_error)(err), + None => match find_source::(err.as_ref()) { + Some(hyper_util_err) => { + if hyper_util_err.is_connect() + || find_source::(hyper_util_err).is_some() + { + return ConnectorError::io(err); + } + err + } + None => err, + }, + }; + + // otherwise, we have no idea! + ConnectorError::other(err, None) +} + +/// Convert a [`hyper::Error`] into a [`ConnectorError`] +fn to_connector_error(err: &hyper::Error) -> fn(BoxError) -> ConnectorError { + if err.is_timeout() || find_source::(err).is_some() { + return ConnectorError::timeout; + } + if err.is_user() { + return ConnectorError::user; + } + if err.is_closed() || err.is_canceled() || find_source::(err).is_some() { + return ConnectorError::io; + } + // We sometimes receive this from S3: hyper::Error(IncompleteMessage) + if err.is_incomplete_message() { + return |err: BoxError| ConnectorError::other(err, Some(ErrorKind::TransientError)); + } + + if let Some(h2_err) = find_source::(err) { + if h2_err.is_go_away() + || (h2_err.is_reset() && h2_err.reason() == Some(Reason::REFUSED_STREAM)) + { + return ConnectorError::io; + } + } + + tracing::warn!(err = %DisplayErrorContext(&err), "unrecognized error from Hyper. If this error should be retried, please file an issue."); + |err: BoxError| ConnectorError::other(err, None) +} + +fn find_source<'a, E: Error + 'static>(err: &'a (dyn Error + 'static)) -> Option<&'a E> { + let mut next = Some(err); + while let Some(err) = next { + if let Some(matching_err) = err.downcast_ref::() { + return Some(matching_err); + } + next = err.source(); + } + None +} + +// TODO(https://github.com/awslabs/aws-sdk-rust/issues/1090): CacheKey must also include ptr equality to any +// runtime components that are used—sleep_impl as a base (unless we prohibit overriding sleep impl) +// If we decide to put a DnsResolver in RuntimeComponents, then we'll need to handle that as well. +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +struct CacheKey { + connect_timeout: Option, + read_timeout: Option, +} + +impl From<&HttpConnectorSettings> for CacheKey { + fn from(value: &HttpConnectorSettings) -> Self { + Self { + connect_timeout: value.connect_timeout(), + read_timeout: value.read_timeout(), + } + } +} + +struct HyperClient { + connector_cache: RwLock>, + client_builder: hyper_util::client::legacy::Builder, + connector_fn: F, +} + +impl fmt::Debug for HyperClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HyperClient") + .field("connector_cache", &self.connector_cache) + .field("client_builder", &self.client_builder) + .finish() + } +} + +impl HttpClient for HyperClient +where + F: Fn( + hyper_util::client::legacy::Builder, + Option<&HttpConnectorSettings>, + Option<&RuntimeComponents>, + ) -> Connector + + Send + + Sync + + 'static, +{ + fn http_connector( + &self, + settings: &HttpConnectorSettings, + components: &RuntimeComponents, + ) -> SharedHttpConnector { + let key = CacheKey::from(settings); + let mut connector = self.connector_cache.read().unwrap().get(&key).cloned(); + if connector.is_none() { + let mut cache = self.connector_cache.write().unwrap(); + // Short-circuit if another thread already wrote a connector to the cache for this key + if !cache.contains_key(&key) { + let start = components.time_source().map(|ts| ts.now()); + let connector = (self.connector_fn)( + self.client_builder.clone(), + Some(settings), + Some(components), + ); + let end = components.time_source().map(|ts| ts.now()); + if let (Some(start), Some(end)) = (start, end) { + if let Ok(elapsed) = end.duration_since(start) { + tracing::debug!("new connector created in {:?}", elapsed); + } + } + let connector = SharedHttpConnector::new(connector); + cache.insert(key.clone(), connector); + } + connector = cache.get(&key).cloned(); + } + + connector.expect("cache populated above") + } + + fn validate_base_client_config( + &self, + _: &RuntimeComponentsBuilder, + _: &ConfigBag, + ) -> Result<(), BoxError> { + // Initialize the TCP connector at this point so that native certs load + // at client initialization time instead of upon first request. We do it + // here rather than at construction so that it won't run if this is not + // the selected HTTP client for the base config (for example, if this was + // the default HTTP client, and it was overridden by a later plugin). + let _ = (self.connector_fn)(self.client_builder.clone(), None, None); + Ok(()) + } + + fn connector_metadata(&self) -> Option { + Some(ConnectorMetadata::new("hyper", Some(Cow::Borrowed("1.x")))) + } +} + +/// Builder for a hyper-backed [`HttpClient`] implementation. +/// +/// This builder can be used to customize the underlying TCP connector used, as well as +/// hyper client configuration. +/// +/// # Examples +/// +/// Construct a Hyper client with the RusTLS TLS implementation. +/// This can be useful when you want to share a Hyper connector between multiple +/// generated Smithy clients. +#[derive(Clone, Default, Debug)] +pub struct Builder { + client_builder: Option, + pool_idle_timeout: Option>, + #[allow(unused)] + tls_provider: Tls, +} + +cfg_tls! { + use aws_smithy_runtime_api::client::dns::ResolveDns; + + impl ConnectorBuilder { + /// Build a [`Connector`] that will use the default DNS resolver implementation. + pub fn build(self) -> Connector { + let http_connector = self.base_connector(); + self.build_https(http_connector) + } + + /// Configure the TLS context + pub fn tls_context(mut self, ctx: TlsContext) -> Self { + self.tls.context = ctx; + self + } + + /// Configure the TLS context + pub fn set_tls_context(&mut self, ctx: TlsContext) -> &mut Self { + self.tls.context = ctx; + self + } + + /// Build a [`Connector`] that will use the given DNS resolver implementation. + pub fn build_with_resolver(self, resolver: R) -> Connector { + use crate::client::dns::HyperUtilResolver; + let http_connector = self.base_connector_with_resolver(HyperUtilResolver { resolver }); + self.build_https(http_connector) + } + + fn build_https(self, http_connector: HyperHttpConnector) -> Connector + where + R: Clone + Send + Sync + 'static, + R: tower::Service, + R::Response: Iterator, + R::Future: Send, + R::Error: Into>, + { + match &self.tls.provider { + // TODO(hyper1) - fix cfg_rustls! to allow matching on patterns so we can re-use it and not duplicate these cfg matches everywhere + #[cfg(feature = "__rustls")] + tls::Provider::Rustls(crypto_mode) => { + let proxy_config = self.proxy_config.clone() + .unwrap_or_else(proxy::ProxyConfig::disabled); + + let https_connector = tls::rustls_provider::build_connector::wrap_connector( + http_connector, + crypto_mode.clone(), + &self.tls.context, + proxy_config, + ); + self.wrap_connector(https_connector) + }, + #[cfg(feature = "s2n-tls")] + tls::Provider::S2nTls => { + let proxy_config = self.proxy_config.clone() + .unwrap_or_else(proxy::ProxyConfig::disabled); + + let https_connector = tls::s2n_tls_provider::build_connector::wrap_connector( + http_connector, + &self.tls.context, + proxy_config, + ); + self.wrap_connector(https_connector) + } + } + } + } + + impl Builder { + /// Create an HTTPS client with the selected TLS provider. + /// + /// The trusted certificates will be loaded later when this becomes the selected + /// HTTP client for a Smithy client. + pub fn build_https(self) -> SharedHttpClient { + build_with_conn_fn( + self.client_builder, + self.pool_idle_timeout, + move |client_builder, settings, runtime_components| { + let builder = new_conn_builder(client_builder, settings, runtime_components) + .tls_provider(self.tls_provider.provider.clone()) + .tls_context(self.tls_provider.context.clone()); + builder.build() + }, + ) + } + + /// Create an HTTPS client using a custom DNS resolver + pub fn build_with_resolver( + self, + resolver: impl ResolveDns + Clone + 'static, + ) -> SharedHttpClient { + build_with_conn_fn( + self.client_builder, + self.pool_idle_timeout, + move |client_builder, settings, runtime_components| { + let builder = new_conn_builder(client_builder, settings, runtime_components) + .tls_provider(self.tls_provider.provider.clone()) + .tls_context(self.tls_provider.context.clone()); + builder.build_with_resolver(resolver.clone()) + }, + ) + } + + /// Configure the TLS context + pub fn tls_context(mut self, ctx: TlsContext) -> Self { + self.tls_provider.context = ctx; + self + } + } +} + +impl Builder { + /// Set an optional timeout for idle sockets being kept-alive. + /// + /// Pass `None` to disable timeout. + /// + /// Defaults to Hyper's default timeout, which is currently 90 seconds - see + /// [hyper_util::client::legacy::Builder::pool_idle_timeout], + /// but unlike that function, there is no need to call `pool_timer` yourself. + /// + /// # Examples + /// + /// ```rust + /// # #[cfg(feature = "rustls-aws-lc")] + /// # { + /// use aws_smithy_http_client::{Builder, tls}; + /// use std::time::Duration; + /// + /// let client = Builder::new() + /// .pool_idle_timeout(Duration::from_secs(30)) + /// .tls_provider(tls::Provider::Rustls(tls::rustls_provider::CryptoMode::AwsLc)) + /// .build_https(); + /// # } + /// # Ok::<(), Box>(()) + /// ``` + pub fn pool_idle_timeout(mut self, val: D) -> Self + where + D: Into>, + { + self.pool_idle_timeout = Some(val.into()); + self + } + + /// Set an optional timeout for idle sockets being kept-alive. + /// + /// Pass `None` to use Hyper's default timeout, `Some(None)` to disable timeouts. + /// + /// This is the mutable version of [`pool_idle_timeout`](Self::pool_idle_timeout). + /// + /// # Examples + /// + /// ```rust + /// # #[cfg(feature = "rustls-aws-lc")] + /// # { + /// use std::time::Duration; + /// use aws_smithy_http_client::{Builder, tls}; + /// + /// let mut client = Builder::new(); + /// client.set_pool_idle_timeout(Some(Some(Duration::from_secs(30)))); + /// client + /// .tls_provider(tls::Provider::Rustls(tls::rustls_provider::CryptoMode::AwsLc)) + /// .build_https(); + /// # } + /// # Ok::<(), Box>(()) + /// ``` + pub fn set_pool_idle_timeout(&mut self, val: Option>) -> &mut Self { + self.pool_idle_timeout = val; + self + } +} + +impl Builder { + /// Creates a new builder. + pub fn new() -> Self { + Self::default() + } + + /// Returns a [`SharedHttpClient`] that calls the given `connector` function to select an HTTP(S) connector. + #[doc(hidden)] + pub fn build_with_connector_fn(self, connector_fn: F) -> SharedHttpClient + where + F: Fn(Option<&HttpConnectorSettings>, Option<&RuntimeComponents>) -> Connector + + Send + + Sync + + 'static, + { + build_with_conn_fn( + self.client_builder, + self.pool_idle_timeout, + move |_builder, settings, runtime_components| { + connector_fn(settings, runtime_components) + }, + ) + } + + /// Build a new HTTP client without TLS enabled + #[doc(hidden)] + pub fn build_http(self) -> SharedHttpClient { + build_with_conn_fn( + self.client_builder, + self.pool_idle_timeout, + move |client_builder, settings, runtime_components| { + let builder = new_conn_builder(client_builder, settings, runtime_components); + builder.build_http() + }, + ) + } + + /// Set the TLS implementation to use + pub fn tls_provider(self, provider: tls::Provider) -> Builder { + Builder { + client_builder: self.client_builder, + pool_idle_timeout: self.pool_idle_timeout, + tls_provider: TlsProviderSelected { + provider, + context: TlsContext::default(), + }, + } + } +} + +pub(crate) fn build_with_conn_fn( + client_builder: Option, + pool_idle_timeout: Option>, + connector_fn: F, +) -> SharedHttpClient +where + F: Fn( + hyper_util::client::legacy::Builder, + Option<&HttpConnectorSettings>, + Option<&RuntimeComponents>, + ) -> Connector + + Send + + Sync + + 'static, +{ + let client_builder = + client_builder.unwrap_or_else(|| new_tokio_hyper_builder(pool_idle_timeout)); + SharedHttpClient::new(HyperClient { + connector_cache: RwLock::new(HashMap::new()), + client_builder, + connector_fn, + }) +} + +#[allow(dead_code)] +pub(crate) fn build_with_tcp_conn_fn( + client_builder: Option, + pool_idle_timeout: Option>, + tcp_connector_fn: F, +) -> SharedHttpClient +where + F: Fn() -> C + Send + Sync + 'static, + C: Clone + Send + Sync + 'static, + C: tower::Service, + C::Response: Connection + Read + Write + Send + Sync + Unpin + 'static, + C::Future: Unpin + Send + 'static, + C::Error: Into, + C: Connect, +{ + build_with_conn_fn( + client_builder, + pool_idle_timeout, + move |client_builder, settings, runtime_components| { + let builder = new_conn_builder(client_builder, settings, runtime_components); + builder.wrap_connector(tcp_connector_fn()) + }, + ) +} + +fn new_conn_builder( + client_builder: hyper_util::client::legacy::Builder, + settings: Option<&HttpConnectorSettings>, + runtime_components: Option<&RuntimeComponents>, +) -> ConnectorBuilder { + let mut builder = Connector::builder().hyper_builder(client_builder); + builder.set_connector_settings(settings.cloned()); + if let Some(components) = runtime_components { + builder.set_sleep_impl(components.sleep_impl()); + } + builder +} + +#[cfg(test)] +mod test { + use std::io::{Error, ErrorKind}; + use std::pin::Pin; + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::Arc; + use std::task::{Context, Poll}; + + use crate::client::timeout::test::NeverConnects; + use aws_smithy_async::assert_elapsed; + use aws_smithy_async::rt::sleep::TokioSleep; + use aws_smithy_async::time::SystemTimeSource; + use aws_smithy_runtime_api::client::runtime_components::RuntimeComponentsBuilder; + use http_1x::Uri; + use hyper::rt::ReadBufCursor; + use hyper_util::client::legacy::connect::Connected; + + use super::*; + + #[tokio::test] + async fn connector_selection() { + // Create a client that increments a count every time it creates a new Connector + let creation_count = Arc::new(AtomicU32::new(0)); + let http_client = build_with_tcp_conn_fn(None, None, { + let count = creation_count.clone(); + move || { + count.fetch_add(1, Ordering::Relaxed); + NeverConnects + } + }); + + // This configuration should result in 4 separate connectors with different timeout settings + let settings = [ + HttpConnectorSettings::builder() + .connect_timeout(Duration::from_secs(3)) + .build(), + HttpConnectorSettings::builder() + .read_timeout(Duration::from_secs(3)) + .build(), + HttpConnectorSettings::builder() + .connect_timeout(Duration::from_secs(3)) + .read_timeout(Duration::from_secs(3)) + .build(), + HttpConnectorSettings::builder() + .connect_timeout(Duration::from_secs(5)) + .read_timeout(Duration::from_secs(3)) + .build(), + ]; + + // Kick off thousands of parallel tasks that will try to create a connector + let components = RuntimeComponentsBuilder::for_tests() + .with_time_source(Some(SystemTimeSource::new())) + .build() + .unwrap(); + let mut handles = Vec::new(); + for setting in &settings { + for _ in 0..1000 { + let client = http_client.clone(); + handles.push(tokio::spawn({ + let setting = setting.clone(); + let components = components.clone(); + async move { + let _ = client.http_connector(&setting, &components); + } + })); + } + } + for handle in handles { + handle.await.unwrap(); + } + + // Verify only 4 connectors were created amidst the chaos + assert_eq!(4, creation_count.load(Ordering::Relaxed)); + } + + #[tokio::test] + async fn hyper_io_error() { + let connector = TestConnection { + inner: HangupStream, + }; + let adapter = Connector::builder().wrap_connector(connector).adapter; + let err = adapter + .call(HttpRequest::get("https://socket-hangup.com").unwrap()) + .await + .expect_err("socket hangup"); + assert!(err.is_io(), "unexpected error type: {:?}", err); + } + + // ---- machinery to make a Hyper connector that responds with an IO Error + #[derive(Clone)] + struct HangupStream; + + impl Connection for HangupStream { + fn connected(&self) -> Connected { + Connected::new() + } + } + + impl Read for HangupStream { + fn poll_read( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: ReadBufCursor<'_>, + ) -> Poll> { + Poll::Ready(Err(Error::new( + ErrorKind::ConnectionReset, + "connection reset", + ))) + } + } + + impl Write for HangupStream { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &[u8], + ) -> Poll> { + Poll::Pending + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } + } + + #[derive(Clone)] + struct TestConnection { + inner: T, + } + + impl tower::Service for TestConnection + where + T: Clone + Connection, + { + type Response = T; + type Error = BoxError; + type Future = std::future::Ready>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _req: Uri) -> Self::Future { + std::future::ready(Ok(self.inner.clone())) + } + } + + #[tokio::test] + async fn http_connect_timeout_works() { + let tcp_connector = NeverConnects::default(); + let connector_settings = HttpConnectorSettings::builder() + .connect_timeout(Duration::from_secs(1)) + .build(); + let hyper = Connector::builder() + .connector_settings(connector_settings) + .sleep_impl(SharedAsyncSleep::new(TokioSleep::new())) + .wrap_connector(tcp_connector) + .adapter; + let now = tokio::time::Instant::now(); + tokio::time::pause(); + let resp = hyper + .call(HttpRequest::get("https://static-uri.com").unwrap()) + .await + .unwrap_err(); + assert!( + resp.is_timeout(), + "expected resp.is_timeout() to be true but it was false, resp == {:?}", + resp + ); + let message = DisplayErrorContext(&resp).to_string(); + let expected = "timeout: client error (Connect): HTTP connect timeout occurred after 1s"; + assert!( + message.contains(expected), + "expected '{message}' to contain '{expected}'" + ); + assert_elapsed!(now, Duration::from_secs(1)); + } + + #[tokio::test] + async fn http_read_timeout_works() { + let tcp_connector = crate::client::timeout::test::NeverReplies; + let connector_settings = HttpConnectorSettings::builder() + .connect_timeout(Duration::from_secs(1)) + .read_timeout(Duration::from_secs(2)) + .build(); + let hyper = Connector::builder() + .connector_settings(connector_settings) + .sleep_impl(SharedAsyncSleep::new(TokioSleep::new())) + .wrap_connector(tcp_connector) + .adapter; + let now = tokio::time::Instant::now(); + tokio::time::pause(); + let err = hyper + .call(HttpRequest::get("https://fake-uri.com").unwrap()) + .await + .unwrap_err(); + assert!( + err.is_timeout(), + "expected err.is_timeout() to be true but it was false, err == {err:?}", + ); + let message = format!("{}", DisplayErrorContext(&err)); + let expected = "timeout: HTTP read timeout occurred after 2s"; + assert!( + message.contains(expected), + "expected '{message}' to contain '{expected}'" + ); + assert_elapsed!(now, Duration::from_secs(2)); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn connection_refused_works() { + use crate::client::dns::HyperUtilResolver; + use aws_smithy_runtime_api::client::dns::{DnsFuture, ResolveDns}; + use std::net::{IpAddr, Ipv4Addr}; + + #[derive(Debug, Clone, Default)] + struct TestResolver; + impl ResolveDns for TestResolver { + fn resolve_dns<'a>(&'a self, _name: &'a str) -> DnsFuture<'a> { + let localhost_v4 = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); + DnsFuture::ready(Ok(vec![localhost_v4])) + } + } + + let connector_settings = HttpConnectorSettings::builder() + .connect_timeout(Duration::from_secs(20)) + .build(); + + let resolver = HyperUtilResolver { + resolver: TestResolver, + }; + let connector = Connector::builder().base_connector_with_resolver(resolver); + + let hyper = Connector::builder() + .connector_settings(connector_settings) + .sleep_impl(SharedAsyncSleep::new(TokioSleep::new())) + .wrap_connector(connector) + .adapter; + + let resp = hyper + .call(HttpRequest::get("http://static-uri:50227.com").unwrap()) + .await + .unwrap_err(); + assert!( + resp.is_io(), + "expected resp.is_io() to be true but it was false, resp == {:?}", + resp + ); + let message = DisplayErrorContext(&resp).to_string(); + let expected = "Connection refused"; + assert!( + message.contains(expected), + "expected '{message}' to contain '{expected}'" + ); + } + + #[cfg(feature = "s2n-tls")] + #[tokio::test] + async fn s2n_tls_provider() { + // Create an HttpConnector with the s2n-tls provider. + let client = Builder::new() + .tls_provider(tls::Provider::S2nTls) + .build_https(); + let connector_settings = HttpConnectorSettings::builder().build(); + + // HyperClient::http_connector invokes TimeSource::now to determine how long it takes to + // create new HttpConnectors. As such, a real time source must be provided. + let runtime_components = RuntimeComponentsBuilder::for_tests() + .with_time_source(Some(SystemTimeSource::new())) + .build() + .unwrap(); + + let connector = client.http_connector(&connector_settings, &runtime_components); + + // Ensure that s2n-tls is used as the underlying TLS provider when selected. + // + // s2n-tls-hyper will error when given an invalid scheme. Ensure that this error is produced + // from s2n-tls-hyper, and not another TLS provider. + let error = connector + .call(HttpRequest::get("notascheme://amazon.com").unwrap()) + .await + .unwrap_err(); + let error = error.into_source(); + let s2n_error = error + .source() + .unwrap() + .downcast_ref::() + .unwrap(); + assert!(matches!( + s2n_error, + s2n_tls_hyper::error::Error::InvalidScheme + )); + } +} diff --git a/sources/aws-smithy-http-client/src/client/connect.rs b/sources/aws-smithy-http-client/src/client/connect.rs new file mode 100644 index 000000000..79ea90318 --- /dev/null +++ b/sources/aws-smithy-http-client/src/client/connect.rs @@ -0,0 +1,164 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +use crate::client::connect; +use crate::proxy; +use aws_smithy_runtime_api::box_error::BoxError; +use http_1x::Uri; +use hyper::rt::{Read, ReadBufCursor, Write}; +use hyper_util::client::legacy::connect::{Connected, Connection}; +use pin_project_lite::pin_project; +use std::future::Future; +use std::io; +use std::io::IoSlice; +use std::pin::Pin; +use std::task::{Context, Poll}; + +pub(crate) trait AsyncConn: + Read + Write + Connection + Send + Sync + Unpin + 'static +{ +} + +impl AsyncConn for T {} + +pub(crate) type BoxConn = Box; + +// Future for connecting +pub(crate) type Connecting = Pin> + Send>>; + +pin_project! { + pub(crate) struct Conn { + #[pin] + pub(super)inner: BoxConn, + pub(super) is_proxy: bool, + } +} + +impl Connection for Conn { + fn connected(&self) -> Connected { + self.inner.connected().proxy(self.is_proxy) + } +} + +impl Read for Conn { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: ReadBufCursor<'_>, + ) -> Poll> { + let this = self.project(); + Read::poll_read(this.inner, cx, buf) + } +} + +impl Write for Conn { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let this = self.project(); + Write::poll_write(this.inner, cx, buf) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[IoSlice<'_>], + ) -> Poll> { + let this = self.project(); + Write::poll_write_vectored(this.inner, cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.project(); + Write::poll_flush(this.inner, cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.project(); + Write::poll_shutdown(this.inner, cx) + } +} + +/// HTTP-only proxy connector for handling HTTP requests through HTTP proxies +/// +/// This connector handles the HTTP proxy logic when no TLS provider is selected, +/// including request URL modification and proxy authentication. +#[derive(Debug, Clone)] +pub(crate) struct HttpProxyConnector { + inner: C, + proxy_config: proxy::ProxyConfig, +} + +impl HttpProxyConnector { + pub(crate) fn new(inner: C, proxy_config: proxy::ProxyConfig) -> Self { + Self { + inner, + proxy_config, + } + } +} + +impl tower::Service for HttpProxyConnector +where + C: tower::Service + Clone + Send + 'static, + C::Response: hyper::rt::Read + + hyper::rt::Write + + hyper_util::client::legacy::connect::Connection + + Send + + Sync + + Unpin + + 'static, + C::Future: Send + 'static, + C::Error: Into, +{ + type Response = connect::Conn; + type Error = BoxError; + type Future = connect::Connecting; + + fn poll_ready( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.inner.poll_ready(cx).map_err(Into::into) + } + + fn call(&mut self, dst: Uri) -> Self::Future { + // Check if this request should be proxied + let proxy_intercept = if !self.proxy_config.is_disabled() { + let matcher = self.proxy_config.clone().into_hyper_util_matcher(); + matcher.intercept(&dst) + } else { + None + }; + + if let Some(intercept) = proxy_intercept { + // HTTP through proxy: Connect to proxy server + let proxy_uri = intercept.uri().clone(); + let fut = self.inner.call(proxy_uri); + Box::pin(async move { + let conn = fut.await.map_err(Into::into)?; + Ok(connect::Conn { + inner: Box::new(conn), + is_proxy: true, + }) + }) + } else { + // Direct connection + let fut = self.inner.call(dst); + Box::pin(async move { + let conn = fut.await.map_err(Into::into)?; + Ok(connect::Conn { + inner: Box::new(conn), + is_proxy: false, + }) + }) + } + } +} diff --git a/sources/aws-smithy-http-client/src/client/dns.rs b/sources/aws-smithy-http-client/src/client/dns.rs new file mode 100644 index 000000000..bea83aff2 --- /dev/null +++ b/sources/aws-smithy-http-client/src/client/dns.rs @@ -0,0 +1,41 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +use aws_smithy_runtime_api::client::dns::ResolveDns; +use hyper_util::client::legacy::connect::dns::Name; +use std::error::Error; +use std::future::Future; +use std::net::SocketAddr; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::vec; + +/// A bridge that allows our `ResolveDns` trait to work with Hyper's `Resolver` interface (based on tower) +#[derive(Clone)] +#[allow(dead_code)] +pub(crate) struct HyperUtilResolver { + pub(crate) resolver: R, +} + +impl tower::Service for HyperUtilResolver { + type Response = vec::IntoIter; + type Error = Box; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, req: Name) -> Self::Future { + let resolver = self.resolver.clone(); + Box::pin(async move { + let dns_entries = resolver.resolve_dns(req.as_str()).await?; + Ok(dns_entries + .into_iter() + .map(|ip_addr| SocketAddr::new(ip_addr, 0)) + .collect::>() + .into_iter()) + }) + } +} diff --git a/sources/aws-smithy-http-client/src/client/proxy.rs b/sources/aws-smithy-http-client/src/client/proxy.rs new file mode 100644 index 000000000..dfeaacfbd --- /dev/null +++ b/sources/aws-smithy-http-client/src/client/proxy.rs @@ -0,0 +1,816 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Proxy configuration for HTTP clients +//! +//! This module provides types and utilities for configuring HTTP and HTTPS proxies, +//! including support for environment variable detection, authentication, and bypass rules. + +use http_1x::Uri; +use hyper_util::client::proxy::matcher::Matcher; +use std::fmt; + +/// Proxy configuration for HTTP clients +/// +/// Supports HTTP and HTTPS proxy configuration with authentication and bypass rules. +/// Can be configured programmatically or automatically detected from environment variables. +/// +/// # Examples +/// +/// ```rust +/// use aws_smithy_http_client::proxy::ProxyConfig; +/// +/// // HTTP proxy for all traffic +/// let config = ProxyConfig::http("http://proxy.example.com:8080")?; +/// +/// // HTTPS traffic through HTTP proxy (common case - no TLS needed for proxy connection) +/// let config = ProxyConfig::https("http://proxy.example.com:8080")? +/// .with_basic_auth("username", "password") +/// .no_proxy("localhost,*.internal"); +/// +/// // Detect from environment variables +/// let config = ProxyConfig::from_env(); +/// # Ok::<(), Box>(()) +/// ``` +#[derive(Debug, Clone)] +pub struct ProxyConfig { + inner: ProxyConfigInner, +} + +/// Internal configuration representation +#[derive(Debug, Clone)] +enum ProxyConfigInner { + /// Use environment variable detection + FromEnvironment, + /// Explicit HTTP proxy + Http { + uri: Uri, + auth: Option, + no_proxy: Option, + }, + /// Explicit HTTPS proxy + Https { + uri: Uri, + auth: Option, + no_proxy: Option, + }, + /// Proxy for all traffic + All { + uri: Uri, + auth: Option, + no_proxy: Option, + }, + /// Explicitly disabled + Disabled, +} + +/// Proxy authentication configuration +/// +/// Stored for later conversion to hyper-util format. +#[derive(Debug, Clone)] +struct ProxyAuth { + /// Username for authentication + username: String, + /// Password for authentication + password: String, +} + +/// Errors that can occur during proxy configuration +#[derive(Debug)] +pub struct ProxyError { + kind: ErrorKind, +} + +#[derive(Debug)] +enum ErrorKind { + InvalidUrl(String), +} + +impl From for ProxyError { + fn from(value: ErrorKind) -> Self { + Self { kind: value } + } +} + +impl fmt::Display for ProxyError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.kind { + ErrorKind::InvalidUrl(url) => write!(f, "invalid proxy URL: {url}"), + } + } +} + +impl std::error::Error for ProxyError {} + +impl ProxyConfig { + /// Create a new proxy configuration for HTTP traffic only + /// + /// # Arguments + /// * `proxy_url` - The HTTP proxy URL + /// + /// # Examples + /// ```rust + /// use aws_smithy_http_client::proxy::ProxyConfig; + /// + /// let config = ProxyConfig::http("http://proxy.example.com:8080")?; + /// # Ok::<(), Box>(()) + /// ``` + pub fn http(proxy_url: U) -> Result + where + U: TryInto, + U::Error: fmt::Display, + { + let uri = proxy_url + .try_into() + .map_err(|e| ErrorKind::InvalidUrl(e.to_string()))?; + + Self::validate_proxy_uri(&uri)?; + + Ok(ProxyConfig { + inner: ProxyConfigInner::Http { + uri, + auth: None, + no_proxy: None, + }, + }) + } + + /// Create a new proxy configuration for HTTPS traffic only + /// + /// This proxy will only be used for `https://` requests. HTTP requests + /// will connect directly unless a separate HTTP proxy is configured. + /// + /// The proxy URL itself can use either HTTP or HTTPS scheme: + /// - `http://proxy.example.com:8080` - Connect to proxy using HTTP (no TLS needed) + /// - `https://proxy.example.com:8080` - Connect to proxy using HTTPS (TLS required) + /// + /// **Note**: If the proxy URL itself uses HTTPS scheme, TLS support must be + /// available when building the connector, otherwise connections will fail. + /// + /// # Arguments + /// * `proxy_url` - The proxy URL + /// + /// # Examples + /// ```rust + /// use aws_smithy_http_client::proxy::ProxyConfig; + /// + /// // HTTPS traffic through HTTP proxy (no TLS needed for proxy connection) + /// let config = ProxyConfig::https("http://proxy.example.com:8080")?; + /// + /// // HTTPS traffic through HTTPS proxy (TLS needed for proxy connection) + /// let config = ProxyConfig::https("https://secure-proxy.example.com:8080")?; + /// # Ok::<(), Box>(()) + /// ``` + pub fn https(proxy_url: U) -> Result + where + U: TryInto, + U::Error: fmt::Display, + { + let uri = proxy_url + .try_into() + .map_err(|e| ErrorKind::InvalidUrl(e.to_string()))?; + + Self::validate_proxy_uri(&uri)?; + + Ok(ProxyConfig { + inner: ProxyConfigInner::Https { + uri, + auth: None, + no_proxy: None, + }, + }) + } + + /// Create a new proxy configuration for all HTTP and HTTPS traffic + /// + /// This proxy will be used for both `http://` and `https://` requests. + /// This is equivalent to setting both HTTP and HTTPS proxies to the same URL. + /// + /// **Note**: If the proxy URL itself uses HTTPS scheme, TLS support must be + /// available when building the connector, otherwise connections will fail. + /// + /// # Arguments + /// * `proxy_url` - The proxy URL + /// + /// # Examples + /// ```rust + /// use aws_smithy_http_client::proxy::ProxyConfig; + /// + /// let config = ProxyConfig::all("http://proxy.example.com:8080")?; + /// # Ok::<(), Box>(()) + /// ``` + pub fn all(proxy_url: U) -> Result + where + U: TryInto, + U::Error: fmt::Display, + { + let uri = proxy_url + .try_into() + .map_err(|e| ErrorKind::InvalidUrl(e.to_string()))?; + + Self::validate_proxy_uri(&uri)?; + + Ok(ProxyConfig { + inner: ProxyConfigInner::All { + uri, + auth: None, + no_proxy: None, + }, + }) + } + + /// Create a proxy configuration that disables all proxy usage + /// + /// This is useful for explicitly disabling proxy support even when + /// environment variables are set. + /// + /// # Examples + /// ```rust + /// use aws_smithy_http_client::proxy::ProxyConfig; + /// + /// let config = ProxyConfig::disabled(); + /// ``` + pub fn disabled() -> Self { + ProxyConfig { + inner: ProxyConfigInner::Disabled, + } + } + + /// Add basic authentication to this proxy configuration + /// + /// # Arguments + /// * `username` - Username for proxy authentication + /// * `password` - Password for proxy authentication + /// + /// # Examples + /// ```rust + /// use aws_smithy_http_client::proxy::ProxyConfig; + /// + /// let config = ProxyConfig::http("http://proxy.example.com:8080")? + /// .with_basic_auth("username", "password"); + /// # Ok::<(), Box>(()) + /// ``` + pub fn with_basic_auth(mut self, username: U, password: P) -> Self + where + U: Into, + P: Into, + { + let auth = ProxyAuth { + username: username.into(), + password: password.into(), + }; + + match &mut self.inner { + ProxyConfigInner::Http { + auth: ref mut a, .. + } => *a = Some(auth), + ProxyConfigInner::Https { + auth: ref mut a, .. + } => *a = Some(auth), + ProxyConfigInner::All { + auth: ref mut a, .. + } => *a = Some(auth), + ProxyConfigInner::FromEnvironment | ProxyConfigInner::Disabled => { + // Cannot add auth to environment or disabled configs + } + } + + self + } + + /// Add NO_PROXY rules to this configuration + /// + /// NO_PROXY rules specify hosts that should bypass the proxy and connect directly. + /// + /// # Arguments + /// * `rules` - Comma-separated list of bypass rules + /// + /// # Examples + /// ```rust + /// use aws_smithy_http_client::proxy::ProxyConfig; + /// + /// let config = ProxyConfig::http("http://proxy.example.com:8080")? + /// .no_proxy("localhost,127.0.0.1,*.internal,10.0.0.0/8"); + /// # Ok::<(), Box>(()) + /// ``` + pub fn no_proxy>(mut self, rules: S) -> Self { + let rules_str = rules.as_ref().to_string(); + + match &mut self.inner { + ProxyConfigInner::Http { + no_proxy: ref mut n, + .. + } => *n = Some(rules_str), + ProxyConfigInner::Https { + no_proxy: ref mut n, + .. + } => *n = Some(rules_str), + ProxyConfigInner::All { + no_proxy: ref mut n, + .. + } => *n = Some(rules_str), + ProxyConfigInner::FromEnvironment | ProxyConfigInner::Disabled => { + // Cannot add no_proxy to environment or disabled configs + // Environment configs will use NO_PROXY env var + // FIXME - is this what we want? + } + } + + self + } + + /// Create proxy configuration from environment variables + /// + /// Reads standard proxy environment variables: + /// - `HTTP_PROXY` / `http_proxy`: HTTP proxy URL + /// - `HTTPS_PROXY` / `https_proxy`: HTTPS proxy URL + /// - `ALL_PROXY` / `all_proxy`: Proxy for all protocols (fallback) + /// - `NO_PROXY` / `no_proxy`: Comma-separated bypass rules + /// + /// If no proxy environment variables are set, this returns a configuration + /// that won't intercept any requests (equivalent to no proxy). + /// + /// # Examples + /// ```rust + /// use aws_smithy_http_client::proxy::ProxyConfig; + /// + /// // Always succeeds, even if no environment variables are set + /// let config = ProxyConfig::from_env(); + /// ``` + pub fn from_env() -> Self { + // Delegate to environment variable parsing + // If no env vars are set, creates a matcher that doesn't intercept anything + ProxyConfig { + inner: ProxyConfigInner::FromEnvironment, + } + } + + /// Check if proxy is disabled (no proxy configuration) + pub fn is_disabled(&self) -> bool { + matches!(self.inner, ProxyConfigInner::Disabled) + } + + /// Check if this configuration uses environment variables + pub fn is_from_env(&self) -> bool { + matches!(self.inner, ProxyConfigInner::FromEnvironment) + } + + /// Convert this configuration to internal proxy matcher + /// + /// This method converts the user-friendly configuration to the internal + /// proxy matching implementation used by the HTTP client. + pub(crate) fn into_hyper_util_matcher(self) -> Matcher { + match self.inner { + ProxyConfigInner::FromEnvironment => Matcher::from_env(), + ProxyConfigInner::Http { + uri, + auth, + no_proxy, + } => { + let mut builder = Matcher::builder(); + + // Set HTTP proxy with authentication embedded in URL if present + let proxy_url = Self::build_proxy_url(uri, auth); + builder = builder.http(proxy_url); + + // Add NO_PROXY rules if present + if let Some(no_proxy_rules) = no_proxy { + builder = builder.no(no_proxy_rules); + } + + builder.build() + } + ProxyConfigInner::Https { + uri, + auth, + no_proxy, + } => { + let mut builder = Matcher::builder(); + + // Set HTTPS proxy with authentication embedded in URL if present + let proxy_url = Self::build_proxy_url(uri, auth); + builder = builder.https(proxy_url); + + // Add NO_PROXY rules if present + if let Some(no_proxy_rules) = no_proxy { + builder = builder.no(no_proxy_rules); + } + + builder.build() + } + ProxyConfigInner::All { + uri, + auth, + no_proxy, + } => { + let mut builder = Matcher::builder(); + + // Set proxy for all traffic with authentication embedded in URL if present + let proxy_url = Self::build_proxy_url(uri, auth); + builder = builder.all(proxy_url); + + // Add NO_PROXY rules if present + if let Some(no_proxy_rules) = no_proxy { + builder = builder.no(no_proxy_rules); + } + + builder.build() + } + ProxyConfigInner::Disabled => { + // Create an empty matcher that won't intercept anything + Matcher::builder().build() + } + } + } + + /// Check if this proxy configuration requires TLS support + /// + /// Returns true if any of the configured proxy URLs use HTTPS scheme, + /// which requires TLS to establish the connection to the proxy server. + pub(crate) fn requires_tls(&self) -> bool { + match &self.inner { + ProxyConfigInner::Http { uri, .. } => uri.scheme_str() == Some("https"), + ProxyConfigInner::Https { uri, .. } => uri.scheme_str() == Some("https"), + ProxyConfigInner::All { uri, .. } => uri.scheme_str() == Some("https"), + ProxyConfigInner::FromEnvironment => { + // Check environment variables for HTTPS proxy URLs + Self::env_vars_require_tls() + } + ProxyConfigInner::Disabled => false, + } + } + + /// Check if any environment proxy variables contain HTTPS URLs + fn env_vars_require_tls() -> bool { + let proxy_vars = [ + "HTTP_PROXY", + "http_proxy", + "HTTPS_PROXY", + "https_proxy", + "ALL_PROXY", + "all_proxy", + ]; + + for var in &proxy_vars { + if let Ok(proxy_url) = std::env::var(var) { + if !proxy_url.is_empty() { + // Simple check for https:// scheme + if proxy_url.starts_with("https://") { + return true; + } + } + } + } + false + } + + fn validate_proxy_uri(uri: &Uri) -> Result<(), ProxyError> { + // Validate scheme + match uri.scheme_str() { + Some("http") | Some("https") => {} + Some(scheme) => { + return Err( + ErrorKind::InvalidUrl(format!("unsupported proxy scheme: {scheme}")).into(), + ); + } + None => { + return Err(ErrorKind::InvalidUrl( + "proxy URL must include scheme (http:// or https://)".to_string(), + ) + .into()); + } + } + + // Validate host + if uri.host().is_none() { + return Err(ErrorKind::InvalidUrl("proxy URL must include host".to_string()).into()); + } + + Ok(()) + } + + fn build_proxy_url(uri: Uri, auth: Option) -> String { + let uri_str = uri.to_string(); + + if let Some(auth) = auth { + // Embed authentication in the URL: scheme://username:password@host:port/path + if let Some(scheme_end) = uri_str.find("://") { + let scheme = &uri_str[..scheme_end + 3]; + let rest = &uri_str[scheme_end + 3..]; + + // Check if auth is already present in the URI + if rest.contains('@') { + // Auth already present, return as-is + uri_str + } else { + // Add auth to the URI + format!("{}{}:{}@{}", scheme, auth.username, auth.password, rest) + } + } else { + // Invalid URI format, return as-is + uri_str + } + } else { + // No authentication, return URI as-is + uri_str + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + + #[test] + fn test_proxy_config_http() { + let config = ProxyConfig::http("http://proxy.example.com:8080").unwrap(); + assert!(!config.is_disabled()); + assert!(!config.is_from_env()); + } + + #[test] + fn test_proxy_config_https() { + let config = ProxyConfig::https("http://proxy.example.com:8080").unwrap(); + assert!(!config.is_disabled()); + assert!(!config.is_from_env()); + } + + #[test] + fn test_proxy_config_all() { + let config = ProxyConfig::all("http://proxy.example.com:8080").unwrap(); + assert!(!config.is_disabled()); + assert!(!config.is_from_env()); + } + + #[test] + fn test_proxy_config_disabled() { + let config = ProxyConfig::disabled(); + assert!(config.is_disabled()); + assert!(!config.is_from_env()); + } + + #[test] + fn test_proxy_config_with_auth() { + let config = ProxyConfig::http("http://proxy.example.com:8080") + .unwrap() + .with_basic_auth("user", "pass"); + + // Auth is stored internally + assert!(!config.is_disabled()); + } + + #[test] + fn test_proxy_config_with_no_proxy() { + let config = ProxyConfig::http("http://proxy.example.com:8080") + .unwrap() + .no_proxy("localhost,*.internal"); + + // NO_PROXY rules are stored internally + assert!(!config.is_disabled()); + } + + #[test] + fn test_proxy_config_invalid_url() { + let result = ProxyConfig::http("not-a-url"); + assert!(result.is_err()); + } + + #[test] + fn test_proxy_config_invalid_scheme() { + let result = ProxyConfig::http("ftp://proxy.example.com:8080"); + assert!(result.is_err()); + } + + #[test] + #[serial_test::serial] + fn test_proxy_config_from_env_with_vars() { + // Save original environment + let original_http = env::var("HTTP_PROXY"); + + // Set test environment + env::set_var("HTTP_PROXY", "http://test-proxy:8080"); + + let config = ProxyConfig::from_env(); + assert!(config.is_from_env()); + + // Restore original environment + match original_http { + Ok(val) => env::set_var("HTTP_PROXY", val), + Err(_) => env::remove_var("HTTP_PROXY"), + } + } + + #[test] + #[serial_test::serial] + fn test_proxy_config_from_env_without_vars() { + // Save original environment + let original_vars: Vec<_> = [ + "HTTP_PROXY", + "http_proxy", + "HTTPS_PROXY", + "https_proxy", + "ALL_PROXY", + "all_proxy", + ] + .iter() + .map(|var| (*var, env::var(var))) + .collect(); + + // Clear all proxy environment variables + for (var, _) in &original_vars { + env::remove_var(var); + } + + let config = ProxyConfig::from_env(); + assert!(config.is_from_env()); + + // Restore original environment + for (var, original_value) in original_vars { + match original_value { + Ok(val) => env::set_var(var, val), + Err(_) => env::remove_var(var), + } + } + } + + #[test] + #[serial_test::serial] + fn test_auth_cannot_be_added_to_env_config() { + // Save original environment + let original_http = env::var("HTTP_PROXY"); + env::set_var("HTTP_PROXY", "http://test-proxy:8080"); + + let config = ProxyConfig::from_env().with_basic_auth("user", "pass"); // This should be ignored + + assert!(config.is_from_env()); + + // Restore original environment + match original_http { + Ok(val) => env::set_var("HTTP_PROXY", val), + Err(_) => env::remove_var("HTTP_PROXY"), + } + } + + #[test] + #[serial_test::serial] + fn test_no_proxy_cannot_be_added_to_env_config() { + // Save original environment + let original_http = env::var("HTTP_PROXY"); + env::set_var("HTTP_PROXY", "http://test-proxy:8080"); + + let config = ProxyConfig::from_env().no_proxy("localhost"); // This should be ignored + + assert!(config.is_from_env()); + + // Restore original environment + match original_http { + Ok(val) => env::set_var("HTTP_PROXY", val), + Err(_) => env::remove_var("HTTP_PROXY"), + } + } + + #[test] + fn test_build_proxy_url_without_auth() { + let uri = "http://proxy.example.com:8080".parse().unwrap(); + let url = ProxyConfig::build_proxy_url(uri, None); + assert_eq!(url, "http://proxy.example.com:8080/"); + } + + #[test] + fn test_build_proxy_url_with_auth() { + let uri = "http://proxy.example.com:8080".parse().unwrap(); + let auth = ProxyAuth { + username: "user".to_string(), + password: "pass".to_string(), + }; + let url = ProxyConfig::build_proxy_url(uri, Some(auth)); + assert_eq!(url, "http://user:pass@proxy.example.com:8080/"); + } + + #[test] + fn test_build_proxy_url_with_existing_auth() { + let uri = "http://existing:creds@proxy.example.com:8080" + .parse() + .unwrap(); + let auth = ProxyAuth { + username: "user".to_string(), + password: "pass".to_string(), + }; + let url = ProxyConfig::build_proxy_url(uri, Some(auth)); + // Should not override existing auth + assert_eq!(url, "http://existing:creds@proxy.example.com:8080/"); + } + + #[test] + #[serial_test::serial] + fn test_into_hyper_util_matcher_from_env() { + // Save original environment + let original_http = env::var("HTTP_PROXY"); + env::set_var("HTTP_PROXY", "http://test-proxy:8080"); + + let config = ProxyConfig::from_env(); + let matcher = config.into_hyper_util_matcher(); + + // Test that the matcher intercepts HTTP requests + let test_uri = "http://example.com".parse().unwrap(); + let intercept = matcher.intercept(&test_uri); + assert!(intercept.is_some()); + + // Restore original environment + match original_http { + Ok(val) => env::set_var("HTTP_PROXY", val), + Err(_) => env::remove_var("HTTP_PROXY"), + } + } + + #[test] + fn test_into_hyper_util_matcher_http() { + let config = ProxyConfig::http("http://proxy.example.com:8080").unwrap(); + let matcher = config.into_hyper_util_matcher(); + + // Test that the matcher intercepts HTTP requests + let test_uri = "http://example.com".parse().unwrap(); + let intercept = matcher.intercept(&test_uri); + assert!(intercept.is_some()); + // The intercept URI might be normalized + assert!(intercept + .unwrap() + .uri() + .to_string() + .starts_with("http://proxy.example.com:8080")); + + // Test that it doesn't intercept HTTPS requests + let https_uri = "https://example.com".parse().unwrap(); + let https_intercept = matcher.intercept(&https_uri); + assert!(https_intercept.is_none()); + } + + #[test] + fn test_into_hyper_util_matcher_with_auth() { + let config = ProxyConfig::http("http://proxy.example.com:8080") + .unwrap() + .with_basic_auth("user", "pass"); + let matcher = config.into_hyper_util_matcher(); + + // Test that the matcher intercepts HTTP requests + let test_uri = "http://example.com".parse().unwrap(); + let intercept = matcher.intercept(&test_uri); + assert!(intercept.is_some()); + + let intercept = intercept.unwrap(); + // The proxy URI should contain the host (auth is handled separately) + assert!(intercept + .uri() + .to_string() + .contains("proxy.example.com:8080")); + + // Test that basic auth is available + assert!(intercept.basic_auth().is_some()); + } + + #[test] + fn test_into_hyper_util_matcher_disabled() { + let config = ProxyConfig::disabled(); + let matcher = config.into_hyper_util_matcher(); + + // Test that the matcher doesn't intercept any requests + let test_uri = "http://example.com".parse().unwrap(); + let intercept = matcher.intercept(&test_uri); + assert!(intercept.is_none()); + } + + #[test] + #[serial_test::serial] + fn test_requires_tls_detection() { + // HTTP proxy should not require TLS + let http_config = ProxyConfig::http("http://proxy.example.com:8080").unwrap(); + assert!(!http_config.requires_tls()); + + // HTTPS proxy URL should require TLS + let https_config = ProxyConfig::http("https://proxy.example.com:8080").unwrap(); + assert!(https_config.requires_tls()); + + // All proxy with HTTP URL should not require TLS + let all_http_config = ProxyConfig::all("http://proxy.example.com:8080").unwrap(); + assert!(!all_http_config.requires_tls()); + + // Environment config with HTTPS proxy should require TLS + env::set_var("HTTP_PROXY", "https://proxy.example.com:8080"); + let env_config = ProxyConfig::from_env(); + assert!(env_config.requires_tls()); // Now detects HTTPS in env vars + env::remove_var("HTTP_PROXY"); + + // Environment config with HTTP proxy should not require TLS + env::set_var("HTTP_PROXY", "http://proxy.example.com:8080"); + let env_config = ProxyConfig::from_env(); + assert!(!env_config.requires_tls()); + env::remove_var("HTTP_PROXY"); + + // Disabled config should not require TLS + let disabled_config = ProxyConfig::disabled(); + assert!(!disabled_config.requires_tls()); + } +} diff --git a/sources/aws-smithy-http-client/src/client/timeout.rs b/sources/aws-smithy-http-client/src/client/timeout.rs new file mode 100644 index 000000000..56c3f153d --- /dev/null +++ b/sources/aws-smithy-http-client/src/client/timeout.rs @@ -0,0 +1,316 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use std::error::Error; +use std::fmt::Formatter; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::Duration; + +use http_1x::Uri; +use pin_project_lite::pin_project; + +use aws_smithy_async::future::timeout::{TimedOutError, Timeout}; +use aws_smithy_async::rt::sleep::Sleep; +use aws_smithy_async::rt::sleep::{AsyncSleep, SharedAsyncSleep}; +use aws_smithy_runtime_api::box_error::BoxError; + +#[derive(Debug)] +pub(crate) struct HttpTimeoutError { + kind: &'static str, + duration: Duration, +} + +impl std::fmt::Display for HttpTimeoutError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} timeout occurred after {:?}", + self.kind, self.duration + ) + } +} + +impl Error for HttpTimeoutError { + // We implement the `source` function as returning a `TimedOutError` because when `downcast_error` + // or `find_source` is called with an `HttpTimeoutError` (or another error wrapping an `HttpTimeoutError`) + // this method will be checked to determine if it's a timeout-related error. + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(&TimedOutError) + } +} + +/// Timeout wrapper that will timeout on the initial TCP connection +/// +/// # Stability +/// This interface is unstable. +#[derive(Clone, Debug)] +pub(crate) struct ConnectTimeout { + inner: I, + timeout: Option<(SharedAsyncSleep, Duration)>, +} + +impl ConnectTimeout { + /// Create a new `ConnectTimeout` around `inner`. + /// + /// Typically, `I` will implement [`hyper_util::client::legacy::connect::Connect`]. + pub(crate) fn new(inner: I, sleep: SharedAsyncSleep, timeout: Duration) -> Self { + Self { + inner, + timeout: Some((sleep, timeout)), + } + } + + pub(crate) fn no_timeout(inner: I) -> Self { + Self { + inner, + timeout: None, + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct HttpReadTimeout { + inner: I, + timeout: Option<(SharedAsyncSleep, Duration)>, +} + +impl HttpReadTimeout { + /// Create a new `HttpReadTimeout` around `inner`. + /// + /// Typically, `I` will implement [`tower::Service>`]. + pub(crate) fn new(inner: I, sleep: SharedAsyncSleep, timeout: Duration) -> Self { + Self { + inner, + timeout: Some((sleep, timeout)), + } + } + + pub(crate) fn no_timeout(inner: I) -> Self { + Self { + inner, + timeout: None, + } + } +} + +pin_project! { + /// Timeout future for Tower services + /// + /// Timeout future to handle timing out, mapping errors, and the possibility of not timing out + /// without incurring an additional allocation for each timeout layer. + #[project = MaybeTimeoutFutureProj] + pub enum MaybeTimeoutFuture { + Timeout { + #[pin] + timeout: Timeout, + error_type: &'static str, + duration: Duration, + }, + NoTimeout { + #[pin] + future: F + } + } +} + +impl Future for MaybeTimeoutFuture +where + F: Future>, + E: Into, +{ + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let (timeout_future, kind, &mut duration) = match self.project() { + MaybeTimeoutFutureProj::NoTimeout { future } => { + return future.poll(cx).map_err(|err| err.into()); + } + MaybeTimeoutFutureProj::Timeout { + timeout, + error_type, + duration, + } => (timeout, error_type, duration), + }; + match timeout_future.poll(cx) { + Poll::Ready(Ok(response)) => Poll::Ready(response.map_err(|err| err.into())), + Poll::Ready(Err(_timeout)) => { + Poll::Ready(Err(HttpTimeoutError { kind, duration }.into())) + } + Poll::Pending => Poll::Pending, + } + } +} + +impl tower::Service for ConnectTimeout +where + I: tower::Service, + I::Error: Into, +{ + type Response = I::Response; + type Error = BoxError; + type Future = MaybeTimeoutFuture; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(|err| err.into()) + } + + fn call(&mut self, req: Uri) -> Self::Future { + match &self.timeout { + Some((sleep, duration)) => { + let sleep = sleep.sleep(*duration); + MaybeTimeoutFuture::Timeout { + timeout: Timeout::new(self.inner.call(req), sleep), + error_type: "HTTP connect", + duration: *duration, + } + } + None => MaybeTimeoutFuture::NoTimeout { + future: self.inner.call(req), + }, + } + } +} + +impl tower::Service> for HttpReadTimeout +where + I: tower::Service>, + I::Error: Send + Sync + Error + 'static, +{ + type Response = I::Response; + type Error = BoxError; + type Future = MaybeTimeoutFuture; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(|err| err.into()) + } + + fn call(&mut self, req: http_1x::Request) -> Self::Future { + match &self.timeout { + Some((sleep, duration)) => { + let sleep = sleep.sleep(*duration); + MaybeTimeoutFuture::Timeout { + timeout: Timeout::new(self.inner.call(req), sleep), + error_type: "HTTP read", + duration: *duration, + } + } + None => MaybeTimeoutFuture::NoTimeout { + future: self.inner.call(req), + }, + } + } +} + +#[cfg(test)] +pub(crate) mod test { + use hyper::rt::ReadBufCursor; + use hyper_util::client::legacy::connect::{Connected, Connection}; + use hyper_util::rt::TokioIo; + use tokio::net::TcpStream; + + use aws_smithy_async::future::never::Never; + + use aws_smithy_runtime_api::box_error::BoxError; + use aws_smithy_runtime_api::client::result::ConnectorError; + use http::Uri; + use hyper::http; + use hyper::rt::{Read, Write}; + use std::future::Future; + use std::pin::Pin; + use std::task::{Context, Poll}; + + #[allow(unused)] + fn connect_timeout_is_correct() { + is_send_sync::>(); + } + + #[allow(unused)] + fn is_send_sync() {} + + /// A service that will never return whatever it is you want + /// + /// Returned futures will return Pending forever + #[non_exhaustive] + #[derive(Clone, Default, Debug)] + pub(crate) struct NeverConnects; + impl tower::Service for NeverConnects { + type Response = TokioIo; + type Error = ConnectorError; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _uri: Uri) -> Self::Future { + Box::pin(async move { + Never::new().await; + unreachable!() + }) + } + } + + /// A service that will connect but never send any data + #[derive(Clone, Debug, Default)] + pub(crate) struct NeverReplies; + impl tower::Service for NeverReplies { + type Response = EmptyStream; + type Error = BoxError; + type Future = std::future::Ready>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _req: Uri) -> Self::Future { + std::future::ready(Ok(EmptyStream)) + } + } + + /// A stream that will never return or accept any data + #[non_exhaustive] + #[derive(Debug, Default)] + pub(crate) struct EmptyStream; + impl Read for EmptyStream { + fn poll_read( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: ReadBufCursor<'_>, + ) -> Poll> { + Poll::Pending + } + } + impl Write for EmptyStream { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &[u8], + ) -> Poll> { + Poll::Pending + } + + fn poll_flush( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + + fn poll_shutdown( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + } + + impl Connection for EmptyStream { + fn connected(&self) -> Connected { + Connected::new() + } + } +} diff --git a/sources/aws-smithy-http-client/src/client/tls.rs b/sources/aws-smithy-http-client/src/client/tls.rs new file mode 100644 index 000000000..977b66a63 --- /dev/null +++ b/sources/aws-smithy-http-client/src/client/tls.rs @@ -0,0 +1,148 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +use crate::cfg::{cfg_rustls, cfg_s2n_tls}; +use crate::HttpClientError; + +/// Choice of underlying cryptography library +#[derive(Debug, PartialEq, Clone)] +#[non_exhaustive] +pub enum Provider { + #[cfg(feature = "__rustls")] + /// TLS provider based on [rustls](https://github.com/rustls/rustls) + Rustls(rustls_provider::CryptoMode), + /// TLS provider based on [s2n-tls](https://github.com/aws/s2n-tls) + #[cfg(feature = "s2n-tls")] + S2nTls, +} + +#[cfg(not(all(aws_sdk_unstable, feature = "__rustls")))] +impl Eq for Provider {} + +/// TLS related configuration object +#[derive(Debug, Clone)] +pub struct TlsContext { + #[allow(unused)] + trust_store: TrustStore, +} + +impl TlsContext { + /// Create a new [TlsContext] builder + pub fn builder() -> TlsContextBuilder { + TlsContextBuilder::new() + } +} + +impl Default for TlsContext { + fn default() -> Self { + TlsContext::builder().build().expect("valid default config") + } +} + +/// Builder for TLS related configuration +#[derive(Debug)] +pub struct TlsContextBuilder { + trust_store: TrustStore, +} + +impl TlsContextBuilder { + fn new() -> Self { + TlsContextBuilder { + trust_store: TrustStore::default(), + } + } + + /// Configure the trust store to use for the TLS context + pub fn with_trust_store(mut self, trust_store: TrustStore) -> Self { + self.trust_store = trust_store; + self + } + + /// Build a new [TlsContext] + pub fn build(self) -> Result { + Ok(TlsContext { + trust_store: self.trust_store, + }) + } +} + +/// PEM encoded certificate +#[allow(unused)] +#[derive(Debug, Clone)] +struct CertificatePEM(Vec); + +impl From<&[u8]> for CertificatePEM { + fn from(value: &[u8]) -> Self { + CertificatePEM(value.to_vec()) + } +} + +/// Container for root certificates able to provide a root-of-trust for connection authentication +/// +/// Platform native root certificates are enabled by default. To start with a clean trust +/// store use [TrustStore::empty] +#[derive(Debug, Clone)] +pub struct TrustStore { + enable_native_roots: bool, + custom_certs: Vec, +} + +impl TrustStore { + /// Create a new empty trust store + pub fn empty() -> Self { + Self { + enable_native_roots: false, + custom_certs: Vec::new(), + } + } + + /// Enable or disable using the platform's native trusted root certificate store + /// + /// Default: true + pub fn with_native_roots(mut self, enable_native_roots: bool) -> Self { + self.enable_native_roots = enable_native_roots; + self + } + + /// Add the PEM encoded certificate to the trust store + /// + /// This may be called more than once to add multiple certificates. + /// NOTE: PEM certificate contents are not validated until passed to the configured + /// TLS provider. + pub fn with_pem_certificate(mut self, pem_bytes: impl Into>) -> Self { + // ideally we'd validate here but rustls-pki-types converts to DER when loading and S2N + // still expects PEM encoding. Store the raw bytes and let the TLS implementation validate + self.custom_certs.push(CertificatePEM(pem_bytes.into())); + self + } + + /// Add the PEM encoded certificate to the trust store + /// + /// This may be called more than once to add multiple certificates. + /// NOTE: PEM certificate contents are not validated until passed to the configured + /// TLS provider. + pub fn add_pem_certificate(&mut self, pem_bytes: impl Into>) -> &mut Self { + self.custom_certs.push(CertificatePEM(pem_bytes.into())); + self + } +} + +impl Default for TrustStore { + fn default() -> Self { + Self { + enable_native_roots: true, + custom_certs: Vec::new(), + } + } +} + +cfg_rustls! { + /// rustls based support and adapters + pub mod rustls_provider; +} + +cfg_s2n_tls! { + /// s2n-tls based support and adapters + pub(crate) mod s2n_tls_provider; +} diff --git a/sources/aws-smithy-http-client/src/client/tls/rustls_provider.rs b/sources/aws-smithy-http-client/src/client/tls/rustls_provider.rs new file mode 100644 index 000000000..f8a0a1993 --- /dev/null +++ b/sources/aws-smithy-http-client/src/client/tls/rustls_provider.rs @@ -0,0 +1,472 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +use crate::client::tls::Provider; +use rustls::crypto::CryptoProvider; + +/// Choice of underlying cryptography library (this only applies to rustls) +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum CryptoMode { + /// Crypto based on [ring](https://github.com/briansmith/ring) + #[cfg(feature = "rustls-ring")] + Ring, + /// Crypto based on [aws-lc](https://github.com/aws/aws-lc-rs) + #[cfg(feature = "rustls-aws-lc")] + AwsLc, + /// FIPS compliant variant of [aws-lc](https://github.com/aws/aws-lc-rs) + #[cfg(feature = "rustls-aws-lc-fips")] + AwsLcFips, + /// Use a caller-supplied [`CryptoProvider`]. + /// + /// Unlike the built-in modes, the cipher-suite restriction normally + /// applied by smithy-rs is skipped -- the caller is expected + /// to select the applicable cipher suites via the supplied provider. + Custom(CryptoProvider), +} + +impl std::cmp::PartialEq for CryptoMode { + fn eq(&self, other: &CryptoMode) -> bool { + match (self, other) { + #[cfg(feature = "rustls-ring")] + (Self::Ring, Self::Ring) => true, + #[cfg(feature = "rustls-aws-lc")] + (Self::AwsLc, Self::AwsLc) => true, + #[cfg(feature = "rustls-aws-lc-fips")] + (Self::AwsLcFips, Self::AwsLcFips) => true, + // `CryptoProvider` does not implement PartialEq, so any + // `CryptoMode::Custom` value will always compare not equal to + // any other. + #[allow(unreachable_patterns)] + _ => false, + } + } +} + +impl CryptoMode { + fn provider(self) -> CryptoProvider { + match self { + #[cfg(feature = "rustls-aws-lc")] + CryptoMode::AwsLc => rustls::crypto::aws_lc_rs::default_provider(), + + #[cfg(feature = "rustls-ring")] + CryptoMode::Ring => rustls::crypto::ring::default_provider(), + + #[cfg(feature = "rustls-aws-lc-fips")] + CryptoMode::AwsLcFips => { + let provider = rustls::crypto::default_fips_provider(); + assert!( + provider.fips(), + "FIPS was requested but the provider did not support FIPS" + ); + provider + } + CryptoMode::Custom(provider) => provider, + } + } + + fn is_custom(&self) -> bool { + matches!(self, Self::Custom(_)) + } +} + +impl Provider { + /// Create a TLS provider based on [rustls](https://github.com/rustls/rustls) + /// and the given [`CryptoMode`] + pub fn rustls(mode: CryptoMode) -> Provider { + Provider::Rustls(mode) + } +} + +pub(crate) mod build_connector { + use crate::client::tls::rustls_provider::CryptoMode; + use crate::tls::TlsContext; + use client::connect::HttpConnector; + use hyper_util::client::legacy as client; + use rustls::crypto::CryptoProvider; + use rustls_native_certs::CertificateResult; + use rustls_pki_types::pem::PemObject; + use rustls_pki_types::CertificateDer; + use std::sync::Arc; + use std::sync::LazyLock; + + /// Cached native certificates + /// + /// Creating a `with_native_roots()` hyper_rustls client re-loads system certs + /// each invocation (which can take 300ms on OSx). Cache the loaded certs + /// to avoid repeatedly incurring that cost. + pub(crate) static NATIVE_ROOTS: LazyLock>> = LazyLock::new(|| { + let CertificateResult { certs, errors, .. } = rustls_native_certs::load_native_certs(); + if !errors.is_empty() { + tracing::warn!("native root CA certificate loading errors: {errors:?}") + } + + if certs.is_empty() { + tracing::warn!("no native root CA certificates found!"); + } + + // NOTE: unlike hyper-rustls::with_native_roots we don't validate here, we'll do that later + // for now we have a collection of certs that may or may not be valid. + certs + }); + + pub(crate) fn restrict_ciphers(base: CryptoProvider) -> CryptoProvider { + let suites = &[ + rustls::CipherSuite::TLS13_AES_256_GCM_SHA384, + rustls::CipherSuite::TLS13_AES_128_GCM_SHA256, + // TLS1.2 suites + rustls::CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + rustls::CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + rustls::CipherSuite::TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + rustls::CipherSuite::TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + rustls::CipherSuite::TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + ]; + let supported_suites = suites + .iter() + .flat_map(|suite| { + base.cipher_suites + .iter() + .find(|s| &s.suite() == suite) + .cloned() + }) + .collect::>(); + CryptoProvider { + cipher_suites: supported_suites, + ..base + } + } + + impl TlsContext { + pub(crate) fn rustls_root_certs(&self) -> rustls::RootCertStore { + let mut roots = rustls::RootCertStore::empty(); + if self.trust_store.enable_native_roots { + let (valid, _invalid) = roots.add_parsable_certificates(NATIVE_ROOTS.clone()); + debug_assert!(valid > 0, "TrustStore configured to enable native roots but no valid root certificates parsed!"); + } + + for pem_cert in &self.trust_store.custom_certs { + let ders = CertificateDer::pem_slice_iter(&pem_cert.0) + .collect::, _>>() + .expect("valid PEM certificate"); + for cert in ders { + roots.add(cert).expect("cert parsable") + } + } + + roots + } + } + + /// Create a rustls ClientConfig with smithy-rs defaults + /// + /// This centralizes the rustls ClientConfig creation logic to ensure + /// consistency between the main HTTPS connector and tunnel handlers. + pub(crate) fn create_rustls_client_config( + crypto_mode: CryptoMode, + tls_context: &TlsContext, + ) -> rustls::ClientConfig { + let skip_restrict = crypto_mode.is_custom(); + let provider = if skip_restrict { + crypto_mode.provider() + } else { + restrict_ciphers(crypto_mode.provider()) + }; + let root_certs = tls_context.rustls_root_certs(); + rustls::ClientConfig::builder_with_provider(Arc::new(provider)) + .with_safe_default_protocol_versions() + .expect("Error with the TLS configuration. Please file a bug report under https://github.com/smithy-lang/smithy-rs/issues.") + .with_root_certificates(root_certs) + .with_no_client_auth() + } + + pub(crate) fn wrap_connector( + mut conn: HttpConnector, + crypto_mode: CryptoMode, + tls_context: &TlsContext, + proxy_config: crate::client::proxy::ProxyConfig, + ) -> super::connect::RustTlsConnector { + let client_config = create_rustls_client_config(crypto_mode, tls_context); + conn.enforce_http(false); + let https_connector = hyper_rustls::HttpsConnectorBuilder::new() + .with_tls_config(client_config.clone()) + .https_or_http() + .enable_http1() + .enable_http2() + .wrap_connector(conn); + + super::connect::RustTlsConnector::new(https_connector, client_config, proxy_config) + } +} + +pub(crate) mod connect { + use crate::client::connect::{Conn, Connecting}; + use crate::client::proxy::ProxyConfig; + use aws_smithy_runtime_api::box_error::BoxError; + use http_1x::uri::Scheme; + use http_1x::Uri; + use hyper::rt::{Read, ReadBufCursor, Write}; + use hyper_rustls::MaybeHttpsStream; + use hyper_util::client::legacy::connect::{Connected, Connection, HttpConnector}; + use hyper_util::client::proxy::matcher::Matcher; + use hyper_util::rt::TokioIo; + use pin_project_lite::pin_project; + use std::error::Error; + use std::sync::Arc; + use std::{ + io::{self, IoSlice}, + pin::Pin, + task::{Context, Poll}, + }; + use tokio::io::{AsyncRead, AsyncWrite}; + use tokio::net::TcpStream; + use tokio_rustls::client::TlsStream; + use tower::Service; + + #[derive(Debug, Clone)] + pub(crate) struct RustTlsConnector { + https: hyper_rustls::HttpsConnector>, + tls_config: Arc, + proxy_matcher: Option>, // Pre-computed for performance + } + + impl RustTlsConnector { + pub(super) fn new( + https: hyper_rustls::HttpsConnector>, + tls_config: rustls::ClientConfig, + proxy_config: ProxyConfig, + ) -> Self { + // Pre-compute the proxy matcher once during construction + let proxy_matcher = if proxy_config.is_disabled() { + None + } else { + Some(Arc::new(proxy_config.into_hyper_util_matcher())) + }; + + Self { + https, + tls_config: Arc::new(tls_config), + proxy_matcher, + } + } + } + + impl Service for RustTlsConnector + where + R: Clone + Send + Sync + 'static, + R: Service, + R::Response: Iterator, + R::Future: Send, + R::Error: Into>, + { + type Response = Conn; + type Error = BoxError; + type Future = Connecting; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.https.poll_ready(cx).map_err(Into::into) + } + + fn call(&mut self, dst: Uri) -> Self::Future { + // Check if this request should be proxied using pre-computed matcher + let proxy_intercept = if let Some(ref matcher) = self.proxy_matcher { + matcher.intercept(&dst) + } else { + None + }; + + if let Some(intercept) = proxy_intercept { + if dst.scheme() == Some(&Scheme::HTTPS) { + // HTTPS through HTTP proxy: Use CONNECT tunneling + manual TLS + self.handle_https_through_proxy(dst, intercept) + } else { + // HTTP through proxy: Direct connection to proxy + self.handle_http_through_proxy(dst, intercept) + } + } else { + // Direct connection: Use the existing HTTPS connector + self.handle_direct_connection(dst) + } + } + } + + impl RustTlsConnector + where + R: Clone + Send + Sync + 'static, + R: Service, + R::Response: Iterator, + R::Future: Send, + R::Error: Into>, + { + fn handle_direct_connection(&mut self, dst: Uri) -> Connecting { + let fut = self.https.call(dst); + Box::pin(async move { + let conn = fut.await?; + Ok(Conn { + inner: Box::new(conn), + is_proxy: false, + }) + }) + } + + fn handle_http_through_proxy( + &mut self, + _dst: Uri, + intercept: hyper_util::client::proxy::matcher::Intercept, + ) -> Connecting { + // For HTTP through proxy, connect to the proxy and let it handle the request + let proxy_uri = intercept.uri().clone(); + let fut = self.https.call(proxy_uri); + Box::pin(async move { + let conn = fut.await?; + Ok(Conn { + inner: Box::new(conn), + is_proxy: true, + }) + }) + } + + fn handle_https_through_proxy( + &mut self, + dst: Uri, + intercept: hyper_util::client::proxy::matcher::Intercept, + ) -> Connecting { + use rustls_pki_types::ServerName; + // For HTTPS through HTTP proxy, we need to: + // 1. Establish CONNECT tunnel using the HTTPS connector + // 2. Perform manual TLS handshake over the tunneled stream + + let tunnel = hyper_util::client::legacy::connect::proxy::Tunnel::new( + intercept.uri().clone(), + self.https.clone(), + ); + + // Configure tunnel with authentication if present + let mut tunnel = if let Some(auth) = intercept.basic_auth() { + tunnel.with_auth(auth.clone()) + } else { + tunnel + }; + + let tls_config = self.tls_config.clone(); + let dst_clone = dst.clone(); + + Box::pin(async move { + // Establish CONNECT tunnel + tracing::trace!("tunneling HTTPS over proxy"); + let tunneled = tunnel + .call(dst_clone.clone()) + .await + .map_err(|e| BoxError::from(format!("CONNECT tunnel failed: {e}")))?; + + // Stage 2: Manual TLS handshake over tunneled stream + let host = dst_clone + .host() + .ok_or("missing host in URI for TLS handshake")?; + + let server_name = ServerName::try_from(host.to_owned()).map_err(|e| { + BoxError::from(format!("invalid server name for TLS handshake: {e}")) + })?; + + let tls_connector = tokio_rustls::TlsConnector::from(tls_config) + .connect(server_name, TokioIo::new(tunneled)) + .await?; + + Ok(Conn { + inner: Box::new(RustTlsConn { + inner: TokioIo::new(tls_connector), + }), + is_proxy: true, + }) + }) + } + } + + pin_project! { + pub(crate) struct RustTlsConn { + #[pin] pub(super) inner: TokioIo> + } + } + + impl Connection for RustTlsConn>> { + fn connected(&self) -> Connected { + if self.inner.inner().get_ref().1.alpn_protocol() == Some(b"h2") { + self.inner + .inner() + .get_ref() + .0 + .inner() + .connected() + .negotiated_h2() + } else { + self.inner.inner().get_ref().0.inner().connected() + } + } + } + + impl Connection for RustTlsConn>>> { + fn connected(&self) -> Connected { + if self.inner.inner().get_ref().1.alpn_protocol() == Some(b"h2") { + self.inner + .inner() + .get_ref() + .0 + .inner() + .connected() + .negotiated_h2() + } else { + self.inner.inner().get_ref().0.inner().connected() + } + } + } + impl Read for RustTlsConn { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: ReadBufCursor<'_>, + ) -> Poll> { + let this = self.project(); + Read::poll_read(this.inner, cx, buf) + } + } + + impl Write for RustTlsConn { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let this = self.project(); + Write::poll_write(this.inner, cx, buf) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[IoSlice<'_>], + ) -> Poll> { + let this = self.project(); + Write::poll_write_vectored(this.inner, cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + + fn poll_flush( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + let this = self.project(); + Write::poll_flush(this.inner, cx) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + let this = self.project(); + Write::poll_shutdown(this.inner, cx) + } + } +} diff --git a/sources/aws-smithy-http-client/src/client/tls/s2n_tls_provider.rs b/sources/aws-smithy-http-client/src/client/tls/s2n_tls_provider.rs new file mode 100644 index 000000000..888221f0d --- /dev/null +++ b/sources/aws-smithy-http-client/src/client/tls/s2n_tls_provider.rs @@ -0,0 +1,319 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +pub(crate) mod build_connector { + use crate::tls::TlsContext; + use client::connect::HttpConnector; + use hyper_util::client::legacy as client; + use s2n_tls::security::Policy; + use std::sync::LazyLock; + + // Default S2N security policy which sets protocol versions and cipher suites + // See https://aws.github.io/s2n-tls/usage-guide/ch06-security-policies.html + const S2N_POLICY_VERSION: &str = "20230317"; + + fn base_config() -> s2n_tls::config::Builder { + let mut builder = s2n_tls::config::Config::builder(); + let policy = Policy::from_version(S2N_POLICY_VERSION).unwrap(); + builder + .set_security_policy(&policy) + .expect("valid s2n security policy"); + // default is true + builder.with_system_certs(false).unwrap(); + builder + } + + static CACHED_CONFIG: LazyLock = LazyLock::new(|| { + let mut config = base_config(); + config.with_system_certs(true).unwrap(); + // actually loads the system certs + config.build().expect("valid s2n config") + }); + + impl TlsContext { + fn s2n_config(&self) -> s2n_tls::config::Config { + // TODO(s2n-tls): s2n does not support turning a config back into a builder or a way to load a trust store and re-use it + // instead if we are only using the defaults then use a cached config, otherwise pay the cost to build a new one + if self.trust_store.enable_native_roots && self.trust_store.custom_certs.is_empty() { + CACHED_CONFIG.clone() + } else { + let mut config = base_config(); + config + .with_system_certs(self.trust_store.enable_native_roots) + .unwrap(); + for pem_cert in &self.trust_store.custom_certs { + config + .trust_pem(pem_cert.0.as_slice()) + .expect("valid certificate"); + } + config.build().expect("valid s2n config") + } + } + } + + pub(crate) fn wrap_connector( + mut http_connector: HttpConnector, + tls_context: &TlsContext, + proxy_config: crate::client::proxy::ProxyConfig, + ) -> super::connect::S2nTlsConnector { + let config = tls_context.s2n_config(); + http_connector.enforce_http(false); + let mut builder = s2n_tls_hyper::connector::HttpsConnector::builder_with_http( + http_connector, + config.clone(), + ); + builder.with_plaintext_http(true); + let https_connector = builder.build(); + + super::connect::S2nTlsConnector::new(https_connector, config, proxy_config) + } +} + +pub(crate) mod connect { + use crate::client::connect::{Conn, Connecting}; + use crate::client::proxy::ProxyConfig; + use aws_smithy_runtime_api::box_error::BoxError; + use http_1x::uri::Scheme; + use http_1x::Uri; + use hyper_util::client::legacy::connect::{Connected, Connection, HttpConnector}; + use hyper_util::client::proxy::matcher::Matcher; + use hyper_util::rt::TokioIo; + use std::error::Error; + use std::sync::Arc; + use std::{ + io::IoSlice, + pin::Pin, + task::{Context, Poll}, + }; + use tower::Service; + + #[derive(Clone)] + pub(crate) struct S2nTlsConnector { + https: s2n_tls_hyper::connector::HttpsConnector>, + tls_config: s2n_tls::config::Config, + proxy_matcher: Option>, // Pre-computed for performance + } + + impl S2nTlsConnector { + pub(super) fn new( + https: s2n_tls_hyper::connector::HttpsConnector>, + tls_config: s2n_tls::config::Config, + proxy_config: ProxyConfig, + ) -> Self { + // Pre-compute the proxy matcher once during construction + let proxy_matcher = if proxy_config.is_disabled() { + None + } else { + Some(Arc::new(proxy_config.into_hyper_util_matcher())) + }; + + Self { + https, + tls_config, + proxy_matcher, + } + } + } + + impl Service for S2nTlsConnector + where + R: Clone + Send + Sync + 'static, + R: Service, + R::Response: Iterator, + R::Future: Send, + R::Error: Into>, + { + type Response = Conn; + type Error = BoxError; + type Future = Connecting; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.https.poll_ready(cx).map_err(Into::into) + } + + fn call(&mut self, dst: Uri) -> Self::Future { + // Check if this request should be proxied using pre-computed matcher + let proxy_intercept = if let Some(ref matcher) = self.proxy_matcher { + matcher.intercept(&dst) + } else { + None + }; + + if let Some(intercept) = proxy_intercept { + if dst.scheme() == Some(&Scheme::HTTPS) { + // HTTPS through HTTP proxy: Use CONNECT tunneling + manual TLS + self.handle_https_through_proxy(dst, intercept) + } else { + // HTTP through proxy: Direct connection to proxy + self.handle_http_through_proxy(dst, intercept) + } + } else { + // Direct connection: Use the existing HTTPS connector + self.handle_direct_connection(dst) + } + } + } + + impl S2nTlsConnector + where + R: Clone + Send + Sync + 'static, + R: Service, + R::Response: Iterator, + R::Future: Send, + R::Error: Into>, + { + fn handle_direct_connection(&mut self, dst: Uri) -> Connecting { + let fut = self.https.call(dst); + Box::pin(async move { + let conn = fut.await?; + Ok(Conn { + inner: Box::new(conn), + is_proxy: false, + }) + }) + } + + fn handle_http_through_proxy( + &mut self, + _dst: Uri, + intercept: hyper_util::client::proxy::matcher::Intercept, + ) -> Connecting { + // For HTTP through proxy, connect to the proxy and let it handle the request + let proxy_uri = intercept.uri().clone(); + let fut = self.https.call(proxy_uri); + Box::pin(async move { + let conn = fut.await?; + Ok(Conn { + inner: Box::new(conn), + is_proxy: true, + }) + }) + } + + fn handle_https_through_proxy( + &mut self, + dst: Uri, + intercept: hyper_util::client::proxy::matcher::Intercept, + ) -> Connecting { + // For HTTPS through HTTP proxy, we need to: + // 1. Establish CONNECT tunnel using the HTTPS connector + // 2. Perform manual TLS handshake over the tunneled stream + + let tunnel = hyper_util::client::legacy::connect::proxy::Tunnel::new( + intercept.uri().clone(), + self.https.clone(), + ); + + // Configure tunnel with authentication if present + let mut tunnel = if let Some(auth) = intercept.basic_auth() { + tunnel.with_auth(auth.clone()) + } else { + tunnel + }; + + let tls_config = self.tls_config.clone(); + let dst_clone = dst.clone(); + + Box::pin(async move { + // Stage 1: Establish CONNECT tunnel + tracing::trace!("tunneling HTTPS over proxy using s2n-tls"); + let tunneled = tunnel + .call(dst_clone.clone()) + .await + .map_err(|e| BoxError::from(format!("CONNECT tunnel failed: {e}")))?; + + // Stage 2: Manual TLS handshake over tunneled stream + let host = dst_clone + .host() + .ok_or("missing host in URI for TLS handshake")?; + + // s2n-tls uses string server names (simpler than rustls ServerName) + let tls_connector = s2n_tls_tokio::TlsConnector::new(tls_config); + let tls_stream = tls_connector + .connect(host, TokioIo::new(tunneled)) + .await + .map_err(|e| BoxError::from(format!("s2n-tls handshake failed: {e}")))?; + + Ok(Conn { + inner: Box::new(S2nTlsConn { + inner: TokioIo::new(tls_stream), + }), + is_proxy: true, + }) + }) + } + } + + // Simple wrapper that implements Connection for s2n-tls streams + struct S2nTlsConn + where + T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, + { + inner: TokioIo>, + } + + impl Connection for S2nTlsConn + where + T: Connection + tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, + { + fn connected(&self) -> Connected { + // For tunneled connections, we can't easily access the underlying connection info + // from s2n-tls, so we'll return a basic Connected instance + Connected::new() + } + } + + impl hyper::rt::Read for S2nTlsConn + where + T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, + { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: hyper::rt::ReadBufCursor<'_>, + ) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_read(cx, buf) + } + } + + impl hyper::rt::Write for S2nTlsConn + where + T: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, + { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_write(cx, buf) + } + + fn poll_flush( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_flush(cx) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_shutdown(cx) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + } +} diff --git a/sources/aws-smithy-http-client/src/error.rs b/sources/aws-smithy-http-client/src/error.rs new file mode 100644 index 000000000..e8e04a549 --- /dev/null +++ b/sources/aws-smithy-http-client/src/error.rs @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +use aws_smithy_runtime_api::box_error::BoxError; +use std::fmt; + +/// HTTP client errors +/// +/// This is normally due to configuration issues, internal SDK bugs, or other user error. +#[derive(Debug)] +pub struct HttpClientError { + source: Option, +} + +impl fmt::Display for HttpClientError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "unknown HTTP client error") + } +} + +impl std::error::Error for HttpClientError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.source.as_ref().map(|err| err.as_ref() as _) + } +} diff --git a/sources/aws-smithy-experimental/src/hyper_1_0.rs b/sources/aws-smithy-http-client/src/hyper_legacy.rs similarity index 62% rename from sources/aws-smithy-experimental/src/hyper_1_0.rs rename to sources/aws-smithy-http-client/src/hyper_legacy.rs index 8b729b030..2d4f9c6ed 100644 --- a/sources/aws-smithy-experimental/src/hyper_1_0.rs +++ b/sources/aws-smithy-http-client/src/hyper_legacy.rs @@ -3,13 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ +use crate::hyper_legacy::timeout_middleware::HttpTimeoutError; use aws_smithy_async::future::timeout::TimedOutError; use aws_smithy_async::rt::sleep::{default_async_sleep, AsyncSleep, SharedAsyncSleep}; -use aws_smithy_runtime::client::http::connection_poisoning::CaptureSmithyConnection; use aws_smithy_runtime_api::box_error::BoxError; +use aws_smithy_runtime_api::client::connection::CaptureSmithyConnection; use aws_smithy_runtime_api::client::connection::ConnectionMetadata; use aws_smithy_runtime_api::client::connector_metadata::ConnectorMetadata; -use aws_smithy_runtime_api::client::dns::ResolveDns; use aws_smithy_runtime_api::client::http::{ HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpClient, SharedHttpConnector, @@ -24,253 +24,143 @@ use aws_smithy_types::body::SdkBody; use aws_smithy_types::config_bag::ConfigBag; use aws_smithy_types::error::display::DisplayErrorContext; use aws_smithy_types::retry::ErrorKind; -use client::connect::Connection; -use h2::Reason; -use http::{Extensions, Uri}; -use hyper::rt::{Read, Write}; -use hyper_util::client::legacy as client; -use hyper_util::client::legacy::connect::dns::Name; -use hyper_util::client::legacy::connect::{ - capture_connection, CaptureConnection, Connect, HttpInfo, -}; -use hyper_util::rt::TokioExecutor; -use rustls::crypto::CryptoProvider; +use h2_0_3::Reason; +use hyper_0_14::client::connect::{capture_connection, CaptureConnection, Connection, HttpInfo}; use std::borrow::Cow; use std::collections::HashMap; use std::error::Error; -use std::future::Future; -use std::net::SocketAddr; -use std::pin::Pin; +use std::fmt; use std::sync::RwLock; -use std::task::{Context, Poll}; use std::time::Duration; -use std::{fmt, vec}; - -#[derive(Debug, Eq, PartialEq, Clone, Copy)] -#[non_exhaustive] -pub enum CryptoMode { - #[cfg(feature = "crypto-ring")] - Ring, - #[cfg(feature = "crypto-aws-lc")] - AwsLc, - #[cfg(feature = "crypto-aws-lc-fips")] - AwsLcFips, -} - -impl CryptoMode { - fn provider(self) -> CryptoProvider { - match self { - #[cfg(feature = "crypto-aws-lc")] - CryptoMode::AwsLc => rustls::crypto::aws_lc_rs::default_provider(), - - #[cfg(feature = "crypto-ring")] - CryptoMode::Ring => rustls::crypto::ring::default_provider(), - - #[cfg(feature = "crypto-aws-lc-fips")] - CryptoMode::AwsLcFips => { - let provider = rustls::crypto::default_fips_provider(); - assert!( - provider.fips(), - "FIPS was requested but the provider did not support FIPS" - ); - provider +use tokio::io::{AsyncRead, AsyncWrite}; + +#[cfg(feature = "legacy-rustls-ring")] +mod default_connector { + use aws_smithy_async::rt::sleep::SharedAsyncSleep; + use aws_smithy_runtime_api::client::http::HttpConnectorSettings; + use legacy_hyper_rustls as hyper_rustls; + use legacy_rustls as rustls; + use std::sync::LazyLock; + + // Creating a `with_native_roots` HTTP client takes 300ms on OS X. Cache this so that we + // don't need to repeatedly incur that cost. + pub(crate) static HTTPS_NATIVE_ROOTS: LazyLock< + hyper_rustls::HttpsConnector, + > = LazyLock::new(default_tls); + + fn default_tls() -> hyper_rustls::HttpsConnector { + use legacy_rustls::client::WantsTransparencyPolicyOrClientCert; + use legacy_rustls::{ClientConfig, ConfigBuilder, WantsVerifier}; + use rustls_native_certs; + // polyfill with_native_roots from https://docs.rs/hyper-rustls/0.24.2/src/hyper_rustls/config.rs.html#22-70 + // to use the new rustls_native_certs, since rustls_native_certs 0.6 depends on rustls-pemfile which is deprecated + fn with_native_roots( + this: ConfigBuilder, + ) -> ConfigBuilder { + let mut roots = rustls::RootCertStore::empty(); + let mut valid_count = 0; + let mut invalid_count = 0; + + for cert in + rustls_native_certs::load_native_certs().expect("could not load platform certs") + { + let cert = rustls::Certificate(cert.to_vec()); + match roots.add(&cert) { + Ok(_) => valid_count += 1, + Err(err) => { + tracing::trace!("invalid cert der {:?}", cert.0); + tracing::debug!("certificate parsing failed: {:?}", err); + invalid_count += 1 + } + } } - } - } -} - -/// A bridge that allows our `ResolveDns` trait to work with Hyper's `Resolver` interface (based on tower) -#[derive(Clone)] -struct HyperUtilResolver { - resolver: R, -} - -impl tower::Service for HyperUtilResolver { - type Response = vec::IntoIter; - type Error = Box; - type Future = Pin> + Send>>; - - fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn call(&mut self, req: Name) -> Self::Future { - let resolver = self.resolver.clone(); - Box::pin(async move { - let dns_entries = resolver.resolve_dns(req.as_str()).await?; - Ok(dns_entries - .into_iter() - .map(|ip_addr| SocketAddr::new(ip_addr, 0)) - .collect::>() - .into_iter()) - }) - } -} - -#[allow(unused_imports)] -mod cached_connectors { - use client::connect::HttpConnector; - use hyper_util::client::legacy as client; - use hyper_util::client::legacy::connect::dns::GaiResolver; - - use crate::hyper_1_0::build_connector::make_tls; - use crate::hyper_1_0::{CryptoMode, Inner}; - - #[cfg(feature = "crypto-ring")] - pub(crate) static HTTPS_NATIVE_ROOTS_RING: once_cell::sync::Lazy< - hyper_rustls::HttpsConnector, - > = once_cell::sync::Lazy::new(|| make_tls(GaiResolver::new(), CryptoMode::Ring.provider())); - - #[cfg(feature = "crypto-aws-lc")] - pub(crate) static HTTPS_NATIVE_ROOTS_AWS_LC: once_cell::sync::Lazy< - hyper_rustls::HttpsConnector, - > = once_cell::sync::Lazy::new(|| make_tls(GaiResolver::new(), CryptoMode::AwsLc.provider())); - - #[cfg(feature = "crypto-aws-lc-fips")] - pub(crate) static HTTPS_NATIVE_ROOTS_AWS_LC_FIPS: once_cell::sync::Lazy< - hyper_rustls::HttpsConnector, - > = once_cell::sync::Lazy::new(|| { - make_tls(GaiResolver::new(), CryptoMode::AwsLcFips.provider()) - }); - - pub(super) fn cached_https(mode: Inner) -> hyper_rustls::HttpsConnector { - match mode { - #[cfg(feature = "crypto-ring")] - Inner::Standard(CryptoMode::Ring) => HTTPS_NATIVE_ROOTS_RING.clone(), - #[cfg(feature = "crypto-aws-lc")] - Inner::Standard(CryptoMode::AwsLc) => HTTPS_NATIVE_ROOTS_AWS_LC.clone(), - #[cfg(feature = "crypto-aws-lc-fips")] - Inner::Standard(CryptoMode::AwsLcFips) => HTTPS_NATIVE_ROOTS_AWS_LC_FIPS.clone(), - #[allow(unreachable_patterns)] - Inner::Standard(_) => unreachable!("unexpected mode"), - Inner::Custom(provider) => make_tls(GaiResolver::new(), provider), - } - } -} + tracing::debug!( + "with_native_roots processed {} valid and {} invalid certs", + valid_count, + invalid_count + ); + assert!(!roots.is_empty(), "no CA certificates found"); -mod build_connector { - use crate::hyper_1_0::{HyperUtilResolver, Inner}; - use aws_smithy_runtime_api::client::dns::ResolveDns; - use client::connect::HttpConnector; - use headers::Authorization; - use hyper::Uri; - use hyper_http_proxy::{Proxy, ProxyConnector}; - use hyper_util::client::legacy as client; - use rustls::crypto::CryptoProvider; - use std::sync::Arc; - use url::Url; - - fn restrict_ciphers(base: CryptoProvider) -> CryptoProvider { - let suites = &[ - rustls::CipherSuite::TLS13_AES_256_GCM_SHA384, - rustls::CipherSuite::TLS13_AES_128_GCM_SHA256, - // TLS1.2 suites - rustls::CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - rustls::CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - rustls::CipherSuite::TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - rustls::CipherSuite::TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - rustls::CipherSuite::TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, - ]; - let supported_suites = suites - .iter() - .flat_map(|suite| { - base.cipher_suites - .iter() - .find(|s| &s.suite() == suite) - .cloned() - }) - .collect::>(); - CryptoProvider { - cipher_suites: supported_suites, - ..base + this.with_root_certificates(roots) } - } - - pub(crate) fn make_tls( - resolver: R, - crypto_provider: CryptoProvider, - ) -> hyper_rustls::HttpsConnector> { - use hyper_rustls::ConfigBuilderExt; - let mut base_connector = HttpConnector::new_with_resolver(resolver); - base_connector.enforce_http(false); hyper_rustls::HttpsConnectorBuilder::new() .with_tls_config( - rustls::ClientConfig::builder_with_provider(Arc::new(restrict_ciphers(crypto_provider))) + with_native_roots(rustls::ClientConfig::builder() + .with_cipher_suites(&[ + // TLS1.3 suites + rustls::cipher_suite::TLS13_AES_256_GCM_SHA384, + rustls::cipher_suite::TLS13_AES_128_GCM_SHA256, + // TLS1.2 suites + rustls::cipher_suite::TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + rustls::cipher_suite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + rustls::cipher_suite::TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + rustls::cipher_suite::TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + rustls::cipher_suite::TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + ]) + .with_safe_default_kx_groups() .with_safe_default_protocol_versions() - .expect("Error with the TLS configuration. Please file a bug report under https://github.com/smithy-lang/smithy-rs/issues.") - .with_native_roots().expect("error with TLS configuration.") + .expect("Error with the TLS configuration. Please file a bug report under https://github.com/smithy-lang/smithy-rs/issues.")) .with_no_client_auth() ) .https_or_http() .enable_http1() .enable_http2() - .wrap_connector(base_connector) + .build() } - pub(super) fn https_with_resolver( - crypto_provider: Inner, - resolver: R, - ) -> hyper_rustls::HttpsConnector>> { - make_tls(HyperUtilResolver { resolver }, crypto_provider.provider()) + pub(super) fn base( + settings: &HttpConnectorSettings, + sleep: Option, + ) -> super::HyperConnectorBuilder { + let mut hyper = super::HyperConnector::builder().connector_settings(settings.clone()); + if let Some(sleep) = sleep { + hyper = hyper.sleep_impl(sleep); + } + hyper } - pub(super) fn https_with_proxy( - https_connector: hyper_rustls::HttpsConnector, - https_proxy: &str, - no_proxy: Option>, - ) -> hyper_http_proxy::ProxyConnector> { - // Determines whether a request of a given scheme, host and port should be proxied - // according to `https_proxy` and `no_proxy`. - - let intercept = move |scheme: Option<&str>, host: Option<&str>, _port| { - if let Some(host) = host { - if let Some(no_proxy) = &no_proxy { - if scheme != Some("https") { - return false; - } - if no_proxy.iter().any(|s| s == "*") { - // Don't proxy anything - return false; - } - // If the host matches one of the no proxy list entries, return false (don't proxy) - // Note that we're not doing anything fancy here for checking `no_proxy` since - // we only expect requests here to be going out to some AWS API endpoint. - return !no_proxy.iter().any(|no_proxy_host| { - !no_proxy_host.is_empty() && host.ends_with(no_proxy_host) - }); - } - true - } else { - false - } - }; + /// Return a default HTTPS connector backed by the `rustls` crate. + /// + /// It requires a minimum TLS version of 1.2. + /// It allows you to connect to both `http` and `https` URLs. + pub(super) fn https() -> hyper_rustls::HttpsConnector { + HTTPS_NATIVE_ROOTS.clone() + } +} - let mut proxy_uri = https_proxy.parse::().expect("Invalid proxy URI"); +/// Given `HttpConnectorSettings` and an `SharedAsyncSleep`, create a `SharedHttpConnector` from defaults depending on what cargo features are activated. +pub fn default_connector( + settings: &HttpConnectorSettings, + sleep: Option, +) -> Option { + #[cfg(feature = "legacy-rustls-ring")] + { + tracing::trace!(settings = ?settings, sleep = ?sleep, "creating a new default connector"); + let hyper = default_connector::base(settings, sleep).build_https(); + Some(SharedHttpConnector::new(hyper)) + } + #[cfg(not(feature = "legacy-rustls-ring"))] + { + tracing::trace!(settings = ?settings, sleep = ?sleep, "no default connector available"); + None + } +} - // If the proxy's URI doesn't have a scheme, assume HTTP for the scheme and let the proxy - // server forward HTTPS connections and start a tunnel. - if proxy_uri.scheme().is_none() { - proxy_uri = format!("http://{https_proxy}") - .parse::() - .expect("Unable to parse proxy URI as HTTPS"); - } - let mut proxy = Proxy::new(intercept, proxy_uri); - // Parse https_proxy as URL to extract out auth information if any - let proxy_url = - Url::parse(&proxy.uri().to_string()).expect("Unable to parse HTTPS proxy as URL"); - - if !proxy_url.username().is_empty() || proxy_url.password().is_some() { - proxy.set_authorization(Authorization::basic( - proxy_url.username(), - proxy_url.password().unwrap_or_default(), - )); - } - ProxyConnector::from_proxy(https_connector, proxy) - .expect("Failed to create proxy connector") +/// Creates a hyper-backed HTTPS client from defaults depending on what cargo features are activated. +pub fn default_client() -> Option { + #[cfg(feature = "legacy-rustls-ring")] + { + tracing::trace!("creating a new default hyper 0.14.x client"); + Some(HyperClientBuilder::new().build_https()) + } + #[cfg(not(feature = "legacy-rustls-ring"))] + { + tracing::trace!("no default connector available"); + None } } -/// [`HttpConnector`] that uses [`hyper`] to make HTTP requests. +/// [`HttpConnector`] that uses [`hyper_0_14`] to make HTTP requests. /// /// This connector also implements socket connect and read timeouts. /// @@ -297,67 +187,23 @@ impl HttpConnector for HyperConnector { /// Builder for [`HyperConnector`]. #[derive(Default, Debug)] -pub struct HyperConnectorBuilder { +pub struct HyperConnectorBuilder { connector_settings: Option, sleep_impl: Option, - client_builder: Option, - #[allow(unused)] - crypto: Crypto, -} - -#[derive(Default)] -#[non_exhaustive] -pub struct CryptoUnset {} - -pub struct CryptoProviderSelected { - crypto_provider: Inner, -} - -#[derive(Clone)] -enum Inner { - Standard(CryptoMode), - #[allow(dead_code)] - Custom(CryptoProvider), -} - -impl Inner { - fn provider(&self) -> CryptoProvider { - match self { - Inner::Standard(mode) => mode.provider(), - Inner::Custom(provider) => provider.clone(), - } - } + client_builder: Option, } -#[cfg(any(feature = "crypto-aws-lc", feature = "crypto-ring"))] -impl HyperConnectorBuilder { - pub fn build_from_resolver( - self, - resolver: R, - ) -> HyperConnector { - let connector = - build_connector::https_with_resolver(self.crypto.crypto_provider.clone(), resolver); - self.build(connector) - } -} - -impl HyperConnectorBuilder { +impl HyperConnectorBuilder { /// Create a [`HyperConnector`] from this builder and a given connector. - pub(crate) fn build(self, tcp_connector: C) -> HyperConnector + pub fn build(self, tcp_connector: C) -> HyperConnector where - C: Send + Sync + 'static, - C: Clone, - C: tower::Service, - C::Response: Read + Write + Connection + Send + Sync + Unpin, - C: Connect, + C: Clone + Send + Sync + 'static, + C: hyper_0_14::service::Service, + C::Response: Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, C::Future: Unpin + Send + 'static, C::Error: Into, { - let client_builder = - self.client_builder - .unwrap_or(hyper_util::client::legacy::Builder::new( - TokioExecutor::new(), - )); + let client_builder = self.client_builder.unwrap_or_default(); let sleep_impl = self.sleep_impl.or_else(default_async_sleep); let (connect_timeout, read_timeout) = self .connector_settings @@ -390,6 +236,12 @@ impl HyperConnectorBuilder { } } + /// Create a [`HyperConnector`] with the default rustls HTTPS implementation. + #[cfg(feature = "legacy-rustls-ring")] + pub fn build_https(self) -> HyperConnector { + self.build(default_connector::https()) + } + /// Set the async sleep implementation used for timeouts /// /// Calling this is only necessary for testing or to use something other than @@ -423,35 +275,32 @@ impl HyperConnectorBuilder { self } - /// Override the Hyper client [`Builder`](hyper_util::client::legacy::Builder) used to construct this client. + /// Override the Hyper client [`Builder`](hyper_0_14::client::Builder) used to construct this client. /// /// This enables changing settings like forcing HTTP2 and modifying other default client behavior. - pub(crate) fn hyper_builder( - mut self, - hyper_builder: hyper_util::client::legacy::Builder, - ) -> Self { - self.set_hyper_builder(Some(hyper_builder)); + pub fn hyper_builder(mut self, hyper_builder: hyper_0_14::client::Builder) -> Self { + self.client_builder = Some(hyper_builder); self } - /// Override the Hyper client [`Builder`](hyper_util::client::legacy::Builder) used to construct this client. + /// Override the Hyper client [`Builder`](hyper_0_14::client::Builder) used to construct this client. /// /// This enables changing settings like forcing HTTP2 and modifying other default client behavior. - pub(crate) fn set_hyper_builder( + pub fn set_hyper_builder( &mut self, - hyper_builder: Option, + hyper_builder: Option, ) -> &mut Self { self.client_builder = hyper_builder; self } } -/// Adapter to use a Hyper 1.0-based Client as an `HttpConnector` +/// Adapter from a [`hyper_0_14::Client`] to [`HttpConnector`]. /// /// This adapter also enables TCP `CONNECT` and HTTP `READ` timeouts via [`HyperConnector::builder`]. struct Adapter { client: timeout_middleware::HttpReadTimeout< - hyper_util::client::legacy::Client, SdkBody>, + hyper_0_14::Client, SdkBody>, >, } @@ -467,7 +316,7 @@ impl fmt::Debug for Adapter { fn extract_smithy_connection(capture_conn: &CaptureConnection) -> Option { let capture_conn = capture_conn.clone(); if let Some(conn) = capture_conn.clone().connection_metadata().as_ref() { - let mut extensions = Extensions::new(); + let mut extensions = http_02x::Extensions::new(); conn.get_extras(&mut extensions); let http_info = extensions.get::(); let mut builder = ConnectionMetadata::builder() @@ -492,17 +341,18 @@ fn extract_smithy_connection(capture_conn: &CaptureConnection) -> Option HttpConnector for Adapter where C: Clone + Send + Sync + 'static, - C: tower::Service, - C::Response: Connection + Read + Write + Unpin + 'static, - timeout_middleware::ConnectTimeout: Connect, + C: hyper_0_14::service::Service, + C::Response: Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, C::Future: Unpin + Send + 'static, C::Error: Into, { fn call(&self, request: HttpRequest) -> HttpConnectorFuture { - let mut request = match request.try_into_http1x() { + use hyper_0_14::service::Service; + + let mut request = match request.try_into_http02x() { Ok(request) => request, Err(err) => { - return HttpConnectorFuture::ready(Err(ConnectorError::user(err.into()))); + return HttpConnectorFuture::ready(Err(ConnectorError::other(err.into(), None))); } }; let capture_connection = capture_connection(&mut request); @@ -513,13 +363,12 @@ where .set_connection_retriever(move || extract_smithy_connection(&capture_connection)); } let mut client = self.client.clone(); - use tower::Service; let fut = client.call(request); HttpConnectorFuture::new(async move { let response = fut .await .map_err(downcast_error)? - .map(SdkBody::from_body_1_x); + .map(SdkBody::from_body_0_4); match HttpResponse::try_from(response) { Ok(response) => Ok(response), Err(err) => Err(ConnectorError::other(err.into(), None)), @@ -541,41 +390,40 @@ fn downcast_error(err: BoxError) -> ConnectorError { }; // generally, the top of chain will probably be a hyper error. Go through a set of hyper specific // error classifications - let err = match find_source::(err.as_ref()) { - Some(hyper_error) => return to_connector_error(hyper_error)(err), - None => err, + let err = match err.downcast::() { + Ok(hyper_error) => return to_connector_error(*hyper_error), + Err(box_error) => box_error, }; // otherwise, we have no idea! ConnectorError::other(err, None) } -/// Convert a [`hyper::Error`] into a [`ConnectorError`] -fn to_connector_error(err: &hyper::Error) -> fn(BoxError) -> ConnectorError { - if err.is_timeout() || find_source::(err).is_some() { - return ConnectorError::timeout; +/// Convert a [`hyper_0_14::Error`] into a [`ConnectorError`] +fn to_connector_error(err: hyper_0_14::Error) -> ConnectorError { + if err.is_timeout() || find_source::(&err).is_some() { + return ConnectorError::timeout(err.into()); } if err.is_user() { - return ConnectorError::user; + return ConnectorError::user(err.into()); } - if err.is_closed() || err.is_canceled() || find_source::(err).is_some() { - return ConnectorError::io; + if err.is_closed() || err.is_canceled() || find_source::(&err).is_some() { + return ConnectorError::io(err.into()); } // We sometimes receive this from S3: hyper::Error(IncompleteMessage) if err.is_incomplete_message() { - return |err: BoxError| ConnectorError::other(err, Some(ErrorKind::TransientError)); + return ConnectorError::other(err.into(), Some(ErrorKind::TransientError)); } - - if let Some(h2_err) = find_source::(err) { + if let Some(h2_err) = find_source::(&err) { if h2_err.is_go_away() || (h2_err.is_reset() && h2_err.reason() == Some(Reason::REFUSED_STREAM)) { - return ConnectorError::io; + return ConnectorError::io(err.into()); } } tracing::warn!(err = %DisplayErrorContext(&err), "unrecognized error from Hyper. If this error should be retried, please file an issue."); - |err: BoxError| ConnectorError::other(err, None) + ConnectorError::other(err.into(), None) } fn find_source<'a, E: Error + 'static>(err: &'a (dyn Error + 'static)) -> Option<&'a E> { @@ -589,9 +437,6 @@ fn find_source<'a, E: Error + 'static>(err: &'a (dyn Error + 'static)) -> Option None } -// TODO(https://github.com/awslabs/aws-sdk-rust/issues/1090): CacheKey must also include ptr equality to any -// runtime components that are used—sleep_impl as a base (unless we prohibit overriding sleep impl) -// If we decide to put a DnsResolver in RuntimeComponents, then we'll need to handle that as well. #[derive(Clone, Debug, Eq, PartialEq, Hash)] struct CacheKey { connect_timeout: Option, @@ -609,7 +454,7 @@ impl From<&HttpConnectorSettings> for CacheKey { struct HyperClient { connector_cache: RwLock>, - client_builder: hyper_util::client::legacy::Builder, + client_builder: hyper_0_14::client::Builder, tcp_connector_fn: F, } @@ -626,8 +471,8 @@ impl HttpClient for HyperClient where F: Fn() -> C + Send + Sync, C: Clone + Send + Sync + 'static, - C: tower::Service, - C::Response: Connection + Read + Write + Send + Sync + Unpin + 'static, + C: hyper_0_14::service::Service, + C::Response: Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, C::Future: Unpin + Send + 'static, C::Error: Into, { @@ -679,7 +524,7 @@ where } fn connector_metadata(&self) -> Option { - Some(ConnectorMetadata::new("hyper", Some(Cow::Borrowed("1.x")))) + Some(ConnectorMetadata::new("hyper", Some(Cow::Borrowed("0.x")))) } } @@ -690,113 +535,131 @@ where /// /// # Examples /// -/// Construct a Hyper client with the RusTLS TLS implementation. +/// Construct a Hyper client with the default TLS implementation (rustls). /// This can be useful when you want to share a Hyper connector between multiple /// generated Smithy clients. +/// +/// ```no_run,ignore +/// use aws_smithy_http_client::hyper_014::HyperClientBuilder; +/// +/// let http_client = HyperClientBuilder::new().build_https(); +/// +/// // This connector can then be given to a generated service Config +/// let config = my_service_client::Config::builder() +/// .endpoint_url("http://localhost:1234") +/// .http_client(http_client) +/// .build(); +/// let client = my_service_client::Client::from_conf(config); +/// ``` +/// +/// ## Use a Hyper client with WebPKI roots +/// +/// A use case for where you may want to use the [`HyperClientBuilder`] is when +/// setting Hyper client settings that aren't otherwise exposed by the `Config` +/// builder interface. Some examples include changing: +/// +/// - Hyper client settings +/// - Allowed TLS cipher suites +/// - Using an alternative TLS connector library (not the default, rustls) +/// - CA trust root certificates (illustrated using WebPKI below) +/// +/// ```no_run,ignore +/// use aws_smithy_http_client::hyper_014::HyperClientBuilder; +/// +/// let https_connector = hyper_rustls::HttpsConnectorBuilder::new() +/// .with_webpki_roots() +/// .https_only() +/// .enable_http1() +/// .enable_http2() +/// .build(); +/// let http_client = HyperClientBuilder::new().build(https_connector); +/// +/// // This connector can then be given to a generated service Config +/// let config = my_service_client::Config::builder() +/// .endpoint_url("https://example.com") +/// .http_client(http_client) +/// .build(); +/// let client = my_service_client::Client::from_conf(config); +/// ``` #[derive(Clone, Default, Debug)] -pub struct HyperClientBuilder { - client_builder: Option, - crypto_provider: Crypto, +pub struct HyperClientBuilder { + client_builder: Option, } -impl HyperClientBuilder { - /// Create a hyper client using RusTLS for TLS +impl HyperClientBuilder { + /// Creates a new builder. + pub fn new() -> Self { + Self::default() + } + + /// Override the Hyper client [`Builder`](hyper_0_14::client::Builder) used to construct this client. + /// + /// This enables changing settings like forcing HTTP2 and modifying other default client behavior. + pub fn hyper_builder(mut self, hyper_builder: hyper_0_14::client::Builder) -> Self { + self.client_builder = Some(hyper_builder); + self + } + + /// Override the Hyper client [`Builder`](hyper_0_14::client::Builder) used to construct this client. + /// + /// This enables changing settings like forcing HTTP2 and modifying other default client behavior. + pub fn set_hyper_builder( + &mut self, + hyper_builder: Option, + ) -> &mut Self { + self.client_builder = hyper_builder; + self + } + + /// Create a hyper client with the default rustls HTTPS implementation. /// /// The trusted certificates will be loaded later when this becomes the selected /// HTTP client for a Smithy client. + #[cfg(feature = "legacy-rustls-ring")] pub fn build_https(self) -> SharedHttpClient { - let crypto = self.crypto_provider.crypto_provider; - build_with_fn(self.client_builder, move || { - cached_connectors::cached_https(crypto.clone()) - }) + self.build_with_fn(default_connector::https) } - /// Create a hyper client using a custom DNS resolver - pub fn build_with_resolver( - self, - resolver: impl ResolveDns + Clone + 'static, - ) -> SharedHttpClient { - build_with_fn(self.client_builder, move || { - build_connector::https_with_resolver( - self.crypto_provider.crypto_provider.clone(), - resolver.clone(), - ) - }) + /// Create a [`SharedHttpClient`] from this builder and a given connector. + /// + #[cfg_attr( + feature = "legacy-rustls-ring", + doc = "Use [`build_https`](HyperClientBuilder::build_https) if you don't want to provide a custom TCP connector." + )] + pub fn build(self, tcp_connector: C) -> SharedHttpClient + where + C: Clone + Send + Sync + 'static, + C: hyper_0_14::service::Service, + C::Response: Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, + C::Future: Unpin + Send + 'static, + C::Error: Into, + { + self.build_with_fn(move || tcp_connector.clone()) } - /// Create a hyper client using a proxy connector - pub fn build_with_proxy(self, https_proxy: H, no_proxy: Option<&[N]>) -> SharedHttpClient + fn build_with_fn(self, tcp_connector_fn: F) -> SharedHttpClient where - H: AsRef + Clone + Send + Sync + 'static, - N: AsRef, + F: Fn() -> C + Send + Sync + 'static, + C: Clone + Send + Sync + 'static, + C: hyper_0_14::service::Service, + C::Response: Connection + AsyncRead + AsyncWrite + Send + Unpin + 'static, + C::Future: Unpin + Send + 'static, + C::Error: Into, { - let crypto = self.crypto_provider.crypto_provider; - let no_proxy: Option> = - no_proxy.map(|n| n.iter().map(|s| s.as_ref().to_owned()).collect()); - build_with_fn(self.client_builder, move || { - build_connector::https_with_proxy( - cached_connectors::cached_https(crypto.clone()), - https_proxy.as_ref(), - no_proxy.clone(), - ) + SharedHttpClient::new(HyperClient { + connector_cache: RwLock::new(HashMap::new()), + client_builder: self.client_builder.unwrap_or_default(), + tcp_connector_fn, }) } } -impl HyperClientBuilder { - /// Creates a new builder. - pub fn new() -> Self { - Self::default() - } - - pub fn crypto_mode(self, provider: CryptoMode) -> HyperClientBuilder { - HyperClientBuilder { - client_builder: self.client_builder, - crypto_provider: CryptoProviderSelected { - crypto_provider: Inner::Standard(provider), - }, - } - } - - /// This interface will be broken in the future - /// - /// This exposes `CryptoProvider` from `rustls` directly and this API has no stability guarantee. - #[cfg(crypto_unstable)] - pub fn crypto_provider_unstable( - self, - provider: CryptoProvider, - ) -> HyperClientBuilder { - HyperClientBuilder { - client_builder: self.client_builder, - crypto_provider: CryptoProviderSelected { - crypto_provider: Inner::Custom(provider), - }, - } - } -} - -fn build_with_fn( - client_builder: Option, - tcp_connector_fn: F, -) -> SharedHttpClient -where - F: Fn() -> C + Send + Sync + 'static, - C: Clone + Send + Sync + 'static, - C: tower::Service, - C::Response: Connection + Read + Write + Send + Sync + Unpin + 'static, - C::Future: Unpin + Send + 'static, - C::Error: Into, - C: Connect, -{ - SharedHttpClient::new(HyperClient { - connector_cache: RwLock::new(HashMap::new()), - client_builder: client_builder - .unwrap_or_else(|| hyper_util::client::legacy::Builder::new(TokioExecutor::new())), - tcp_connector_fn, - }) -} - mod timeout_middleware { + use aws_smithy_async::future::timeout::{TimedOutError, Timeout}; + use aws_smithy_async::rt::sleep::Sleep; + use aws_smithy_async::rt::sleep::{AsyncSleep, SharedAsyncSleep}; + use aws_smithy_runtime_api::box_error::BoxError; + use pin_project_lite::pin_project; use std::error::Error; use std::fmt::Formatter; use std::future::Future; @@ -804,14 +667,6 @@ mod timeout_middleware { use std::task::{Context, Poll}; use std::time::Duration; - use http::Uri; - use pin_project_lite::pin_project; - - use aws_smithy_async::future::timeout::{TimedOutError, Timeout}; - use aws_smithy_async::rt::sleep::Sleep; - use aws_smithy_async::rt::sleep::{AsyncSleep, SharedAsyncSleep}; - use aws_smithy_runtime_api::box_error::BoxError; - #[derive(Debug)] pub(crate) struct HttpTimeoutError { kind: &'static str, @@ -850,7 +705,7 @@ mod timeout_middleware { impl ConnectTimeout { /// Create a new `ConnectTimeout` around `inner`. /// - /// Typically, `I` will implement [`hyper_util::client::legacy::connect::Connect`]. + /// Typically, `I` will implement [`hyper_0_14::client::connect::Connect`]. pub(crate) fn new(inner: I, sleep: SharedAsyncSleep, timeout: Duration) -> Self { Self { inner, @@ -875,7 +730,7 @@ mod timeout_middleware { impl HttpReadTimeout { /// Create a new `HttpReadTimeout` around `inner`. /// - /// Typically, `I` will implement [`tower::Service>`]. + /// Typically, `I` will implement [`hyper_0_14::service::Service>`]. pub(crate) fn new(inner: I, sleep: SharedAsyncSleep, timeout: Duration) -> Self { Self { inner, @@ -939,9 +794,9 @@ mod timeout_middleware { } } - impl tower::Service for ConnectTimeout + impl hyper_0_14::service::Service for ConnectTimeout where - I: tower::Service, + I: hyper_0_14::service::Service, I::Error: Into, { type Response = I::Response; @@ -952,7 +807,7 @@ mod timeout_middleware { self.inner.poll_ready(cx).map_err(|err| err.into()) } - fn call(&mut self, req: Uri) -> Self::Future { + fn call(&mut self, req: http_02x::Uri) -> Self::Future { match &self.timeout { Some((sleep, duration)) => { let sleep = sleep.sleep(*duration); @@ -969,10 +824,9 @@ mod timeout_middleware { } } - impl tower::Service> for HttpReadTimeout + impl hyper_0_14::service::Service> for HttpReadTimeout where - I: tower::Service>, - I::Error: Send + Sync + Error + 'static, + I: hyper_0_14::service::Service, Error = hyper_0_14::Error>, { type Response = I::Response; type Error = BoxError; @@ -982,7 +836,7 @@ mod timeout_middleware { self.inner.poll_ready(cx).map_err(|err| err.into()) } - fn call(&mut self, req: http::Request) -> Self::Future { + fn call(&mut self, req: http_02x::Request) -> Self::Future { match &self.timeout { Some((sleep, duration)) => { let sleep = sleep.sleep(*duration); @@ -1001,19 +855,23 @@ mod timeout_middleware { #[cfg(test)] pub(crate) mod test { - use std::time::Duration; - - use hyper::rt::ReadBufCursor; - use hyper_util::client::legacy::connect::Connected; - use hyper_util::rt::TokioIo; - use tokio::net::TcpStream; - + use crate::hyper_014::HyperConnector; use aws_smithy_async::assert_elapsed; use aws_smithy_async::future::never::Never; use aws_smithy_async::rt::sleep::{SharedAsyncSleep, TokioSleep}; + use aws_smithy_runtime_api::box_error::BoxError; + use aws_smithy_runtime_api::client::http::HttpConnectorSettings; + use aws_smithy_runtime_api::client::orchestrator::HttpRequest; + use aws_smithy_runtime_api::client::result::ConnectorError; use aws_smithy_types::error::display::DisplayErrorContext; - - use super::super::*; + use hyper_0_14::client::connect::{Connected, Connection}; + use std::future::Future; + use std::pin::Pin; + use std::task::{Context, Poll}; + use std::time::Duration; + use tokio::io::ReadBuf; + use tokio::io::{AsyncRead, AsyncWrite}; + use tokio::net::TcpStream; #[allow(unused)] fn connect_timeout_is_correct() { @@ -1029,8 +887,8 @@ mod timeout_middleware { #[non_exhaustive] #[derive(Clone, Default, Debug)] pub(crate) struct NeverConnects; - impl tower::Service for NeverConnects { - type Response = TokioIo; + impl hyper_0_14::service::Service for NeverConnects { + type Response = TcpStream; type Error = ConnectorError; type Future = Pin> + Send>>; @@ -1038,7 +896,7 @@ mod timeout_middleware { Poll::Ready(Ok(())) } - fn call(&mut self, _uri: Uri) -> Self::Future { + fn call(&mut self, _uri: http_02x::Uri) -> Self::Future { Box::pin(async move { Never::new().await; unreachable!() @@ -1049,7 +907,7 @@ mod timeout_middleware { /// A service that will connect but never send any data #[derive(Clone, Debug, Default)] struct NeverReplies; - impl tower::Service for NeverReplies { + impl hyper_0_14::service::Service for NeverReplies { type Response = EmptyStream; type Error = BoxError; type Future = std::future::Ready>; @@ -1058,7 +916,7 @@ mod timeout_middleware { Poll::Ready(Ok(())) } - fn call(&mut self, _req: Uri) -> Self::Future { + fn call(&mut self, _req: http_02x::Uri) -> Self::Future { std::future::ready(Ok(EmptyStream)) } } @@ -1067,16 +925,16 @@ mod timeout_middleware { #[non_exhaustive] #[derive(Debug, Default)] struct EmptyStream; - impl Read for EmptyStream { + impl AsyncRead for EmptyStream { fn poll_read( self: Pin<&mut Self>, _cx: &mut Context<'_>, - _buf: ReadBufCursor<'_>, - ) -> Poll> { + _buf: &mut ReadBuf<'_>, + ) -> Poll> { Poll::Pending } } - impl Write for EmptyStream { + impl AsyncWrite for EmptyStream { fn poll_write( self: Pin<&mut Self>, _cx: &mut Context<'_>, @@ -1124,11 +982,12 @@ mod timeout_middleware { .unwrap_err(); assert!( resp.is_timeout(), - "expected resp.is_timeout() to be true but it was false, resp == {resp:?}" + "expected resp.is_timeout() to be true but it was false, resp == {:?}", + resp ); let message = DisplayErrorContext(&resp).to_string(); let expected = - "timeout: client error (Connect): HTTP connect timeout occurred after 1s"; + "timeout: error trying to connect: HTTP connect timeout occurred after 1s"; assert!( message.contains(expected), "expected '{message}' to contain '{expected}'" @@ -1171,32 +1030,31 @@ mod timeout_middleware { #[cfg(test)] mod test { + use crate::hyper_legacy::timeout_middleware::test::NeverConnects; + use crate::hyper_legacy::{HyperClientBuilder, HyperConnector}; + use aws_smithy_async::time::SystemTimeSource; + use aws_smithy_runtime_api::box_error::BoxError; + use aws_smithy_runtime_api::client::http::{HttpClient, HttpConnectorSettings}; + use aws_smithy_runtime_api::client::orchestrator::HttpRequest; + use aws_smithy_runtime_api::client::runtime_components::RuntimeComponentsBuilder; + use hyper_0_14::client::connect::{Connected, Connection}; use std::io::{Error, ErrorKind}; use std::pin::Pin; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; use std::task::{Context, Poll}; - - use http::Uri; - use hyper::rt::ReadBufCursor; - use hyper_util::client::legacy::connect::Connected; - - use aws_smithy_async::time::SystemTimeSource; - use aws_smithy_runtime_api::client::runtime_components::RuntimeComponentsBuilder; - - use crate::hyper_1_0::timeout_middleware::test::NeverConnects; - - use super::*; + use std::time::Duration; + use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; #[tokio::test] async fn connector_selection() { // Create a client that increments a count every time it creates a new HyperConnector let creation_count = Arc::new(AtomicU32::new(0)); - let http_client = build_with_fn(None, { + let http_client = HyperClientBuilder::new().build_with_fn({ let count = creation_count.clone(); move || { count.fetch_add(1, Ordering::Relaxed); - NeverConnects + NeverConnects::default() } }); @@ -1254,7 +1112,7 @@ mod test { .call(HttpRequest::get("https://socket-hangup.com").unwrap()) .await .expect_err("socket hangup"); - assert!(err.is_io(), "unexpected error type: {err:?}"); + assert!(err.is_io(), "{:?}", err); } // ---- machinery to make a Hyper connector that responds with an IO Error @@ -1267,11 +1125,11 @@ mod test { } } - impl Read for HangupStream { + impl AsyncRead for HangupStream { fn poll_read( self: Pin<&mut Self>, _cx: &mut Context<'_>, - _buf: ReadBufCursor<'_>, + _buf: &mut ReadBuf<'_>, ) -> Poll> { Poll::Ready(Err(Error::new( ErrorKind::ConnectionReset, @@ -1280,7 +1138,7 @@ mod test { } } - impl Write for HangupStream { + impl AsyncWrite for HangupStream { fn poll_write( self: Pin<&mut Self>, _cx: &mut Context<'_>, @@ -1303,7 +1161,7 @@ mod test { inner: T, } - impl tower::Service for TestConnection + impl hyper_0_14::service::Service for TestConnection where T: Clone + Connection, { @@ -1315,7 +1173,7 @@ mod test { Poll::Ready(Ok(())) } - fn call(&mut self, _req: Uri) -> Self::Future { + fn call(&mut self, _req: http_02x::Uri) -> Self::Future { std::future::ready(Ok(self.inner.clone())) } } diff --git a/sources/aws-smithy-http-client/src/lib.rs b/sources/aws-smithy-http-client/src/lib.rs new file mode 100644 index 000000000..85cae6880 --- /dev/null +++ b/sources/aws-smithy-http-client/src/lib.rs @@ -0,0 +1,97 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* Automatically managed default lints */ +#![cfg_attr(docsrs, feature(doc_cfg))] +/* End of automatically managed default lints */ + +//! HTTP client implementation for smithy-rs generated code. +//! +//! # Crate Features +//! +//! - `default-client`: Enable default HTTP client implementation (based on hyper 1.x). +//! - `rustls-ring`: Enable TLS provider based on `rustls` using `ring` as the crypto provider +//! - `rustls-aws-lc`: Enable TLS provider based on `rustls` using `aws-lc` as the crypto provider +//! - `rustls-aws-lc-fips`: Same as `rustls-aws-lc` feature but using a FIPS compliant version of `aws-lc` +//! - `s2n-tls`: Enable TLS provider based on `s2n-tls` using `aws-lc` as the crypto provider. +//! - `hyper-014`: (Deprecated) HTTP client implementation based on hyper-0.14.x. +//! - `test-util`: Enables utilities for unit tests. DO NOT ENABLE IN PRODUCTION. + +#![warn( + missing_docs, + rustdoc::missing_crate_level_docs, + unreachable_pub, + rust_2018_idioms +)] + +// ideally hyper_014 would just be exposed as is but due to +// https://github.com/rust-lang/rust/issues/47238 we get clippy warnings we can't suppress +#[cfg(feature = "hyper-014")] +pub(crate) mod hyper_legacy; + +/// Legacy HTTP and TLS connectors that use hyper 0.14.x and rustls. +#[cfg(feature = "hyper-014")] +#[deprecated = "hyper 0.14.x support is deprecated, please migrate to 1.x client"] +pub mod hyper_014 { + pub use crate::hyper_legacy::*; +} + +/// Default HTTP and TLS connectors +#[cfg(feature = "default-client")] +pub(crate) mod client; +#[cfg(feature = "default-client")] +pub use client::{default_connector, proxy, tls, Builder, Connector, ConnectorBuilder}; + +#[cfg(feature = "test-util")] +pub mod test_util; + +mod error; +pub use error::HttpClientError; + +#[allow(unused_macros, unused_imports)] +#[macro_use] +pub(crate) mod cfg { + /// Any TLS provider enabled + macro_rules! cfg_tls { + ($($item:item)*) => { + $( + #[cfg(any( + feature = "__rustls", + feature = "s2n-tls", + ))] + #[cfg_attr(docsrs, doc(cfg(any( + feature = "__rustls", + feature = "s2n-tls", + ))))] + $item + )* + } + } + + /// Any rustls provider enabled + macro_rules! cfg_rustls { + ($($item:item)*) => { + $( + #[cfg(feature = "__rustls")] + #[cfg_attr(docsrs, doc(cfg(feature = "__rustls")))] + $item + )* + } + } + + macro_rules! cfg_s2n_tls { + ($($item:item)*) => { + $( + #[cfg(feature = "s2n-tls")] + #[cfg_attr(docsrs, doc(cfg(feature = "s2n-tls")))] + $item + )* + } + } + + pub(crate) use cfg_rustls; + pub(crate) use cfg_s2n_tls; + pub(crate) use cfg_tls; +} diff --git a/sources/aws-smithy-http-client/src/test_util.rs b/sources/aws-smithy-http-client/src/test_util.rs new file mode 100644 index 000000000..b3ac6e254 --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util.rs @@ -0,0 +1,67 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Various fake/mock clients for testing. +//! +//! Each test client is useful for different test use cases: +//! - [`capture_request()`]: If you don't care what the response is, but just want to +//! check that the serialized request is what you expect, then use `capture_request`. +//! Or, alternatively, if you don't care what the request is, but want to always +//! respond with a given response, then capture request can also be useful since +//! you can optionally give it a response to return. +#![cfg_attr( + feature = "default-client", + doc = "- [`dvr`]: If you want to record real-world traffic and then replay it later, then DVR's" +)] +//! [`RecordingClient`](dvr::RecordingClient) and [`ReplayingClient`](dvr::ReplayingClient) +//! can accomplish this, and the recorded traffic can be saved to JSON and checked in. Note: if +//! the traffic recording has sensitive information in it, such as signatures or authorization, +//! you will need to manually scrub this out if you intend to store the recording alongside +//! your tests. +//! - [`StaticReplayClient`]: If you want to have a set list of requests and their responses in a test, +//! then the static replay client will be useful. On construction, it takes a list of request/response +//! pairs that represent each expected request and the response for that test. At the end of the test, +//! you can ask the client to verify that the requests matched the expectations. +//! - [`infallible_client_fn`]: Allows you to create a client from an infallible function +//! that takes a request and returns a response. +//! - [`NeverClient`]: Useful for testing timeouts, where you want the client to never respond. +//! +#![cfg_attr( + any(feature = "hyper-014", feature = "default-client"), + doc = " +There is also the [`NeverTcpConnector`], which makes it easy to test connect/read timeouts. + +Finally, for socket-level mocking, see the [`wire`] module. +" +)] + +mod capture_request; +pub use capture_request::{capture_request, CaptureRequestHandler, CaptureRequestReceiver}; + +#[cfg(feature = "legacy-test-util")] +pub use capture_request::legacy_capture_request; + +pub mod dvr; + +mod replay; +pub use replay::{ReplayEvent, StaticReplayClient}; + +mod infallible; +pub use infallible::infallible_client_fn; + +// infallible based on http_02x stack had to be duplicated to avoid breaking API changes +#[allow(missing_docs)] +#[cfg(feature = "legacy-test-util")] +pub mod legacy_infallible; + +mod never; +pub use never::NeverClient; + +#[cfg(any(feature = "hyper-014", feature = "default-client"))] +pub use never::NeverTcpConnector; + +mod body; +#[cfg(all(feature = "default-client", feature = "wire-mock"))] +pub mod wire; diff --git a/sources/aws-smithy-http-client/src/test_util/body.rs b/sources/aws-smithy-http-client/src/test_util/body.rs new file mode 100644 index 000000000..031e1a790 --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util/body.rs @@ -0,0 +1,95 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_smithy_runtime_api::box_error::BoxError; +use aws_smithy_types::body::SdkBody; +use bytes::Bytes; +use http_body_1x::{Frame, SizeHint}; +use pin_project_lite::pin_project; +use std::future::poll_fn; +use std::pin::{pin, Pin}; +use std::task::{Context, Poll}; +use tokio::sync::mpsc; + +/// Create a `SdkBody` with an associated sender half. +/// +/// Useful for sending data from another thread/task and test scenarios. +pub(crate) fn channel_body() -> (Sender, SdkBody) { + let (tx, rx) = mpsc::channel(1); + let sender = Sender { tx }; + let ch_body = ChannelBody { rx }; + (sender, SdkBody::from_body_1_x(ch_body)) +} + +/// Sender half of channel based `SdkBody` implementation useful for testing. +/// +/// Roughly a replacement for hyper 0.14.x `Sender` body. +/// +/// ## Body Closing +/// +/// The request body will always be closed normally when the sender is dropped. If you +/// want to close the connection with an incomplete response, call [`Sender::abort()`] method to +/// abort the body in an abnormal fashion. +#[derive(Debug)] +pub(crate) struct Sender { + tx: mpsc::Sender, BoxError>>, +} + +impl Sender { + /// Send data on data channel when it's ready + pub(crate) async fn send_data(&mut self, chunk: Bytes) -> Result<(), BoxError> { + let frame = Frame::data(chunk); + self.tx.send(Ok(frame)).await.map_err(|e| e.into()) + } + + // TODO(test-utils): we can add support for trailers if needed in the future + + /// Abort the body in an abnormal fashion + pub(crate) fn abort(self) { + let _ = self.tx.clone().try_send(Err("body write aborted".into())); + } +} + +pin_project! { + struct ChannelBody { + rx: mpsc::Receiver, BoxError>> + } +} + +impl http_body_1x::Body for ChannelBody { + type Data = Bytes; + type Error = BoxError; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + let this = self.project(); + this.rx.poll_recv(cx) + } + + fn is_end_stream(&self) -> bool { + self.rx.is_closed() + } + + fn size_hint(&self) -> SizeHint { + SizeHint::default() + } +} + +pub(crate) async fn next_data_frame(body: &mut SdkBody) -> Option> { + use http_body_1x::Body; + let mut pinned = pin!(body); + match poll_fn(|cx| pinned.as_mut().poll_frame(cx)).await? { + Ok(frame) => { + if frame.is_data() { + Some(Ok(frame.into_data().unwrap())) + } else { + None + } + } + Err(err) => Some(Err(err)), + } +} diff --git a/sources/aws-smithy-http-client/src/test_util/capture_request.rs b/sources/aws-smithy-http-client/src/test_util/capture_request.rs new file mode 100644 index 000000000..ca0da601f --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util/capture_request.rs @@ -0,0 +1,172 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_smithy_runtime_api::client::connector_metadata::ConnectorMetadata; +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse}; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::http::HttpError; +use aws_smithy_runtime_api::shared::IntoShared; +use aws_smithy_types::body::SdkBody; +use std::fmt::Debug; +use std::sync::{Arc, Mutex}; +use tokio::sync::oneshot; + +#[derive(Debug)] +struct Inner { + response: Option, + sender: Option>, +} + +/// Test Connection to capture a single request +#[derive(Debug, Clone)] +pub struct CaptureRequestHandler(Arc>); + +impl HttpConnector for CaptureRequestHandler { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let mut inner = self.0.lock().unwrap(); + if let Err(_e) = inner.sender.take().expect("already sent").send(request) { + tracing::trace!("The receiver was already dropped"); + } + HttpConnectorFuture::ready(Ok(inner + .response + .take() + .expect("could not handle second request"))) + } +} + +impl HttpClient for CaptureRequestHandler { + fn http_connector( + &self, + _: &HttpConnectorSettings, + _: &RuntimeComponents, + ) -> SharedHttpConnector { + self.clone().into_shared() + } + + fn connector_metadata(&self) -> Option { + Some(ConnectorMetadata::new("capture-request-handler", None)) + } +} + +/// Receiver for [`CaptureRequestHandler`]. +#[derive(Debug)] +pub struct CaptureRequestReceiver { + receiver: oneshot::Receiver, +} + +impl CaptureRequestReceiver { + /// Expect that a request was sent. Returns the captured request. + /// + /// # Panics + /// If no request was received + #[track_caller] + pub fn expect_request(mut self) -> HttpRequest { + self.receiver.try_recv().expect("no request was received") + } + + /// Expect that no request was captured. Panics if a request was received. + /// + /// # Panics + /// If a request was received + #[track_caller] + pub fn expect_no_request(mut self) { + self.receiver + .try_recv() + .expect_err("expected no request to be received!"); + } +} + +/// Test connection used to capture a single request +/// +/// If response is `None`, it will reply with a 200 response with an empty body +/// +/// Example: +/// ```compile_fail +/// let (capture_client, request) = capture_request(None); +/// let conf = aws_sdk_sts::Config::builder() +/// .http_client(capture_client) +/// .build(); +/// let client = aws_sdk_sts::Client::from_conf(conf); +/// let _ = client.assume_role_with_saml().send().await; +/// // web identity should be unsigned +/// assert_eq!( +/// request.expect_request().headers().get("AUTHORIZATION"), +/// None +/// ); +/// ``` +pub fn capture_request( + response: Option>, +) -> (CaptureRequestHandler, CaptureRequestReceiver) { + capture_request_inner(response) +} + +fn capture_request_inner( + response: Option>, +) -> (CaptureRequestHandler, CaptureRequestReceiver) { + let (tx, rx) = oneshot::channel(); + let http_resp: HttpResponse = match response { + Some(resp) => resp.try_into().expect("valid HttpResponse"), + None => http_1x::Response::builder() + .status(200) + .body(SdkBody::empty()) + .expect("unreachable") + .try_into() + .expect("unreachable"), + }; + ( + CaptureRequestHandler(Arc::new(Mutex::new(Inner { + response: Some(http_resp), + sender: Some(tx), + }))), + CaptureRequestReceiver { receiver: rx }, + ) +} + +#[allow(missing_docs)] +#[cfg(feature = "legacy-test-util")] +pub fn legacy_capture_request( + response: Option>, +) -> (CaptureRequestHandler, CaptureRequestReceiver) { + capture_request_inner(response) +} + +#[cfg(test)] +mod test { + use aws_smithy_runtime_api::client::http::HttpConnector; + use aws_smithy_runtime_api::client::orchestrator::HttpRequest; + use aws_smithy_types::body::SdkBody; + + #[cfg(feature = "legacy-test-util")] + #[tokio::test] + async fn test_can_plug_in_http_02x() { + use super::legacy_capture_request; + let (capture_client, _request) = legacy_capture_request(Some( + http_02x::Response::builder() + .status(202) + .body(SdkBody::empty()) + .expect("unreachable"), + )); + + let resp = capture_client.call(HttpRequest::empty()).await.unwrap(); + assert_eq!(202, resp.status().as_u16()); + } + + #[tokio::test] + async fn test_can_plug_in_http_1x() { + use super::capture_request; + let (capture_client, _request) = capture_request(Some( + http_1x::Response::builder() + .status(202) + .body(SdkBody::empty()) + .expect("unreachable"), + )); + + let resp = capture_client.call(HttpRequest::empty()).await.unwrap(); + assert_eq!(202, resp.status().as_u16()); + } +} diff --git a/sources/aws-smithy-http-client/src/test_util/dvr.rs b/sources/aws-smithy-http-client/src/test_util/dvr.rs new file mode 100644 index 000000000..450fdec8b --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util/dvr.rs @@ -0,0 +1,419 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Extremely Experimental Test Connection +//! +//! Warning: Extremely experimental, API likely to change. +//! +//! DVR is an extremely experimental record & replay framework that supports multi-frame HTTP request / response traffic. + +use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse}; +use aws_smithy_runtime_api::http::Headers; +use aws_smithy_types::base64; +use bytes::Bytes; +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::Path; + +mod record; +mod replay; + +pub use record::RecordingClient; +pub use replay::ReplayingClient; + +/// A complete traffic recording +/// +/// A traffic recording can be replayed with [`RecordingClient`]. +#[derive(Debug, Serialize, Deserialize)] +pub struct NetworkTraffic { + events: Vec, + docs: Option, + version: Version, +} + +impl NetworkTraffic { + /// Network events + pub fn events(&self) -> &Vec { + &self.events + } + + /// Create a NetworkTraffic instance from a file + pub fn from_file(path: impl AsRef) -> Result> { + let contents = std::fs::read_to_string(path)?; + Ok(serde_json::from_str(&contents)?) + } + + /// Create a NetworkTraffic instance from a file + pub fn write_to_file(&self, path: impl AsRef) -> Result<(), Box> { + let serialized = serde_json::to_string_pretty(&self)?; + Ok(std::fs::write(path, serialized)?) + } + + /// Update the network traffic with all `content-length` fields fixed to match the contents + pub fn correct_content_lengths(&mut self) { + let mut content_lengths: HashMap<(ConnectionId, Direction), usize> = HashMap::new(); + for event in &self.events { + if let Action::Data { data, direction } = &event.action { + let entry = content_lengths.entry((event.connection_id, *direction)); + *entry.or_default() += data.copy_to_vec().len(); + } + } + for event in &mut self.events { + let (headers, direction) = match &mut event.action { + Action::Request { + request: Request { headers, .. }, + } => (headers, Direction::Request), + Action::Response { + response: Ok(Response { headers, .. }), + } => (headers, Direction::Response), + _ => continue, + }; + let Some(computed_content_length) = + content_lengths.get(&(event.connection_id, direction)) + else { + continue; + }; + if headers.contains_key("content-length") { + headers.insert( + "content-length".to_string(), + vec![computed_content_length.to_string()], + ); + } + } + } +} + +/// Serialization version of DVR data +#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum Version { + /// Initial network traffic version + V0, +} + +/// A network traffic recording may contain multiple different connections occurring simultaneously +#[derive(Copy, Clone, Debug, Serialize, Deserialize, Hash, PartialEq, Eq)] +pub struct ConnectionId(usize); + +/// A network event +/// +/// Network events consist of a connection identifier and an action. An event is sufficient to +/// reproduce traffic later during replay +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct Event { + connection_id: ConnectionId, + action: Action, +} + +/// An initial HTTP request, roughly equivalent to `http::Request<()>` +/// +/// The initial request phase of an HTTP request. The body will be +/// sent later as a separate action. +#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)] +pub struct Request { + uri: String, + headers: IndexMap>, + method: String, +} + +/// An initial HTTP response roughly equivalent to `http::Response<()>` +/// +/// The initial response phase of an HTTP request. The body will be +/// sent later as a separate action. +#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)] +pub struct Response { + status: u16, + headers: IndexMap>, +} + +#[cfg(feature = "legacy-test-util")] +impl From<&Request> for http_02x::Request<()> { + fn from(request: &Request) -> Self { + let mut builder = http_02x::Request::builder().uri(request.uri.as_str()); + for (k, values) in request.headers.iter() { + for v in values { + builder = builder.header(k, v); + } + } + builder.method(request.method.as_str()).body(()).unwrap() + } +} + +impl From<&Request> for http_1x::Request<()> { + fn from(request: &Request) -> Self { + let mut builder = http_1x::Request::builder().uri(request.uri.as_str()); + for (k, values) in request.headers.iter() { + for v in values { + builder = builder.header(k, v); + } + } + builder.method(request.method.as_str()).body(()).unwrap() + } +} + +impl<'a> From<&'a HttpRequest> for Request { + fn from(req: &'a HttpRequest) -> Self { + let uri = req.uri().to_string(); + let headers = headers_to_map_http(req.headers()); + let method = req.method().to_string(); + Self { + uri, + headers, + method, + } + } +} + +fn headers_to_map_http(headers: &Headers) -> IndexMap> { + let mut out: IndexMap<_, Vec<_>> = IndexMap::new(); + for (header_name, header_value) in headers.iter() { + let entry = out.entry(header_name.to_string()).or_default(); + entry.push(header_value.to_string()); + } + out +} + +fn headers_to_map(headers: &Headers) -> IndexMap> { + let mut out: IndexMap<_, Vec<_>> = IndexMap::new(); + for (header_name, header_value) in headers.iter() { + let entry = out.entry(header_name.to_string()).or_default(); + entry.push( + std::str::from_utf8(header_value.as_ref()) + .unwrap() + .to_string(), + ); + } + out +} + +#[cfg(feature = "legacy-test-util")] +fn headers_to_map_02x(headers: &http_02x::HeaderMap) -> IndexMap> { + let mut out: IndexMap<_, Vec<_>> = IndexMap::new(); + for (header_name, header_value) in headers.iter() { + let entry = out.entry(header_name.to_string()).or_default(); + entry.push( + std::str::from_utf8(header_value.as_ref()) + .unwrap() + .to_string(), + ); + } + out +} + +#[cfg(feature = "legacy-test-util")] +impl<'a, B> From<&'a http_02x::Response> for Response { + fn from(resp: &'a http_02x::Response) -> Self { + let status = resp.status().as_u16(); + let headers = headers_to_map_02x(resp.headers()); + Self { status, headers } + } +} + +fn headers_to_map_1x(headers: &http_1x::HeaderMap) -> IndexMap> { + let mut out: IndexMap<_, Vec<_>> = IndexMap::new(); + for (header_name, header_value) in headers.iter() { + let entry = out.entry(header_name.to_string()).or_default(); + entry.push( + std::str::from_utf8(header_value.as_ref()) + .unwrap() + .to_string(), + ); + } + out +} + +impl<'a, B> From<&'a http_1x::Response> for Response { + fn from(resp: &'a http_1x::Response) -> Self { + let status = resp.status().as_u16(); + let headers = headers_to_map_1x(resp.headers()); + Self { status, headers } + } +} + +impl From<&HttpResponse> for Response { + fn from(resp: &HttpResponse) -> Self { + Self { + status: resp.status().into(), + headers: headers_to_map(resp.headers()), + } + } +} + +/// Error response wrapper +#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)] +pub struct Error(String); + +/// Network Action +#[derive(Debug, Serialize, Deserialize, Clone, Eq, PartialEq)] +#[non_exhaustive] +pub enum Action { + /// Initial HTTP Request + Request { + /// HTTP Request headers, method, and URI + request: Request, + }, + + /// Initial HTTP response or failure + Response { + /// HTTP response or failure + response: Result, + }, + + /// Data segment + Data { + /// Body Data + data: BodyData, + /// Direction: request vs. response + direction: Direction, + }, + + /// End of data + Eof { + /// Succesful vs. failed termination + ok: bool, + /// Direction: request vs. response + direction: Direction, + }, +} + +/// Event direction +/// +/// During replay, this is used to replay data in the right direction +#[derive(Copy, Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Hash)] +pub enum Direction { + /// Request phase + Request, + /// Response phase + Response, +} + +impl Direction { + /// The opposite of a given direction + pub fn opposite(self) -> Self { + match self { + Direction::Request => Direction::Response, + Direction::Response => Direction::Request, + } + } +} + +/// HTTP Body Data Abstraction +/// +/// When the data is a UTF-8 encoded string, it will be serialized as a string for readability. +/// Otherwise, it will be base64 encoded. +#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] +#[non_exhaustive] +pub enum BodyData { + /// UTF-8 encoded data + Utf8(String), + + /// Base64 encoded binary data + Base64(String), +} + +impl BodyData { + /// Convert [`BodyData`] into Bytes. + pub fn into_bytes(self) -> Vec { + match self { + BodyData::Utf8(string) => string.into_bytes(), + BodyData::Base64(string) => base64::decode(string).unwrap(), + } + } + + /// Copy [`BodyData`] into a `Vec`. + pub fn copy_to_vec(&self) -> Vec { + match self { + BodyData::Utf8(string) => string.as_bytes().into(), + BodyData::Base64(string) => base64::decode(string).unwrap(), + } + } +} + +impl From for BodyData { + fn from(data: Bytes) -> Self { + match std::str::from_utf8(data.as_ref()) { + Ok(string) => BodyData::Utf8(string.to_string()), + Err(_) => BodyData::Base64(base64::encode(data)), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::error::Error; + use std::fs; + + use aws_smithy_runtime_api::client::http::HttpConnector; + use aws_smithy_runtime_api::client::http::SharedHttpConnector; + use aws_smithy_types::body::SdkBody; + use aws_smithy_types::byte_stream::ByteStream; + + #[tokio::test] + async fn correctly_fixes_content_lengths() -> Result<(), Box> { + let network_traffic = fs::read_to_string("test-data/example.com.json")?; + let mut network_traffic: NetworkTraffic = serde_json::from_str(&network_traffic)?; + network_traffic.correct_content_lengths(); + let Action::Request { + request: Request { headers, .. }, + } = &network_traffic.events[0].action + else { + panic!("unexpected event") + }; + // content length is not added when it wasn't initially present + assert_eq!(headers.get("content-length"), None); + + let Action::Response { + response: Ok(Response { headers, .. }), + } = &network_traffic.events[3].action + else { + panic!("unexpected event: {:?}", network_traffic.events[3].action); + }; + // content length is not added when it wasn't initially present + let expected_length = "hello from example.com".len(); + assert_eq!( + headers.get("content-length"), + Some(&vec![expected_length.to_string()]) + ); + Ok(()) + } + + #[cfg(feature = "legacy-test-util")] + #[tokio::test] + async fn turtles_all_the_way_down() -> Result<(), Box> { + // create a replaying connection from a recording, wrap a recording connection around it, + // make a request, then verify that the same traffic was recorded. + let network_traffic = fs::read_to_string("test-data/example.com.json")?; + let mut network_traffic: NetworkTraffic = serde_json::from_str(&network_traffic)?; + network_traffic.correct_content_lengths(); + let inner = ReplayingClient::new(network_traffic.events.clone()); + let connection = RecordingClient::new(SharedHttpConnector::new(inner.clone())); + let req = http_02x::Request::post("https://www.example.com") + .body(SdkBody::from("hello world")) + .unwrap(); + let mut resp = connection.call(req.try_into().unwrap()).await.expect("ok"); + let body = std::mem::replace(resp.body_mut(), SdkBody::taken()); + let data = ByteStream::new(body).collect().await.unwrap().into_bytes(); + assert_eq!( + String::from_utf8(data.to_vec()).unwrap(), + "hello from example.com" + ); + assert_eq!( + connection.events().as_slice(), + network_traffic.events.as_slice() + ); + let requests = inner.take_requests().await; + assert_eq!( + requests[0].uri(), + &http_02x::Uri::from_static("https://www.example.com") + ); + assert_eq!( + requests[0].body(), + &Bytes::from_static("hello world".as_bytes()) + ); + Ok(()) + } +} diff --git a/sources/aws-smithy-http-client/src/test_util/dvr/record.rs b/sources/aws-smithy-http-client/src/test_util/dvr/record.rs new file mode 100644 index 000000000..51759dfeb --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util/dvr/record.rs @@ -0,0 +1,262 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use super::{ + Action, BodyData, ConnectionId, Direction, Error, Event, NetworkTraffic, Request, Response, + Version, +}; +use aws_smithy_runtime_api::client::connector_metadata::ConnectorMetadata; +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::HttpRequest; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::shared::IntoShared; +use aws_smithy_types::body::SdkBody; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::{fs, io}; +use tokio::task::JoinHandle; + +/// Recording client +/// +/// `RecordingClient` wraps an inner connection and records all traffic, enabling traffic replay. +/// +/// # Example +/// +/// ```rust,ignore +/// use aws_smithy_async::rt::sleep::default_async_sleep; +/// use aws_smithy_runtime::client::http::hyper_014::default_connector; +/// use aws_smithy_http_client::test_util::dvr::RecordingClient; +/// use aws_smithy_runtime_api::client::http::HttpConnectorSettingsBuilder; +/// use aws_sdk_s3::{Client, Config}; +/// +/// #[tokio::test] +/// async fn test_content_length_enforcement_is_not_applied_to_head_request() { +/// let settings = HttpConnectorSettingsBuilder::default().build(); +/// let http_client = default_connector(&settings, default_async_sleep()).unwrap(); +/// let http_client = RecordingClient::new(http_client); +/// +/// // Since we need to send a real request for this, +/// // you'll need to use your real credentials. +/// let config = aws_config::load_defaults(BehaviorVersion::latest()).await; +/// let config = Config::from(&config).to_builder() +/// .http_client(http_client.clone()) +/// .region(Region::new("us-east-1")) +/// .build(); +/// +/// let client = Client::from_conf(config); +/// let _resp = client +/// .head_object() +/// .key("some-test-file.txt") +/// .bucket("your-test-bucket") +/// .send() +/// .await +/// .unwrap(); +/// +/// // If the request you want to record has a body, don't forget to poll +/// // the body to completion BEFORE calling `dump_to_file`. Otherwise, your +/// // test json won't include the body. +/// // let _body = _resp.body.collect().await.unwrap(); +/// +/// // This path is relative to your project or workspace `Cargo.toml` file. +/// http_client.dump_to_file("tests/data/content-length-enforcement/head-object.json").unwrap(); +/// } +/// ``` +#[derive(Clone, Debug)] +pub struct RecordingClient { + pub(crate) data: Arc>>, + pub(crate) num_events: Arc, + pub(crate) inner: SharedHttpConnector, +} + +#[cfg(feature = "legacy-rustls-ring")] +impl RecordingClient { + /// Construct a recording connection wrapping a default HTTPS implementation without any timeouts. + pub fn https() -> Self { + #[allow(deprecated)] + use crate::hyper_014::HyperConnector; + Self { + data: Default::default(), + num_events: Arc::new(AtomicUsize::new(0)), + #[allow(deprecated)] + inner: SharedHttpConnector::new(HyperConnector::builder().build_https()), + } + } +} + +impl RecordingClient { + /// Create a new recording connection from a connection + pub fn new(underlying_connector: impl HttpConnector + 'static) -> Self { + Self { + data: Default::default(), + num_events: Arc::new(AtomicUsize::new(0)), + inner: underlying_connector.into_shared(), + } + } + + /// Return the traffic recorded by this connection + pub fn events(&self) -> MutexGuard<'_, Vec> { + self.data.lock().unwrap() + } + + /// NetworkTraffic struct suitable for serialization + pub fn network_traffic(&self) -> NetworkTraffic { + NetworkTraffic { + events: self.events().clone(), + docs: Some("todo docs".into()), + version: Version::V0, + } + } + + /// Dump the network traffic to a file + pub fn dump_to_file(&self, path: impl AsRef) -> Result<(), io::Error> { + fs::write( + path, + serde_json::to_string(&self.network_traffic()).unwrap(), + ) + } + + fn next_id(&self) -> ConnectionId { + ConnectionId(self.num_events.fetch_add(1, Ordering::Relaxed)) + } +} + +fn record_body( + body: &mut SdkBody, + event_id: ConnectionId, + direction: Direction, + event_bus: Arc>>, +) -> JoinHandle<()> { + let (sender, output_body) = crate::test_util::body::channel_body(); + let real_body = std::mem::replace(body, output_body); + tokio::spawn(async move { + let mut real_body = real_body; + let mut sender = sender; + loop { + let data = crate::test_util::body::next_data_frame(&mut real_body).await; + match data { + Some(Ok(data)) => { + event_bus.lock().unwrap().push(Event { + connection_id: event_id, + action: Action::Data { + data: BodyData::from(data.clone()), + direction, + }, + }); + // This happens if the real connection is closed during recording. + // Need to think more carefully if this is the correct thing to log in this + // case. + if sender.send_data(data).await.is_err() { + event_bus.lock().unwrap().push(Event { + connection_id: event_id, + action: Action::Eof { + direction: direction.opposite(), + ok: false, + }, + }) + }; + } + None => { + event_bus.lock().unwrap().push(Event { + connection_id: event_id, + action: Action::Eof { + ok: true, + direction, + }, + }); + drop(sender); + break; + } + Some(Err(_err)) => { + event_bus.lock().unwrap().push(Event { + connection_id: event_id, + action: Action::Eof { + ok: false, + direction, + }, + }); + sender.abort(); + break; + } + } + } + }) +} + +impl HttpConnector for RecordingClient { + fn call(&self, mut request: HttpRequest) -> HttpConnectorFuture { + let event_id = self.next_id(); + // A request has three phases: + // 1. A "Request" phase. This is initial HTTP request, headers, & URI + // 2. A body phase. This may contain multiple data segments. + // 3. A finalization phase. An EOF of some sort is sent on the body to indicate that + // the channel should be closed. + + // Phase 1: the initial http request + self.data.lock().unwrap().push(Event { + connection_id: event_id, + action: Action::Request { + request: Request::from(&request), + }, + }); + + // Phase 2: Swap out the real request body for one that will log all traffic that passes + // through it + // This will also handle phase three when the request body runs out of data. + record_body( + request.body_mut(), + event_id, + Direction::Request, + self.data.clone(), + ); + let events = self.data.clone(); + // create a channel we'll use to stream the data while reading it + let resp_fut = self.inner.call(request); + let fut = async move { + let resp = resp_fut.await; + match resp { + Ok(mut resp) => { + // push the initial response event + events.lock().unwrap().push(Event { + connection_id: event_id, + action: Action::Response { + response: Ok(Response::from(&resp)), + }, + }); + + // instrument the body and record traffic + record_body(resp.body_mut(), event_id, Direction::Response, events); + Ok(resp) + } + Err(e) => { + events.lock().unwrap().push(Event { + connection_id: event_id, + action: Action::Response { + response: Err(Error(format!("{}", &e))), + }, + }); + Err(e) + } + } + }; + HttpConnectorFuture::new(fut) + } +} + +impl HttpClient for RecordingClient { + fn http_connector( + &self, + _: &HttpConnectorSettings, + _: &RuntimeComponents, + ) -> SharedHttpConnector { + self.clone().into_shared() + } + + fn connector_metadata(&self) -> Option { + Some(ConnectorMetadata::new("recording-client", None)) + } +} diff --git a/sources/aws-smithy-http-client/src/test_util/dvr/replay.rs b/sources/aws-smithy-http-client/src/test_util/dvr/replay.rs new file mode 100644 index 000000000..d5f1f2294 --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util/dvr/replay.rs @@ -0,0 +1,429 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use super::{Action, ConnectionId, Direction, Event, NetworkTraffic}; +use crate::test_util::replay::DEFAULT_RELAXED_HEADERS; +use aws_smithy_protocol_test::MediaType; +use aws_smithy_runtime_api::client::connector_metadata::ConnectorMetadata; +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse}; +use aws_smithy_runtime_api::client::result::ConnectorError; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::shared::IntoShared; +use aws_smithy_types::body::SdkBody; +use aws_smithy_types::error::display::DisplayErrorContext; +use bytes::{Bytes, BytesMut}; +use std::collections::{HashMap, VecDeque}; +use std::error::Error; +use std::fmt; +use std::ops::DerefMut; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::task::JoinHandle; + +/// Wrapper type to enable optionally waiting for a future to complete +#[derive(Debug)] +enum Waitable { + Loading(JoinHandle), + Value(T), +} + +impl Waitable { + /// Consumes the future and returns the value + async fn take(self) -> T { + match self { + Waitable::Loading(f) => f.await.expect("join failed"), + Waitable::Value(value) => value, + } + } + + /// Waits for the future to be ready + async fn wait(&mut self) { + match self { + Waitable::Loading(f) => *self = Waitable::Value(f.await.expect("join failed")), + Waitable::Value(_) => {} + } + } +} + +/// Replay traffic recorded by a [`RecordingClient`](super::RecordingClient) +#[derive(Clone)] +pub struct ReplayingClient { + live_events: Arc>>>, + verifiable_events: Arc>>, + num_events: Arc, + recorded_requests: Arc>>>>, +} + +// Ideally, this would just derive Debug, but that makes the tests in aws-config think they found AWS secrets +// when really it's just the test response data they're seeing from the Debug impl of this client. +// This is just a quick workaround. A better fix can be considered later. +impl fmt::Debug for ReplayingClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("test_util::dvr::ReplayingClient") + } +} + +enum HeadersToCheck<'a> { + Include(&'a [&'a str]), + Exclude(Option<&'a [&'a str]>), +} + +impl ReplayingClient { + fn next_id(&self) -> ConnectionId { + ConnectionId(self.num_events.fetch_add(1, Ordering::Relaxed)) + } + + /// Validate all headers and bodies + pub async fn full_validate(self, media_type: &str) -> Result<(), Box> { + self.validate_body_and_headers(None, media_type).await + } + + /// Convenience method to validate that the bodies match, using a given [`MediaType`] for + /// comparison, and that the headers are also match excluding the default relaxed headers + /// + /// The current default relaxed headers: + /// - x-amz-user-agent + /// - authorization + pub async fn relaxed_validate(self, media_type: &str) -> Result<(), Box> { + self.validate_body_and_headers_except(DEFAULT_RELAXED_HEADERS, media_type) + .await + } + + /// Validate actual requests against expected requests + pub async fn validate( + self, + checked_headers: &[&str], + body_comparer: impl Fn(&[u8], &[u8]) -> Result<(), Box>, + ) -> Result<(), Box> { + self.validate_base(HeadersToCheck::Include(checked_headers), body_comparer) + .await + } + + /// Validate that the bodies match, using a given [`MediaType`] for comparison + /// + /// The specified headers are also validated. If `checked_headers` is a `None`, it means + /// checking all headers. + pub async fn validate_body_and_headers( + self, + checked_headers: Option<&[&str]>, + media_type: &str, + ) -> Result<(), Box> { + let headers_to_check = match checked_headers { + Some(headers) => HeadersToCheck::Include(headers), + None => HeadersToCheck::Exclude(None), + }; + self.validate_base(headers_to_check, |b1, b2| { + aws_smithy_protocol_test::validate_body( + b1, + std::str::from_utf8(b2).unwrap(), + MediaType::from(media_type), + ) + .map_err(|e| Box::new(e) as _) + }) + .await + } + + /// Validate that the bodies match, using a given [`MediaType`] for comparison + /// + /// The headers are also validated unless listed in `excluded_headers` + pub async fn validate_body_and_headers_except( + self, + excluded_headers: &[&str], + media_type: &str, + ) -> Result<(), Box> { + self.validate_base(HeadersToCheck::Exclude(Some(excluded_headers)), |b1, b2| { + aws_smithy_protocol_test::validate_body( + b1, + std::str::from_utf8(b2).unwrap(), + MediaType::from(media_type), + ) + .map_err(|e| Box::new(e) as _) + }) + .await + } + + async fn validate_base( + self, + checked_headers: HeadersToCheck<'_>, + body_comparer: impl Fn(&[u8], &[u8]) -> Result<(), Box>, + ) -> Result<(), Box> { + let mut actual_requests = + std::mem::take(self.recorded_requests.lock().unwrap().deref_mut()); + for conn_id in 0..self.verifiable_events.len() { + let conn_id = ConnectionId(conn_id); + let expected = self.verifiable_events.get(&conn_id).unwrap(); + let actual = actual_requests + .remove(&conn_id) + .ok_or(format!( + "expected connection {conn_id:?} but request was never sent" + ))? + .take() + .await; + body_comparer(expected.body().as_ref(), actual.body().as_ref())?; + let actual: HttpRequest = actual.map(SdkBody::from).try_into()?; + aws_smithy_protocol_test::assert_uris_match(expected.uri().to_string(), actual.uri()); + let expected_headers = expected + .headers() + .keys() + .map(|k| k.as_str()) + .filter(|k| match checked_headers { + HeadersToCheck::Include(headers) => headers.contains(k), + HeadersToCheck::Exclude(excluded) => match excluded { + Some(headers) => !headers.contains(k), + None => true, + }, + }) + .flat_map(|key| { + let _ = expected.headers().get(key)?; + Some(( + key, + expected + .headers() + .get_all(key) + .iter() + .map(|h| h.to_str().unwrap()) + .collect::>() + .join(", "), + )) + }) + .collect::>(); + aws_smithy_protocol_test::validate_headers(actual.headers(), expected_headers) + .map_err(|err| { + format!( + "event {} validation failed with: {}", + conn_id.0, + DisplayErrorContext(&err) + ) + })?; + } + Ok(()) + } + + /// Return all the recorded requests for further analysis + #[cfg(feature = "legacy-test-util")] + pub async fn take_requests(self) -> Vec> { + let mut recorded_requests = + std::mem::take(self.recorded_requests.lock().unwrap().deref_mut()); + let mut out = Vec::with_capacity(recorded_requests.len()); + for conn_id in 0..recorded_requests.len() { + out.push( + recorded_requests + .remove(&ConnectionId(conn_id)) + .expect("should exist") + .take() + .await, + ) + } + out.into_iter() + .map(|v1r| { + let mut builder = http_02x::Request::builder() + .uri(v1r.uri().to_string()) + .method(v1r.method().as_str()); + for (k, v) in v1r.headers().iter() { + builder = builder.header(k.as_str(), v.as_bytes()) + } + builder.body(v1r.into_body()).expect("valid conversion") + }) + .collect() + } + + /// Build a replay connection from a JSON file + pub fn from_file(path: impl AsRef) -> Result> { + let events: NetworkTraffic = + serde_json::from_str(&std::fs::read_to_string(path.as_ref())?)?; + Ok(Self::new(events.events)) + } + + /// Build a replay connection from a sequence of events + pub fn new(events: Vec) -> Self { + let mut event_map: HashMap<_, VecDeque<_>> = HashMap::new(); + for event in events { + let event_buffer = event_map.entry(event.connection_id).or_default(); + event_buffer.push_back(event); + } + let verifiable_events = event_map + .iter() + .map(|(id, events)| { + let mut body = BytesMut::new(); + for event in events { + if let Action::Data { + direction: Direction::Request, + data, + } = &event.action + { + body.extend_from_slice(&data.copy_to_vec()); + } + } + let initial_request = events.iter().next().expect("must have one event"); + let request = match &initial_request.action { + Action::Request { request } => { + http_1x::Request::from(request).map(|_| Bytes::from(body)) + } + _ => panic!("invalid first event"), + }; + (*id, request) + }) + .collect(); + let verifiable_events = Arc::new(verifiable_events); + + ReplayingClient { + live_events: Arc::new(Mutex::new(event_map)), + num_events: Arc::new(AtomicUsize::new(0)), + recorded_requests: Default::default(), + verifiable_events, + } + } +} + +async fn replay_body(events: VecDeque, mut sender: crate::test_util::body::Sender) { + for event in events { + match event.action { + Action::Request { .. } => panic!(), + Action::Response { .. } => panic!(), + Action::Data { + data, + direction: Direction::Response, + } => { + sender + .send_data(Bytes::from(data.into_bytes())) + .await + .expect("this is in memory traffic that should not fail to send"); + } + Action::Data { + data: _data, + direction: Direction::Request, + } => {} + Action::Eof { + direction: Direction::Request, + .. + } => {} + Action::Eof { + direction: Direction::Response, + ok: true, + .. + } => { + drop(sender); + break; + } + Action::Eof { + direction: Direction::Response, + ok: false, + .. + } => { + sender.abort(); + break; + } + } + } +} + +impl HttpConnector for ReplayingClient { + fn call(&self, mut request: HttpRequest) -> HttpConnectorFuture { + let event_id = self.next_id(); + tracing::debug!("received event {}: {request:?}", event_id.0); + let mut events = match self.live_events.lock().unwrap().remove(&event_id) { + Some(traffic) => traffic, + None => { + return HttpConnectorFuture::ready(Err(ConnectorError::other( + format!("no data for event {}. request: {:?}", event_id.0, request).into(), + None, + ))); + } + }; + + let _initial_request = events.pop_front().unwrap(); + let (sender, body) = crate::test_util::body::channel_body(); + let recording = self.recorded_requests.clone(); + let recorded_request = tokio::spawn(async move { + let mut data_read = vec![]; + while let Some(data) = crate::test_util::body::next_data_frame(request.body_mut()).await + { + data_read + .extend_from_slice(data.expect("in memory request should not fail").as_ref()) + } + request + .try_into_http1x() + .unwrap() + .map(|_body| Bytes::from(data_read)) + }); + let mut recorded_request = Waitable::Loading(recorded_request); + let fut = async move { + let resp: Result<_, ConnectorError> = loop { + let event = events + .pop_front() + .expect("no events, needed a response event"); + match event.action { + // to ensure deterministic behavior if the request EOF happens first in the log, + // wait for the request body to be done before returning a response. + Action::Eof { + direction: Direction::Request, + .. + } => { + recorded_request.wait().await; + } + Action::Request { .. } => panic!("invalid"), + Action::Response { + response: Err(error), + } => break Err(ConnectorError::other(error.0.into(), None)), + Action::Response { + response: Ok(response), + } => { + let mut builder = http_1x::Response::builder().status(response.status); + for (name, values) in response.headers { + for value in values { + builder = builder.header(&name, &value); + } + } + tokio::spawn(async move { + replay_body(events, sender).await; + // insert the finalized body into + }); + break Ok(HttpResponse::try_from( + builder.body(body).expect("valid builder"), + ) + .unwrap()); + } + + Action::Data { + direction: Direction::Request, + data: _data, + } => { + tracing::info!("get request data"); + } + Action::Eof { + direction: Direction::Response, + .. + } => panic!("got eof before response"), + + Action::Data { + data: _, + direction: Direction::Response, + } => panic!("got response data before response"), + } + }; + recording.lock().unwrap().insert(event_id, recorded_request); + resp + }; + HttpConnectorFuture::new(fut) + } +} + +impl HttpClient for ReplayingClient { + fn http_connector( + &self, + _: &HttpConnectorSettings, + _: &RuntimeComponents, + ) -> SharedHttpConnector { + self.clone().into_shared() + } + + fn connector_metadata(&self) -> Option { + Some(ConnectorMetadata::new("replaying-client", None)) + } +} diff --git a/sources/aws-smithy-http-client/src/test_util/infallible.rs b/sources/aws-smithy-http-client/src/test_util/infallible.rs new file mode 100644 index 000000000..a978962c2 --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util/infallible.rs @@ -0,0 +1,84 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_smithy_runtime_api::client::connector_metadata::ConnectorMetadata; +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpClient, + SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse}; +use aws_smithy_runtime_api::client::result::ConnectorError; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::shared::IntoShared; +use aws_smithy_types::body::SdkBody; +use std::fmt; +use std::sync::Arc; + +/// Create a [`SharedHttpClient`] from `Fn(http:Request) -> http::Response` +/// +/// # Examples +/// +/// ```rust +/// # use http_1x as http; +/// use aws_smithy_http_client::test_util::infallible_client_fn; +/// let http_client = infallible_client_fn(|_req| http::Response::builder().status(200).body("OK!").unwrap()); +/// ``` +pub fn infallible_client_fn( + f: impl Fn(http_1x::Request) -> http_1x::Response + Send + Sync + 'static, +) -> SharedHttpClient +where + B: Into, +{ + InfallibleClientFn::new(f).into_shared() +} + +#[derive(Clone)] +struct InfallibleClientFn { + #[allow(clippy::type_complexity)] + response: Arc< + dyn Fn(http_1x::Request) -> Result, ConnectorError> + + Send + + Sync, + >, +} + +impl fmt::Debug for InfallibleClientFn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InfallibleClientFn").finish() + } +} + +impl InfallibleClientFn { + fn new>( + f: impl Fn(http_1x::Request) -> http_1x::Response + Send + Sync + 'static, + ) -> Self { + Self { + response: Arc::new(move |request| Ok(f(request).map(|b| b.into()))), + } + } +} + +impl HttpConnector for InfallibleClientFn { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + HttpConnectorFuture::ready( + (self.response)(request.try_into_http1x().unwrap()) + .map(|res| HttpResponse::try_from(res).unwrap()), + ) + } +} + +impl HttpClient for InfallibleClientFn { + fn http_connector( + &self, + _: &HttpConnectorSettings, + _: &RuntimeComponents, + ) -> SharedHttpConnector { + self.clone().into_shared() + } + + fn connector_metadata(&self) -> Option { + Some(ConnectorMetadata::new("infallible-client", None)) + } +} diff --git a/sources/aws-smithy-http-client/src/test_util/legacy_infallible.rs b/sources/aws-smithy-http-client/src/test_util/legacy_infallible.rs new file mode 100644 index 000000000..a96488714 --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util/legacy_infallible.rs @@ -0,0 +1,83 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_smithy_runtime_api::client::connector_metadata::ConnectorMetadata; +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpClient, + SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse}; +use aws_smithy_runtime_api::client::result::ConnectorError; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::shared::IntoShared; +use aws_smithy_types::body::SdkBody; +use std::fmt; +use std::sync::Arc; + +/// Create a [`SharedHttpClient`] from `Fn(http:Request) -> http::Response` +/// +/// # Examples +/// +/// ```rust +/// use aws_smithy_http_client::test_util::legacy_infallible::infallible_client_fn; +/// let http_client = infallible_client_fn(|_req| http_02x::Response::builder().status(200).body("OK!").unwrap()); +/// ``` +pub fn infallible_client_fn( + f: impl Fn(http_02x::Request) -> http_02x::Response + Send + Sync + 'static, +) -> SharedHttpClient +where + B: Into, +{ + InfallibleClientFn::new(f).into_shared() +} + +#[derive(Clone)] +struct InfallibleClientFn { + #[allow(clippy::type_complexity)] + response: Arc< + dyn Fn(http_02x::Request) -> Result, ConnectorError> + + Send + + Sync, + >, +} + +impl fmt::Debug for InfallibleClientFn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InfallibleClientFn").finish() + } +} + +impl InfallibleClientFn { + fn new>( + f: impl Fn(http_02x::Request) -> http_02x::Response + Send + Sync + 'static, + ) -> Self { + Self { + response: Arc::new(move |request| Ok(f(request).map(|b| b.into()))), + } + } +} + +impl HttpConnector for InfallibleClientFn { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + HttpConnectorFuture::ready( + (self.response)(request.try_into_http02x().unwrap()) + .map(|res| HttpResponse::try_from(res).unwrap()), + ) + } +} + +impl HttpClient for InfallibleClientFn { + fn http_connector( + &self, + _: &HttpConnectorSettings, + _: &RuntimeComponents, + ) -> SharedHttpConnector { + self.clone().into_shared() + } + + fn connector_metadata(&self) -> Option { + Some(ConnectorMetadata::new("infallible-client", None)) + } +} diff --git a/sources/aws-smithy-http-client/src/test_util/never.rs b/sources/aws-smithy-http-client/src/test_util/never.rs new file mode 100644 index 000000000..d90d9ab33 --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util/never.rs @@ -0,0 +1,247 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Test connectors that never return data + +use aws_smithy_async::future::never::Never; +use aws_smithy_runtime_api::client::connector_metadata::ConnectorMetadata; +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::HttpRequest; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::shared::IntoShared; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +/// A client that will never respond. +/// +/// Returned futures will return `Pending` forever +#[derive(Clone, Debug, Default)] +pub struct NeverClient { + invocations: Arc, +} + +impl NeverClient { + /// Create a new never connector. + pub fn new() -> Self { + Default::default() + } + + /// Returns the number of invocations made to this connector. + pub fn num_calls(&self) -> usize { + self.invocations.load(Ordering::SeqCst) + } +} + +impl HttpConnector for NeverClient { + fn call(&self, _request: HttpRequest) -> HttpConnectorFuture { + self.invocations.fetch_add(1, Ordering::SeqCst); + HttpConnectorFuture::new(async move { + Never::new().await; + unreachable!() + }) + } +} + +impl HttpClient for NeverClient { + fn http_connector( + &self, + _: &HttpConnectorSettings, + _: &RuntimeComponents, + ) -> SharedHttpConnector { + self.clone().into_shared() + } + + fn connector_metadata(&self) -> Option { + Some(ConnectorMetadata::new("never-client", None)) + } +} + +/// A TCP connector that never connects. +// In the future, this can be available for multiple hyper version feature flags, with the impls gated between individual features +#[cfg(any(feature = "hyper-014", feature = "default-client"))] +#[derive(Clone, Debug, Default)] +pub struct NeverTcpConnector; + +#[cfg(any(feature = "hyper-014", feature = "default-client"))] +impl NeverTcpConnector { + /// Creates a new `NeverTcpConnector`. + pub fn new() -> Self { + Self + } +} + +#[cfg(feature = "hyper-014")] +impl hyper_0_14::service::Service for NeverTcpConnector { + type Response = hyper_014_support::NeverTcpConnection; + type Error = aws_smithy_runtime_api::box_error::BoxError; + type Future = std::pin::Pin< + Box> + Send + Sync>, + >; + + fn poll_ready( + &mut self, + _: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn call(&mut self, _: http_02x::Uri) -> Self::Future { + Box::pin(async { + Never::new().await; + unreachable!() + }) + } +} + +#[cfg(feature = "default-client")] +mod hyper1_support { + use super::NeverTcpConnector; + use aws_smithy_async::future::never::Never; + use aws_smithy_runtime_api::client::http::SharedHttpClient; + use aws_smithy_runtime_api::client::result::ConnectorError; + use http_1x::Uri; + use hyper_util::rt::TokioIo; + use std::future::Future; + use std::pin::Pin; + use std::task::{Context, Poll}; + use tokio::net::TcpStream; + + impl tower::Service for NeverTcpConnector { + type Response = TokioIo; + type Error = ConnectorError; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _uri: Uri) -> Self::Future { + Box::pin(async move { + Never::new().await; + unreachable!() + }) + } + } + + impl NeverTcpConnector { + /// Convert this connector into a usable HTTP client for testing + #[doc(hidden)] + pub fn into_client(self) -> SharedHttpClient { + crate::client::build_with_tcp_conn_fn(None, None, NeverTcpConnector::new) + } + } +} + +#[cfg(feature = "hyper-014")] +mod hyper_014_support { + use hyper_0_14::client::connect::{Connected, Connection}; + use std::io::Error; + use std::pin::Pin; + use std::task::{Context, Poll}; + use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + + /// A connection type that appeases hyper's trait bounds for a TCP connector, but will panic if any of its traits are used. + #[non_exhaustive] + #[derive(Debug, Default)] + pub struct NeverTcpConnection; + + impl Connection for NeverTcpConnection { + fn connected(&self) -> Connected { + unreachable!() + } + } + + impl AsyncRead for NeverTcpConnection { + fn poll_read( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &mut ReadBuf<'_>, + ) -> Poll> { + unreachable!() + } + } + + impl AsyncWrite for NeverTcpConnection { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &[u8], + ) -> Poll> { + unreachable!() + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + unreachable!() + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + unreachable!() + } + } +} + +#[cfg(test)] +mod test { + + use super::*; + + use aws_smithy_async::rt::sleep::TokioSleep; + use aws_smithy_async::time::SystemTimeSource; + use aws_smithy_runtime_api::client::runtime_components::RuntimeComponentsBuilder; + use tokio::time::Duration; + + #[cfg(feature = "hyper-014")] + #[tokio::test] + async fn never_tcp_connector_plugs_into_hyper_014() { + use super::NeverTcpConnector; + use crate::hyper_014::HyperClientBuilder; + + // it should compile + let client = HyperClientBuilder::new().build(NeverTcpConnector::new()); + let components = RuntimeComponentsBuilder::for_tests() + .with_sleep_impl(Some(TokioSleep::new())) + .with_time_source(Some(SystemTimeSource::new())) + .build() + .unwrap(); + let http_connector = client.http_connector( + &HttpConnectorSettings::builder() + .connect_timeout(Duration::from_millis(100)) + .build(), + &components, + ); + + let err = http_connector + .call(HttpRequest::get("http://fakeuri.com").unwrap()) + .await + .expect_err("it should time out"); + assert!(dbg!(err).is_timeout()); + } + + #[cfg(feature = "default-client")] + #[tokio::test] + async fn never_tcp_connector_plugs_into_hyper_1() { + use super::NeverTcpConnector; + let client = NeverTcpConnector::new().into_client(); + let components = RuntimeComponentsBuilder::for_tests() + .with_sleep_impl(Some(TokioSleep::new())) + .with_time_source(Some(SystemTimeSource::new())) + .build() + .unwrap(); + let http_connector = client.http_connector( + &HttpConnectorSettings::builder() + .connect_timeout(Duration::from_millis(100)) + .build(), + &components, + ); + + let err = http_connector + .call(HttpRequest::get("http://fakeuri.com").unwrap()) + .await + .expect_err("it should time out"); + assert!(dbg!(err).is_timeout()); + } +} diff --git a/sources/aws-smithy-http-client/src/test_util/replay.rs b/sources/aws-smithy-http-client/src/test_util/replay.rs new file mode 100644 index 000000000..e428953b7 --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util/replay.rs @@ -0,0 +1,302 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_smithy_protocol_test::{assert_ok, validate_body, MediaType}; +use aws_smithy_runtime_api::client::connector_metadata::ConnectorMetadata; +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse}; +use aws_smithy_runtime_api::client::result::ConnectorError; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::shared::IntoShared; +use http_1x::header::CONTENT_TYPE; +use std::ops::Deref; +use std::sync::{Arc, Mutex, MutexGuard}; + +type ReplayEvents = Vec; + +pub(crate) const DEFAULT_RELAXED_HEADERS: &[&str] = + &["x-amz-user-agent", "user-agent", "authorization"]; + +/// Test data for the [`StaticReplayClient`]. +/// +/// Each `ReplayEvent` represents one HTTP request and response +/// through the connector. +#[derive(Debug)] +pub struct ReplayEvent { + request: HttpRequest, + response: HttpResponse, +} + +impl ReplayEvent { + /// Creates a new `ReplayEvent`. + pub fn new(request: impl TryInto, response: impl TryInto) -> Self { + Self { + request: request.try_into().ok().expect("invalid request"), + response: response.try_into().ok().expect("invalid response"), + } + } + + /// Returns the test request. + pub fn request(&self) -> &HttpRequest { + &self.request + } + + /// Returns the test response. + pub fn response(&self) -> &HttpResponse { + &self.response + } +} + +impl From<(HttpRequest, HttpResponse)> for ReplayEvent { + fn from((request, response): (HttpRequest, HttpResponse)) -> Self { + Self::new(request, response) + } +} + +#[derive(Debug)] +struct ValidateRequest { + expected: HttpRequest, + actual: HttpRequest, +} + +impl ValidateRequest { + fn assert_matches(&self, index: usize, ignore_headers: &[&str]) { + let (actual, expected) = (&self.actual, &self.expected); + assert_eq!( + expected.uri(), + actual.uri(), + "request[{index}] - URI doesn't match expected value" + ); + for (name, value) in expected.headers() { + if !ignore_headers.contains(&name) { + let actual_header = actual + .headers() + .get(name) + .unwrap_or_else(|| panic!("Request #{index} - Header {name:?} is missing")); + assert_eq!( + value, actual_header, + "request[{index}] - Header {name:?} doesn't match expected value", + ); + } + } + let actual_str = std::str::from_utf8(actual.body().bytes().unwrap_or(&[])); + let expected_str = std::str::from_utf8(expected.body().bytes().unwrap_or(&[])); + let media_type = if actual + .headers() + .get(CONTENT_TYPE) + .map(|v| v.contains("json")) + .unwrap_or(false) + { + MediaType::Json + } else { + MediaType::Other("unknown".to_string()) + }; + match (actual_str, expected_str) { + (Ok(actual), Ok(expected)) => assert_ok(validate_body(actual, expected, media_type)), + _ => assert_eq!( + expected.body().bytes(), + actual.body().bytes(), + "request[{index}] - Body contents didn't match expected value" + ), + }; + } +} + +/// Request/response replaying client for use in tests. +/// +/// This mock client takes a list of request/response pairs named [`ReplayEvent`]. While the client +/// is in use, the responses will be given in the order they appear in the list regardless of what +/// the actual request was. The actual request is recorded, but otherwise not validated against what +/// is in the [`ReplayEvent`]. Later, after the client is finished being used, the +/// [`assert_requests_match`] method can be used to validate the requests. +/// +/// This utility is simpler than [DVR], and thus, is good for tests that don't need +/// to record and replay real traffic. +/// +/// # Example +/// +/// ```no_run +/// # use http_1x as http; +/// use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient}; +/// use aws_smithy_types::body::SdkBody; +/// +/// let http_client = StaticReplayClient::new(vec![ +/// // Event that covers the first request/response +/// ReplayEvent::new( +/// // If `assert_requests_match` is called later, then this request will be matched +/// // against the actual request that was made. +/// http::Request::builder().uri("http://localhost:1234/foo").body(SdkBody::empty()).unwrap(), +/// // This response will be given to the first request regardless of whether it matches the request above. +/// http::Response::builder().status(200).body(SdkBody::empty()).unwrap(), +/// ), +/// // The next ReplayEvent covers the second request/response pair... +/// ]); +/// +/// # /* +/// let config = my_generated_client::Config::builder() +/// .http_client(http_client.clone()) +/// .build(); +/// let client = my_generated_client::Client::from_conf(config); +/// # */ +/// +/// // Do stuff with client... +/// +/// // When you're done, assert the requests match what you expected +/// http_client.assert_requests_match(&[]); +/// ``` +/// +/// [`assert_requests_match`]: StaticReplayClient::assert_requests_match +/// [DVR]: crate::test_util::dvr +#[derive(Clone, Debug)] +pub struct StaticReplayClient { + data: Arc>, + requests: Arc>>, +} + +impl StaticReplayClient { + /// Creates a new event connector. + pub fn new(mut data: ReplayEvents) -> Self { + data.reverse(); + StaticReplayClient { + data: Arc::new(Mutex::new(data)), + requests: Default::default(), + } + } + + /// Returns an iterator over the actual requests that were made. + pub fn actual_requests(&self) -> impl Iterator + '_ { + // The iterator trait doesn't allow us to specify a lifetime on `self` in the `next()` method, + // so we have to do some unsafe code in order to actually implement this iterator without + // angering the borrow checker. + struct Iter<'a> { + // We store an exclusive lock to the data so that the data is completely immutable + _guard: MutexGuard<'a, Vec>, + // We store a pointer into the immutable data for accessing it later + values: *const ValidateRequest, + len: usize, + next_index: usize, + } + impl<'a> Iterator for Iter<'a> { + type Item = &'a HttpRequest; + + fn next(&mut self) -> Option { + // Safety: check the next index is in bounds + if self.next_index >= self.len { + None + } else { + // Safety: It is OK to offset into the pointer and dereference since we did a bounds check. + // It is OK to assign lifetime 'a to the reference since we hold the mutex guard for all of lifetime 'a. + let next = unsafe { + let offset = self.values.add(self.next_index); + &*offset + }; + self.next_index += 1; + Some(&next.actual) + } + } + } + + let guard = self.requests.lock().unwrap(); + Iter { + values: guard.as_ptr(), + len: guard.len(), + _guard: guard, + next_index: 0, + } + } + + fn requests(&self) -> impl Deref> + '_ { + self.requests.lock().unwrap() + } + + /// Asserts the expected requests match the actual requests. + /// + /// The expected requests are given as the connection events when the `EventConnector` + /// is created. The `EventConnector` will record the actual requests and assert that + /// they match the expected requests. + /// + /// A list of headers that should be ignored when comparing requests can be passed + /// for cases where headers are non-deterministic or are irrelevant to the test. + #[track_caller] + pub fn assert_requests_match(&self, ignore_headers: &[&str]) { + for (i, req) in self.requests().iter().enumerate() { + req.assert_matches(i, ignore_headers) + } + let remaining_requests = self.data.lock().unwrap(); + assert!( + remaining_requests.is_empty(), + "Expected {} additional requests (only {} sent)", + remaining_requests.len(), + self.requests().len() + ); + } + + /// Convenience method for `assert_requests_match` that excludes the pre-defined headers to + /// be ignored + /// + /// The pre-defined headers to be ignored: + /// - x-amz-user-agent + /// - authorization + #[track_caller] + pub fn relaxed_requests_match(&self) { + self.assert_requests_match(DEFAULT_RELAXED_HEADERS) + } +} + +impl HttpConnector for StaticReplayClient { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let res = if let Some(event) = self.data.lock().unwrap().pop() { + self.requests.lock().unwrap().push(ValidateRequest { + expected: event.request, + actual: request, + }); + + Ok(event.response) + } else { + Err(ConnectorError::other( + "StaticReplayClient: no more test data available to respond with".into(), + None, + )) + }; + + HttpConnectorFuture::new(async move { res }) + } +} + +impl HttpClient for StaticReplayClient { + fn http_connector( + &self, + _: &HttpConnectorSettings, + _: &RuntimeComponents, + ) -> SharedHttpConnector { + self.clone().into_shared() + } + + fn connector_metadata(&self) -> Option { + Some(ConnectorMetadata::new("static-replay-client", None)) + } +} + +#[cfg(test)] +mod test { + use crate::test_util::{ReplayEvent, StaticReplayClient}; + use aws_smithy_types::body::SdkBody; + + #[test] + fn create_from_either_http_type() { + let _client = StaticReplayClient::new(vec![ReplayEvent::new( + http_1x::Request::builder() + .uri("test") + .body(SdkBody::from("hello")) + .unwrap(), + http_1x::Response::builder() + .status(200) + .body(SdkBody::from("hello")) + .unwrap(), + )]); + } +} diff --git a/sources/aws-smithy-http-client/src/test_util/wire.rs b/sources/aws-smithy-http-client/src/test_util/wire.rs new file mode 100644 index 000000000..8fc3172f4 --- /dev/null +++ b/sources/aws-smithy-http-client/src/test_util/wire.rs @@ -0,0 +1,427 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Utilities for mocking at the socket level +//! +//! Other tools in this module actually operate at the `http::Request` / `http::Response` level. This +//! is useful, but it shortcuts the HTTP implementation (e.g. Hyper). [`WireMockServer`] binds +//! to an actual socket on the host. +//! +//! # Examples +//! ```no_run +//! use aws_smithy_runtime_api::client::http::HttpConnectorSettings; +//! use aws_smithy_http_client::test_util::wire::{check_matches, ReplayedEvent, WireMockServer}; +//! use aws_smithy_http_client::{match_events, ev}; +//! # async fn example() { +//! +//! // This connection binds to a local address +//! let mock = WireMockServer::start(vec![ +//! ReplayedEvent::status(503), +//! ReplayedEvent::status(200) +//! ]).await; +//! +//! # /* +//! // Create a client using the wire mock +//! let config = my_generated_client::Config::builder() +//! .http_client(mock.http_client()) +//! .build(); +//! let client = Client::from_conf(config); +//! +//! // ... do something with +//! # */ +//! +//! // assert that you got the events you expected +//! match_events!(ev!(dns), ev!(connect), ev!(http(200)))(&mock.events()); +//! # } +//! ``` + +#![allow(missing_docs)] + +use aws_smithy_async::future::never::Never; +use aws_smithy_async::future::BoxFuture; +use aws_smithy_runtime_api::client::http::SharedHttpClient; +use bytes::Bytes; +use http_body_util::Full; +use hyper::service::service_fn; +use hyper_util::client::legacy::connect::dns::Name; +use hyper_util::rt::{TokioExecutor, TokioIo}; +use hyper_util::server::graceful::{GracefulConnection, GracefulShutdown}; +use std::collections::HashSet; +use std::convert::Infallible; +use std::error::Error; +use std::future::Future; +use std::iter::Once; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +use tokio::net::TcpListener; +use tokio::sync::oneshot; + +/// An event recorded by [`WireMockServer`]. +#[non_exhaustive] +#[derive(Debug, Clone)] +pub enum RecordedEvent { + DnsLookup(String), + NewConnection, + Response(ReplayedEvent), +} + +type Matcher = ( + Box Result<(), Box>>, + &'static str, +); + +/// This method should only be used by the macro +pub fn check_matches(events: &[RecordedEvent], matchers: &[Matcher]) { + let mut events_iter = events.iter(); + let mut matcher_iter = matchers.iter(); + let mut idx = -1; + loop { + idx += 1; + let bail = |err: Box| { + panic!("failed on event {idx}:\n {err}\n actual recorded events: {events:?}") + }; + match (events_iter.next(), matcher_iter.next()) { + (Some(event), Some((matcher, _msg))) => matcher(event).unwrap_or_else(bail), + (None, None) => return, + (Some(event), None) => { + bail(format!("got {event:?} but no more events were expected").into()) + } + (None, Some((_expect, msg))) => { + bail(format!("expected {msg:?} but no more events were expected").into()) + } + } + } +} + +#[macro_export] +macro_rules! matcher { + ($expect:tt) => { + ( + Box::new(|event: &$crate::test_util::wire::RecordedEvent| { + if !matches!(event, $expect) { + return Err( + format!("expected `{}` but got {:?}", stringify!($expect), event).into(), + ); + } + Ok(()) + }), + stringify!($expect), + ) + }; +} + +/// Helper macro to generate a series of test expectations +#[macro_export] +macro_rules! match_events { + ($( $expect:pat),*) => { + |events| { + $crate::test_util::wire::check_matches(events, &[$( $crate::matcher!($expect) ),*]); + } + }; + } + +/// Helper to generate match expressions for events +#[macro_export] +macro_rules! ev { + (http($status:expr)) => { + $crate::test_util::wire::RecordedEvent::Response( + $crate::test_util::wire::ReplayedEvent::HttpResponse { + status: $status, + .. + }, + ) + }; + (dns) => { + $crate::test_util::wire::RecordedEvent::DnsLookup(_) + }; + (connect) => { + $crate::test_util::wire::RecordedEvent::NewConnection + }; + (timeout) => { + $crate::test_util::wire::RecordedEvent::Response( + $crate::test_util::wire::ReplayedEvent::Timeout, + ) + }; +} + +pub use {ev, match_events, matcher}; + +#[non_exhaustive] +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ReplayedEvent { + Timeout, + HttpResponse { status: u16, body: Bytes }, +} + +impl ReplayedEvent { + pub fn ok() -> Self { + Self::HttpResponse { + status: 200, + body: Bytes::new(), + } + } + + pub fn with_body(body: impl AsRef<[u8]>) -> Self { + Self::HttpResponse { + status: 200, + body: Bytes::copy_from_slice(body.as_ref()), + } + } + + pub fn status(status: u16) -> Self { + Self::HttpResponse { + status, + body: Bytes::new(), + } + } +} + +/// Test server that binds to 127.0.0.1:0 +/// +/// See the [module docs](crate::test_util::wire) for a usage example. +/// +/// Usage: +/// - Call [`WireMockServer::start`] to start the server +/// - Use [`WireMockServer::http_client`] or [`dns_resolver`](WireMockServer::dns_resolver) to configure your client. +/// - Make requests to [`endpoint_url`](WireMockServer::endpoint_url). +/// - Once the test is complete, retrieve a list of events from [`WireMockServer::events`] +#[derive(Debug)] +pub struct WireMockServer { + event_log: Arc>>, + bind_addr: SocketAddr, + // when the sender is dropped, that stops the server + shutdown_hook: oneshot::Sender<()>, +} + +#[derive(Debug, Clone)] +struct SharedGraceful { + graceful: Arc>>, +} + +impl SharedGraceful { + fn new() -> Self { + Self { + graceful: Arc::new(Mutex::new(Some(GracefulShutdown::new()))), + } + } + + fn watch(&self, conn: C) -> impl Future { + let graceful = self.graceful.lock().unwrap(); + graceful + .as_ref() + .expect("graceful not shutdown") + .watch(conn) + } + + async fn shutdown(&self) { + let graceful = { self.graceful.lock().unwrap().take() }; + + if let Some(graceful) = graceful { + graceful.shutdown().await; + } + } +} + +impl WireMockServer { + /// Start a wire mock server with the given events to replay. + pub async fn start(mut response_events: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (tx, mut rx) = oneshot::channel(); + let listener_addr = listener.local_addr().unwrap(); + response_events.reverse(); + let response_events = Arc::new(Mutex::new(response_events)); + let handler_events = response_events; + let wire_events = Arc::new(Mutex::new(vec![])); + let wire_log_for_service = wire_events.clone(); + let poisoned_conns: Arc>> = Default::default(); + let graceful = SharedGraceful::new(); + let conn_builder = Arc::new(hyper_util::server::conn::auto::Builder::new( + TokioExecutor::new(), + )); + + let server = async move { + let poisoned_conns = poisoned_conns.clone(); + let events = handler_events.clone(); + let wire_log = wire_log_for_service.clone(); + loop { + tokio::select! { + Ok((stream, remote_addr)) = listener.accept() => { + tracing::info!("established connection: {:?}", remote_addr); + let poisoned_conns = poisoned_conns.clone(); + let events = events.clone(); + let wire_log = wire_log.clone(); + wire_log.lock().unwrap().push(RecordedEvent::NewConnection); + let io = TokioIo::new(stream); + + let svc = service_fn(move |_req| { + let poisoned_conns = poisoned_conns.clone(); + let events = events.clone(); + let wire_log = wire_log.clone(); + if poisoned_conns.lock().unwrap().contains(&remote_addr) { + tracing::error!("poisoned connection {:?} was reused!", &remote_addr); + panic!("poisoned connection was reused!"); + } + let next_event = events.clone().lock().unwrap().pop(); + async move { + let next_event = next_event + .unwrap_or_else(|| panic!("no more events! Log: {wire_log:?}")); + + wire_log + .lock() + .unwrap() + .push(RecordedEvent::Response(next_event.clone())); + + if next_event == ReplayedEvent::Timeout { + tracing::info!("{} is poisoned", remote_addr); + poisoned_conns.lock().unwrap().insert(remote_addr); + } + tracing::debug!("replying with {:?}", next_event); + let event = generate_response_event(next_event).await; + dbg!(event) + } + }); + + let conn_builder = conn_builder.clone(); + let graceful = graceful.clone(); + tokio::spawn(async move { + let conn = conn_builder.serve_connection(io, svc); + let fut = graceful.watch(conn); + if let Err(e) = fut.await { + panic!("Error serving connection: {e:?}"); + } + }); + }, + _ = &mut rx => { + tracing::info!("wire server: shutdown signalled"); + graceful.shutdown().await; + tracing::info!("wire server: shutdown complete!"); + break; + } + } + } + }; + + tokio::spawn(server); + Self { + event_log: wire_events, + bind_addr: listener_addr, + shutdown_hook: tx, + } + } + + /// Retrieve the events recorded by this connection + pub fn events(&self) -> Vec { + self.event_log.lock().unwrap().clone() + } + + fn bind_addr(&self) -> SocketAddr { + self.bind_addr + } + + pub fn dns_resolver(&self) -> LoggingDnsResolver { + let event_log = self.event_log.clone(); + let bind_addr = self.bind_addr; + LoggingDnsResolver(InnerDnsResolver { + log: event_log, + socket_addr: bind_addr, + }) + } + + /// Prebuilt [`HttpClient`](aws_smithy_runtime_api::client::http::HttpClient) with correctly wired DNS resolver. + /// + /// **Note**: This must be used in tandem with [`Self::dns_resolver`] + pub fn http_client(&self) -> SharedHttpClient { + let resolver = self.dns_resolver(); + crate::client::build_with_tcp_conn_fn(None, None, move || { + hyper_util::client::legacy::connect::HttpConnector::new_with_resolver( + resolver.clone().0, + ) + }) + } + + /// Endpoint to use when connecting + /// + /// This works in tandem with the [`Self::dns_resolver`] to bind to the correct local IP Address + pub fn endpoint_url(&self) -> String { + format!( + "http://this-url-is-converted-to-localhost.com:{}", + self.bind_addr().port() + ) + } + + /// Shuts down the mock server. + pub fn shutdown(self) { + let _ = self.shutdown_hook.send(()); + } +} + +async fn generate_response_event( + event: ReplayedEvent, +) -> Result>, Infallible> { + let resp = match event { + ReplayedEvent::HttpResponse { status, body } => http_1x::Response::builder() + .status(status) + .body(Full::new(body)) + .unwrap(), + ReplayedEvent::Timeout => { + Never::new().await; + unreachable!() + } + }; + Ok::<_, Infallible>(resp) +} + +/// DNS resolver that keeps a log of all lookups +/// +/// Regardless of what hostname is requested, it will always return the same socket address. +#[derive(Clone, Debug)] +pub struct LoggingDnsResolver(InnerDnsResolver); + +// internal implementation so we don't have to expose hyper_util +#[derive(Clone, Debug)] +struct InnerDnsResolver { + log: Arc>>, + socket_addr: SocketAddr, +} + +impl tower::Service for InnerDnsResolver { + type Response = Once; + type Error = Infallible; + type Future = BoxFuture<'static, Self::Response, Self::Error>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, req: Name) -> Self::Future { + let socket_addr = self.socket_addr; + let log = self.log.clone(); + Box::pin(async move { + println!("looking up {req:?}, replying with {socket_addr:?}"); + log.lock() + .unwrap() + .push(RecordedEvent::DnsLookup(req.to_string())); + Ok(std::iter::once(socket_addr)) + }) + } +} + +#[cfg(all(feature = "legacy-test-util", feature = "hyper-014"))] +impl hyper_0_14::service::Service for LoggingDnsResolver { + type Response = Once; + type Error = Infallible; + type Future = BoxFuture<'static, Self::Response, Self::Error>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.0.poll_ready(cx) + } + + fn call(&mut self, req: hyper_0_14::client::connect::dns::Name) -> Self::Future { + use std::str::FromStr; + let adapter = Name::from_str(req.as_str()).expect("valid conversion"); + self.0.call(adapter) + } +} diff --git a/sources/aws-smithy-http-client/test-data/example.com.json b/sources/aws-smithy-http-client/test-data/example.com.json new file mode 100644 index 000000000..821548cc2 --- /dev/null +++ b/sources/aws-smithy-http-client/test-data/example.com.json @@ -0,0 +1,106 @@ +{ + "events": [ + { + "connection_id": 0, + "action": { + "Request": { + "request": { + "uri": "https://www.example.com/", + "headers": {}, + "method": "POST" + } + } + } + }, + { + "connection_id": 0, + "action": { + "Data": { + "data": { + "Utf8": "hello world" + }, + "direction": "Request" + } + } + }, + { + "connection_id": 0, + "action": { + "Eof": { + "ok": true, + "direction": "Request" + } + } + }, + { + "connection_id": 0, + "action": { + "Response": { + "response": { + "Ok": { + "status": 200, + "version": "HTTP/2.0", + "headers": { + "etag": [ + "\"3147526947+ident\"" + ], + "vary": [ + "Accept-Encoding" + ], + "server": [ + "ECS (bsa/EB20)" + ], + "x-cache": [ + "HIT" + ], + "age": [ + "355292" + ], + "content-length": [ + "1256" + ], + "cache-control": [ + "max-age=604800" + ], + "expires": [ + "Mon, 16 Aug 2021 18:51:30 GMT" + ], + "content-type": [ + "text/html; charset=UTF-8" + ], + "date": [ + "Mon, 09 Aug 2021 18:51:30 GMT" + ], + "last-modified": [ + "Thu, 17 Oct 2019 07:18:26 GMT" + ] + } + } + } + } + } + }, + { + "connection_id": 0, + "action": { + "Data": { + "data": { + "Utf8": "hello from example.com" + }, + "direction": "Response" + } + } + }, + { + "connection_id": 0, + "action": { + "Eof": { + "ok": true, + "direction": "Response" + } + } + } + ], + "docs": "test of example.com. response body has been manually changed", + "version": "V0" +} diff --git a/sources/aws-smithy-http-client/tests/openssl.cnf b/sources/aws-smithy-http-client/tests/openssl.cnf new file mode 100644 index 000000000..2e61c109a --- /dev/null +++ b/sources/aws-smithy-http-client/tests/openssl.cnf @@ -0,0 +1,23 @@ +[ v3_end ] +basicConstraints = critical,CA:false +keyUsage = nonRepudiation, digitalSignature +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer:always +subjectAltName = @alt_names + +[ v3_client ] +basicConstraints = critical,CA:false +keyUsage = nonRepudiation, digitalSignature +extendedKeyUsage = critical, clientAuth +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer:always + +[ v3_inter ] +subjectKeyIdentifier = hash +extendedKeyUsage = critical, serverAuth, clientAuth +basicConstraints = CA:true +keyUsage = cRLSign, keyCertSign, digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment, keyAgreement, keyCertSign, cRLSign + +[ alt_names ] +DNS.1 = sdktest.com +DNS.2 = localhost diff --git a/sources/aws-smithy-http-client/tests/proxy_tests.rs b/sources/aws-smithy-http-client/tests/proxy_tests.rs new file mode 100644 index 000000000..ad3376249 --- /dev/null +++ b/sources/aws-smithy-http-client/tests/proxy_tests.rs @@ -0,0 +1,1194 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Integration tests for proxy functionality +//! +//! These tests verify that proxy configuration works end-to-end with real HTTP requests +//! using mock proxy servers. +#![cfg(feature = "default-client")] + +use aws_smithy_async::time::SystemTimeSource; +use aws_smithy_http_client::{proxy::ProxyConfig, tls, Connector}; +use aws_smithy_runtime_api::client::http::{ + http_client_fn, HttpClient, HttpConnector, HttpConnectorSettings, SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::HttpRequest; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponentsBuilder; +use base64::Engine; +use http_1x::{Request, Response, StatusCode}; +use http_body_util::BodyExt; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper_util::rt::TokioIo; +use std::collections::HashMap; +use std::convert::Infallible; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::net::TcpListener; +use tokio::sync::oneshot; + +// ================================================================================================ +// Test Utilities (Mock Proxy Server) +// ================================================================================================ + +/// Mock HTTP server that acts as a proxy endpoint for testing +#[derive(Debug)] +struct MockProxyServer { + conn_count: Arc<()>, + addr: SocketAddr, + shutdown_tx: Option>, + request_log: Arc>>, +} + +/// A recorded request received by the mock proxy server +#[derive(Debug, Clone)] +struct RecordedRequest { + method: String, + uri: String, + headers: HashMap, +} + +impl MockProxyServer { + /// Create a new mock proxy server with a custom request handler + async fn new(handler: F) -> Self + where + F: Fn(RecordedRequest) -> Response + Send + Sync + 'static, + { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let request_log = Arc::new(Mutex::new(Vec::new())); + let request_log_clone = request_log.clone(); + let conn_count = Arc::new(()); + let server_conn_count = conn_count.clone(); + + let handler = Arc::new(handler); + + tokio::spawn(async move { + let mut shutdown_rx = shutdown_rx; + + loop { + tokio::select! { + result = listener.accept() => { + match result { + Ok((stream, _)) => { + let io = TokioIo::new(stream); + let handler = handler.clone(); + let request_log = request_log_clone.clone(); + + let stream_conn_count = server_conn_count.clone(); + tokio::spawn(async move { + let _stream_conn_count = stream_conn_count; + let service = service_fn(move |req: Request| { + let handler = handler.clone(); + let request_log = request_log.clone(); + + async move { + // Record the request + let recorded = RecordedRequest { + method: req.method().to_string(), + uri: req.uri().to_string(), + headers: req.headers().iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) + .collect(), + }; + + request_log.lock().unwrap().push(recorded.clone()); + + // Call the handler + let response = handler(recorded); + + // Convert to hyper response + let (parts, body) = response.into_parts(); + let hyper_response = Response::from_parts(parts, body); + + Ok::<_, Infallible>(hyper_response) + } + }); + + if let Err(err) = hyper::server::conn::http1::Builder::new() + .serve_connection(io, service) + .await + { + eprintln!("Mock proxy server connection error: {}", err); + } + }); + } + Err(_) => break, + } + } + _ = &mut shutdown_rx => { + break; + } + } + } + }); + + Self { + addr, + shutdown_tx: Some(shutdown_tx), + request_log, + conn_count, + } + } + + /// Return the number of active connections to this server + fn conn_count(&self) -> usize { + // 1 reference for the struct MockProxyServer, 1 reference for the + // socket task. + Arc::strong_count(&self.conn_count) + .checked_sub(2) + .expect("de-count 2 refs") + } + + /// Create a simple mock proxy that returns a fixed response + async fn with_response(status: StatusCode, body: &str) -> Self { + let body = body.to_string(); + Self::new(move |_req| { + Response::builder() + .status(status) + .body(body.clone()) + .unwrap() + }) + .await + } + + /// Create a mock proxy that validates basic authentication + async fn with_auth_validation(expected_user: &str, expected_pass: &str) -> Self { + let expected_auth = format!( + "Basic {}", + base64::prelude::BASE64_STANDARD.encode(format!("{}:{}", expected_user, expected_pass)) + ); + + Self::new(move |req| { + if let Some(auth_header) = req.headers.get("proxy-authorization") { + if auth_header == &expected_auth { + Response::builder() + .status(StatusCode::OK) + .body("authenticated".to_string()) + .unwrap() + } else { + Response::builder() + .status(StatusCode::PROXY_AUTHENTICATION_REQUIRED) + .body("invalid credentials".to_string()) + .unwrap() + } + } else { + Response::builder() + .status(StatusCode::PROXY_AUTHENTICATION_REQUIRED) + .header("proxy-authenticate", "Basic realm=\"proxy\"") + .body("authentication required".to_string()) + .unwrap() + } + }) + .await + } + + /// Get the address this server is listening on + fn addr(&self) -> SocketAddr { + self.addr + } + + /// Get all requests received by this server + fn requests(&self) -> Vec { + self.request_log.lock().unwrap().clone() + } +} + +impl Drop for MockProxyServer { + fn drop(&mut self) { + if let Some(tx) = self.shutdown_tx.take() { + let _ = tx.send(()); + } + } +} + +/// Utility for running tests with specific environment variables +#[allow(clippy::await_holding_lock)] +async fn with_env_vars(vars: &[(&str, &str)], test: F) -> R +where + F: FnOnce() -> Fut, + Fut: std::future::Future, +{ + // Use a static mutex to serialize environment variable tests + static ENV_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _guard = ENV_MUTEX.lock().unwrap(); + + // Save original environment + let original_vars: Vec<_> = vars + .iter() + .map(|(key, _)| (*key, std::env::var(key))) + .collect(); + + // Set test environment variables + for (key, value) in vars { + std::env::set_var(key, value); + } + + // Run the test + let result = test().await; + + // Restore original environment + for (key, original_value) in original_vars { + match original_value { + Ok(val) => std::env::set_var(key, val), + Err(_) => std::env::remove_var(key), + } + } + + result +} + +/// Helper function to make HTTP requests through a proxy-configured connector +async fn make_http_request_through_proxy( + proxy_config: ProxyConfig, + target_url: &str, +) -> Result<(StatusCode, String), Box> { + make_http_request_through_proxy_with_pool_timeout( + proxy_config, + Some(Duration::from_secs(90)), + target_url, + ) + .await + .map(|(status, res, _client)| (status, res)) +} + +/// Helper function to make HTTP requests through a proxy-configured connector +async fn make_http_request_through_proxy_with_pool_timeout( + proxy_config: ProxyConfig, + pool_idle_timeout: Option, + target_url: &str, +) -> Result<(StatusCode, String, SharedHttpConnector), Box> { + // Create an HttpClient using http_client_fn with proxy-configured connector + let http_client = http_client_fn(move |settings, _components| { + let connector = Connector::builder() + .proxy_config(proxy_config.clone()) + .pool_idle_timeout(pool_idle_timeout) + .connector_settings(settings.clone()) + .build_http(); + + aws_smithy_runtime_api::client::http::SharedHttpConnector::new(connector) + }); + + // Set up runtime components (following smoke_test_client pattern) + let connector_settings = HttpConnectorSettings::builder().build(); + let runtime_components = RuntimeComponentsBuilder::for_tests() + .with_time_source(Some(SystemTimeSource::new())) + .build() + .unwrap(); + + // Get the HTTP connector from the client + let http_connector = http_client.http_connector(&connector_settings, &runtime_components); + + // Create and make the HTTP request + let request = HttpRequest::get(target_url) + .map_err(|e| Box::new(e) as Box)?; + + let response = http_connector.call(request).await?; + + // Extract status and body + let status = response.status(); + let body_bytes = response.into_body().collect().await?.to_bytes(); + let body_string = String::from_utf8(body_bytes.to_vec())?; + + Ok((status.into(), body_string, http_connector)) +} + +// test the pool idle timeout. The test is in this file because it has a convenient +// infrastructure for making a server that answers smithy requests. +#[tokio::test(start_paused = false)] +// can't set start_paused due to +async fn test_http_proxy_connection_pool_timeout() { + const TIMEOUT: Duration = Duration::from_secs(10); + + // Create a mock proxy server that validates the request was routed through it + let mock_proxy = MockProxyServer::new(|req| { + // Validate that this looks like a proxy request + assert_eq!(req.method, "GET"); + // For HTTP proxy, the URI should be the full target URL + assert_eq!(req.uri, "http://aws.amazon.com/api/data"); + + // Return a successful response that we can identify + Response::builder() + .status(StatusCode::OK) + .body("proxied response from mock server".to_string()) + .unwrap() + }) + .await; + // make sure that conn_count for an empty proxy is 0 + assert_eq!(mock_proxy.conn_count(), 0); + tracing::info!("Start!"); + + // Configure connector with HTTP proxy + let proxy_config = ProxyConfig::http(format!("http://{}", mock_proxy.addr())).unwrap(); + + // Make an HTTP request through the proxy - use safe domain + let target_url = "http://aws.amazon.com/api/data"; + let start = tokio::time::Instant::now(); + let result = + make_http_request_through_proxy_with_pool_timeout(proxy_config, Some(TIMEOUT), target_url) + .await; + // hold _connector to avoid the timer being dropped + let (status, body, _connector) = result.expect("HTTP request through proxy should succeed"); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "proxied response from mock server"); + + // Verify the mock proxy received the expected request + let requests = mock_proxy.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].method, "GET"); + assert_eq!(requests[0].uri, target_url); + + // after making a request, conn count is 1 + assert_eq!(mock_proxy.conn_count(), 1); + + // sleep 1 second below the idle timeout, conn count is still 1 + tokio::time::sleep_until(start + TIMEOUT - Duration::from_secs(1)).await; + assert_eq!(mock_proxy.conn_count(), 1); + + // sleep 2 more seconds, the connection should be disconnected, conn count should be 0 + tokio::time::sleep(Duration::from_secs(3)).await; + assert_eq!(mock_proxy.conn_count(), 0); +} + +#[tokio::test] +async fn test_http_proxy_basic_request() { + // Create a mock proxy server that validates the request was routed through it + let mock_proxy = MockProxyServer::new(|req| { + // Validate that this looks like a proxy request + assert_eq!(req.method, "GET"); + // For HTTP proxy, the URI should be the full target URL + assert_eq!(req.uri, "http://aws.amazon.com/api/data"); + + // Return a successful response that we can identify + Response::builder() + .status(StatusCode::OK) + .body("proxied response from mock server".to_string()) + .unwrap() + }) + .await; + + // Configure connector with HTTP proxy + let proxy_config = ProxyConfig::http(format!("http://{}", mock_proxy.addr())).unwrap(); + + // Make an HTTP request through the proxy - use safe domain + let target_url = "http://aws.amazon.com/api/data"; + let result = make_http_request_through_proxy(proxy_config, target_url).await; + + let (status, body) = result.expect("HTTP request through proxy should succeed"); + + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "proxied response from mock server"); + + // Verify the mock proxy received the expected request + let requests = mock_proxy.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].method, "GET"); + assert_eq!(requests[0].uri, target_url); +} + +#[tokio::test] +async fn test_proxy_authentication() { + // Create a mock proxy that requires authentication + let mock_proxy = MockProxyServer::with_auth_validation("testuser", "testpass").await; + + // Configure connector with authenticated proxy + let proxy_config = ProxyConfig::http(format!("http://{}", mock_proxy.addr())) + .unwrap() + .with_basic_auth("testuser", "testpass"); + + // Make request through authenticated proxy - use safe domain + let target_url = "http://aws.amazon.com/protected/resource"; + let result = make_http_request_through_proxy(proxy_config, target_url).await; + + let (status, body) = result.expect("Authenticated proxy request should succeed"); + + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "authenticated"); + + // Verify the proxy received the request with correct auth + let requests = mock_proxy.requests(); + assert_eq!(requests.len(), 1); + + let expected_auth = format!( + "Basic {}", + base64::prelude::BASE64_STANDARD.encode("testuser:testpass") + ); + assert_eq!( + requests[0].headers.get("proxy-authorization"), + Some(&expected_auth) + ); +} + +/// Tests URL-embedded proxy authentication (http://user:pass@proxy.com format) +/// Verifies that credentials in the proxy URL are properly extracted and used +#[tokio::test] +async fn test_proxy_url_embedded_auth() { + let mock_proxy = MockProxyServer::with_auth_validation("urluser", "urlpass").await; + + // Configure proxy with credentials embedded in URL + let proxy_url = format!("http://urluser:urlpass@{}", mock_proxy.addr()); + let proxy_config = ProxyConfig::http(proxy_url).unwrap(); + + // Make request through proxy with URL-embedded auth + let target_url = "http://aws.amazon.com/api/test"; + let result = make_http_request_through_proxy(proxy_config, target_url).await; + + let (status, body) = result.expect("URL-embedded auth proxy request should succeed"); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "authenticated"); + + // Verify the proxy received the request with correct auth + let requests = mock_proxy.requests(); + assert_eq!(requests.len(), 1); + + let expected_auth = format!( + "Basic {}", + base64::prelude::BASE64_STANDARD.encode("urluser:urlpass") + ); + assert_eq!( + requests[0].headers.get("proxy-authorization"), + Some(&expected_auth) + ); +} + +/// Tests authentication precedence: URL-embedded credentials should take precedence over programmatic auth +/// Verifies that when both URL auth and with_basic_auth() are provided, URL auth wins +#[tokio::test] +async fn test_proxy_auth_precedence() { + let mock_proxy = MockProxyServer::with_auth_validation("urluser", "urlpass").await; + + // Configure proxy with URL-embedded auth AND programmatic auth + // URL auth should take precedence + let proxy_url = format!("http://urluser:urlpass@{}", mock_proxy.addr()); + let proxy_config = ProxyConfig::http(proxy_url) + .unwrap() + .with_basic_auth("programmatic", "auth"); // This should be ignored + + // Make request - should use URL-embedded auth, not programmatic auth + let target_url = "http://aws.amazon.com/precedence/test"; + let result = make_http_request_through_proxy(proxy_config, target_url).await; + + let (status, body) = result.expect("Auth precedence test should succeed"); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "authenticated"); + + // Verify the proxy received the request with URL-embedded auth (not programmatic) + let requests = mock_proxy.requests(); + assert_eq!(requests.len(), 1); + + let expected_auth = format!( + "Basic {}", + base64::prelude::BASE64_STANDARD.encode("urluser:urlpass") + ); + assert_eq!( + requests[0].headers.get("proxy-authorization"), + Some(&expected_auth) + ); +} + +#[tokio::test] +async fn test_proxy_from_environment_variables() { + let mock_proxy = MockProxyServer::with_response(StatusCode::OK, "env proxy response").await; + + with_env_vars( + &[ + ("HTTP_PROXY", &format!("http://{}", mock_proxy.addr())), + ("NO_PROXY", "localhost,127.0.0.1"), + ], + || async { + // Create connector with environment-based proxy config + let proxy_config = ProxyConfig::from_env(); + + // Make request through environment-configured proxy + let target_url = "http://aws.amazon.com/v1/data"; + let result = make_http_request_through_proxy(proxy_config, target_url).await; + + let (status, body) = result.expect("Environment proxy request should succeed"); + + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "env proxy response"); + + // Verify the proxy received the request + let requests = mock_proxy.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].uri, target_url); + }, + ) + .await; +} + +/// Tests that NO_PROXY bypass rules work correctly +/// Verifies that requests to bypassed hosts do not go through the proxy +#[tokio::test] +async fn test_no_proxy_bypass_rules() { + let mock_proxy = MockProxyServer::with_response(StatusCode::OK, "should not reach here").await; + + // Create a second mock server that will act as the "direct" target + let direct_server = MockProxyServer::with_response(StatusCode::OK, "direct connection").await; + + // Configure proxy with NO_PROXY rules that include the direct server's address + // Use just the IP address for the NO_PROXY rule + let direct_ip = "127.0.0.1"; + let proxy_config = ProxyConfig::http(format!("http://{}", mock_proxy.addr())) + .unwrap() + .no_proxy(direct_ip); + + // Make request to the direct server (should bypass proxy due to NO_PROXY rule) + let result = make_http_request_through_proxy( + proxy_config, + &format!("http://{}/test", direct_server.addr()), + ) + .await; + + let (status, body) = result.expect("Direct connection should succeed"); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "direct connection"); + + // Verify the mock proxy received no requests (bypassed) + let proxy_requests = mock_proxy.requests(); + assert_eq!( + proxy_requests.len(), + 0, + "Proxy should not have received any requests due to NO_PROXY bypass" + ); + + // Verify the direct server received the request + let direct_requests = direct_server.requests(); + assert_eq!( + direct_requests.len(), + 1, + "Direct server should have received the request" + ); +} + +/// Tests that disabled proxy configuration results in direct connections +/// Verifies that ProxyConfig::disabled() bypasses all proxy logic +#[tokio::test] +async fn test_proxy_disabled() { + // Create a direct target server + let direct_server = MockProxyServer::with_response(StatusCode::OK, "direct connection").await; + + // Create a disabled proxy configuration + let proxy_config = ProxyConfig::disabled(); + + // Make request with disabled proxy (should go direct to our mock server) + let result = make_http_request_through_proxy( + proxy_config, + &format!("http://{}/get", direct_server.addr()), + ) + .await; + + let (status, body) = result.expect("Direct connection should succeed"); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "direct connection"); + + // Verify the direct server received the request + let requests = direct_server.requests(); + assert_eq!( + requests.len(), + 1, + "Direct server should have received the request" + ); + assert_eq!(requests[0].method, "GET"); + // For direct connections, the URI might be just the path part + assert!( + requests[0].uri == format!("http://{}/get", direct_server.addr()) + || requests[0].uri == "/get", + "URI should be either full URL or path, got: {}", + requests[0].uri + ); +} + +/// Tests HTTPS-only proxy configuration +/// Verifies that HTTP requests bypass HTTPS-only proxies +#[tokio::test] +async fn test_https_proxy_configuration() { + let mock_proxy = MockProxyServer::with_response(StatusCode::OK, "https proxy response").await; + + // Create a direct target server for HTTP requests + let direct_server = + MockProxyServer::with_response(StatusCode::OK, "direct http connection").await; + + // Configure HTTPS-only proxy + let proxy_config = ProxyConfig::https(format!("http://{}", mock_proxy.addr())).unwrap(); + + // Test: HTTP request should NOT go through HTTPS-only proxy, should go direct + let target_url = format!("http://{}/api", direct_server.addr()); + let result = make_http_request_through_proxy(proxy_config.clone(), &target_url).await; + + // The HTTP request should succeed by going directly to our mock server + let (status, body) = result.expect("HTTP request should succeed via direct connection"); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "direct http connection"); + + // Verify the HTTPS-only proxy received no requests + let proxy_requests = mock_proxy.requests(); + assert_eq!( + proxy_requests.len(), + 0, + "HTTP request should not go through HTTPS-only proxy" + ); + + // Verify the direct server received the request + let direct_requests = direct_server.requests(); + assert_eq!( + direct_requests.len(), + 1, + "Direct server should have received the HTTP request" + ); +} + +/// Tests all-traffic proxy configuration +/// Verifies that both HTTP and HTTPS requests go through all-traffic proxies +#[tokio::test] +async fn test_all_traffic_proxy() { + let mock_proxy = MockProxyServer::with_response(StatusCode::OK, "all traffic proxy").await; + + // Configure proxy for all traffic + let proxy_config = ProxyConfig::all(format!("http://{}", mock_proxy.addr())).unwrap(); + + // HTTP request should go through the proxy + let target_url = "http://aws.amazon.com/api/endpoint"; + let result = make_http_request_through_proxy(proxy_config.clone(), target_url).await; + + let (status, body) = result.expect("HTTP request through all-traffic proxy should succeed"); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "all traffic proxy"); + + // Verify the proxy received the HTTP request + let requests = mock_proxy.requests(); + assert_eq!( + requests.len(), + 1, + "Proxy should have received exactly one request" + ); + assert_eq!(requests[0].method, "GET"); + assert_eq!(requests[0].uri, target_url); +} + +/// Tests proxy connection failure handling +/// Verifies that unreachable proxy servers result in appropriate connection errors +#[tokio::test] +async fn test_proxy_connection_failure() { + // Configure proxy pointing to non-existent server + let proxy_config = ProxyConfig::http("http://127.0.0.1:1").unwrap(); // Port 1 should be unavailable + + // Make request through non-existent proxy - use a safe domain that won't cause issues + let target_url = "http://aws.amazon.com/api/test"; + let result = make_http_request_through_proxy(proxy_config, target_url).await; + + // The request should fail with a connection error + assert!( + result.is_err(), + "Request should fail when proxy is unreachable" + ); + + let error = result.unwrap_err(); + let error_msg = error.to_string().to_lowercase(); + + // Verify it's a connection-related error (not a different kind of error) + assert!( + error_msg.contains("connection") + || error_msg.contains("refused") + || error_msg.contains("unreachable") + || error_msg.contains("timeout") + || error_msg.contains("connect") + || error_msg.contains("io error"), // Include generic IO errors + "Error should be connection-related, got: {}", + error + ); +} + +/// Tests proxy authentication failure handling +/// Verifies that incorrect proxy credentials result in 407 Proxy Authentication Required +#[tokio::test] +async fn test_proxy_authentication_failure() { + let mock_proxy = MockProxyServer::with_auth_validation("correct", "password").await; + + // Configure proxy with wrong credentials + let proxy_config = ProxyConfig::http(format!("http://{}", mock_proxy.addr())) + .unwrap() + .with_basic_auth("wrong", "credentials"); + + // Make request with wrong credentials - use safe domain + let target_url = "http://aws.amazon.com/secure/api"; + let result = make_http_request_through_proxy(proxy_config, target_url).await; + + // The request should return 407 Proxy Authentication Required + let (status, _body) = result.expect("Request should complete (even with auth failure)"); + assert_eq!(status, StatusCode::PROXY_AUTHENTICATION_REQUIRED); + + // Verify the proxy received the request (even though auth failed) + let requests = mock_proxy.requests(); + assert_eq!(requests.len(), 1, "Proxy should have received the request"); + + // Verify the wrong credentials were sent + let expected_wrong_auth = format!( + "Basic {}", + base64::prelude::BASE64_STANDARD.encode("wrong:credentials") + ); + assert_eq!( + requests[0].headers.get("proxy-authorization"), + Some(&expected_wrong_auth) + ); +} + +/// Tests that ProxyConfig::disabled() overrides environment proxy settings +/// Verifies that explicit proxy disabling takes precedence over environment variables +#[tokio::test] +async fn test_explicit_proxy_disable_overrides_environment() { + let mock_proxy = MockProxyServer::new(|_req| { + panic!("Request should not reach proxy when explicitly disabled"); + }) + .await; + + // Create a direct target server + let direct_server = MockProxyServer::with_response(StatusCode::OK, "direct connection").await; + + with_env_vars( + &[("HTTP_PROXY", &format!("http://{}", mock_proxy.addr()))], + || async { + // Create connector with explicitly disabled proxy (should override environment) + let proxy_config = ProxyConfig::disabled(); + + // Make request - should go direct despite HTTP_PROXY environment variable + let target_url = format!("http://{}/test", direct_server.addr()); + let result = make_http_request_through_proxy(proxy_config, &target_url).await; + + let (status, body) = result.expect("Direct connection should succeed"); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "direct connection"); + + // Verify the proxy received no requests (disabled) + let proxy_requests = mock_proxy.requests(); + assert_eq!( + proxy_requests.len(), + 0, + "Proxy should not receive requests when explicitly disabled" + ); + + // Verify the direct server received the request + let direct_requests = direct_server.requests(); + assert_eq!( + direct_requests.len(), + 1, + "Direct server should have received the request" + ); + }, + ) + .await; +} + +// ================================================================================================ +// HTTPS/CONNECT Tunneling Tests +// ================================================================================================ +// +// These tests are for HTTPS tunneling through HTTP proxies using the CONNECT method. + +/// Helper function to make HTTPS requests through proxy using TLS providers +/// This is similar to make_http_request_through_proxy but uses TLS-enabled connectors +async fn make_https_request_through_proxy( + proxy_config: ProxyConfig, + target_url: &str, + tls_provider: tls::Provider, +) -> Result<(StatusCode, String), Box> { + let http_client = http_client_fn(move |settings, _components| { + let connector = Connector::builder() + .proxy_config(proxy_config.clone()) + .connector_settings(settings.clone()) + .tls_provider(tls_provider.clone()) + .build(); + + aws_smithy_runtime_api::client::http::SharedHttpConnector::new(connector) + }); + + let connector_settings = HttpConnectorSettings::builder().build(); + let runtime_components = RuntimeComponentsBuilder::for_tests() + .with_time_source(Some(SystemTimeSource::new())) + .build() + .unwrap(); + + let http_connector = http_client.http_connector(&connector_settings, &runtime_components); + + let request = HttpRequest::get(target_url) + .map_err(|e| Box::new(e) as Box)?; + + let response = http_connector.call(request).await?; + + let status = response.status(); + let body_bytes = response.into_body().collect().await?.to_bytes(); + let body_string = String::from_utf8(body_bytes.to_vec())?; + + Ok((status.into(), body_string)) +} + +/// Generic test function for HTTPS CONNECT with authentication +/// Tests that HTTPS requests through HTTP proxy use CONNECT method with proper auth headers +async fn run_https_connect_with_auth_test(tls_provider: tls::Provider, provider_name: &str) { + let mock_proxy = MockProxyServer::new(|req| { + // For HTTPS through HTTP proxy, we should see a CONNECT request + assert_eq!(req.method, "CONNECT"); + assert_eq!(req.uri, "secure.aws.amazon.com:443"); + + // Verify authentication header is present + let expected_auth = format!( + "Basic {}", + base64::prelude::BASE64_STANDARD.encode("connectuser:connectpass") + ); + assert_eq!(req.headers.get("proxy-authorization"), Some(&expected_auth)); + + // Return 400 to avoid dealing with actual TLS tunneling + // The important part is that we got the CONNECT request with correct auth + Response::builder() + .status(StatusCode::BAD_REQUEST) + .body("CONNECT tunnel setup failed".to_string()) + .unwrap() + }) + .await; + + // Configure proxy with authentication + let proxy_config = ProxyConfig::all(format!("http://{}", mock_proxy.addr())) + .unwrap() + .with_basic_auth("connectuser", "connectpass"); + + // Make HTTPS request - should trigger CONNECT method + let target_url = "https://secure.aws.amazon.com/api/secure"; + let result = make_https_request_through_proxy(proxy_config, target_url, tls_provider).await; + + // We expect this to fail with a connection error since we returned 400 + // The important thing is that the CONNECT request was made correctly + assert!( + result.is_err(), + "CONNECT tunnel should fail with 400 response for {}", + provider_name + ); + + // Verify the proxy received the CONNECT request + let requests = mock_proxy.requests(); + assert_eq!( + requests.len(), + 1, + "Proxy should have received exactly one CONNECT request for {}", + provider_name + ); +} + +/// Generic test function for CONNECT without authentication (should get 407) +/// Tests that HTTPS requests without auth get proper 407 response +async fn run_https_connect_auth_required_test(tls_provider: tls::Provider, provider_name: &str) { + let mock_proxy = MockProxyServer::new(|req| { + // For HTTPS through HTTP proxy, we should see a CONNECT request + assert_eq!(req.method, "CONNECT"); + assert_eq!(req.uri, "secure.aws.amazon.com:443"); + + // No auth header should be present + assert!(!req.headers.contains_key("proxy-authorization")); + + // Return 407 Proxy Authentication Required + Response::builder() + .status(StatusCode::PROXY_AUTHENTICATION_REQUIRED) + .body("Proxy authentication required for CONNECT".to_string()) + .unwrap() + }) + .await; + + // Configure proxy without authentication + let proxy_config = ProxyConfig::all(format!("http://{}", mock_proxy.addr())).unwrap(); + + // Make HTTPS request - should trigger CONNECT method and get 407 + let target_url = "https://secure.aws.amazon.com/api/secure"; + let result = make_https_request_through_proxy(proxy_config, target_url, tls_provider).await; + + // We expect this to fail with a connection error since we returned 407 + assert!( + result.is_err(), + "CONNECT tunnel should fail with 407 response for {}", + provider_name + ); + + let error_msg = result.unwrap_err().to_string(); + let error_msg_lower = error_msg.to_lowercase(); + + // The important thing is that the request failed (which means CONNECT was attempted) + // The specific error message format is less critical for this test + // We accept either specific proxy auth errors OR generic connection errors + // since both indicate the CONNECT tunnel attempt was made + assert!( + error_msg_lower.contains("407") + || error_msg_lower.contains("proxy") + || error_msg_lower.contains("auth") + || error_msg_lower.contains("io error") + || error_msg_lower.contains("connection"), + "Error should be connection-related (indicating CONNECT was attempted) for {}, got: {}", + provider_name, + error_msg + ); + + // Verify the proxy received the CONNECT request + let requests = mock_proxy.requests(); + assert_eq!( + requests.len(), + 1, + "Proxy should have received exactly one CONNECT request for {}", + provider_name + ); +} + +/// Tests HTTPS tunneling through HTTP proxy with CONNECT method (rustls provider) +/// Verifies that HTTPS requests through HTTP proxy use CONNECT method with authentication +#[cfg(feature = "rustls-ring")] +#[tokio::test] +async fn test_https_connect_with_auth_rustls() { + run_https_connect_with_auth_test( + tls::Provider::rustls(tls::rustls_provider::CryptoMode::Ring), + "rustls", + ) + .await; +} + +/// Tests CONNECT method without authentication (should get 407) - rustls provider +/// Verifies that HTTPS requests without auth get proper 407 response +#[cfg(feature = "rustls-ring")] +#[tokio::test] +async fn test_https_connect_auth_required_rustls() { + run_https_connect_auth_required_test( + tls::Provider::rustls(tls::rustls_provider::CryptoMode::Ring), + "rustls", + ) + .await; +} + +/// Tests HTTPS tunneling through HTTP proxy with CONNECT method (s2n-tls provider) +/// Verifies that HTTPS requests through HTTP proxy use CONNECT method with authentication +#[cfg(feature = "s2n-tls")] +#[tokio::test] +async fn test_https_connect_with_auth_s2n_tls() { + run_https_connect_with_auth_test(tls::Provider::S2nTls, "s2n-tls").await; +} + +/// Tests CONNECT method without authentication (should get 407) - s2n-tls provider +/// Verifies that HTTPS requests without auth get proper 407 response +#[cfg(feature = "s2n-tls")] +#[tokio::test] +async fn test_https_connect_auth_required_s2n_tls() { + run_https_connect_auth_required_test(tls::Provider::S2nTls, "s2n-tls").await; +} + +/// Tests that HTTP requests through proxy use absolute URI form +/// Verifies that the full URL (including hostname) is sent to the proxy +#[tokio::test] +async fn test_http_proxy_absolute_uri_form() { + let target_host = "api.example.com"; + let target_path = "/v1/data"; + let expected_absolute_uri = format!("http://{}{}", target_host, target_path); + + // Clone for use in closure + let expected_uri_clone = expected_absolute_uri.clone(); + let target_host_clone = target_host.to_string(); + + let mock_proxy = MockProxyServer::new(move |req| { + // For HTTP through proxy, we should see the full absolute URI + assert_eq!(req.method, "GET"); + assert_eq!(req.uri, expected_uri_clone); + + // Host header should still be present + assert_eq!(req.headers.get("host"), Some(&target_host_clone)); + + Response::builder() + .status(StatusCode::OK) + .body("proxied response".to_string()) + .unwrap() + }) + .await; + + let proxy_config = ProxyConfig::http(format!("http://{}", mock_proxy.addr())).unwrap(); + + let result = make_http_request_through_proxy(proxy_config, &expected_absolute_uri).await; + + let (status, body) = result.expect("HTTP request through proxy should succeed"); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "proxied response"); + + // Verify the proxy received the request with absolute URI + let requests = mock_proxy.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].uri, expected_absolute_uri); +} + +/// Tests that direct HTTP requests (no proxy) use origin form URI +/// Verifies that only the path is sent when connecting directly +#[tokio::test] +async fn test_direct_http_origin_uri_form() { + let target_path = "/v1/data"; + + // Create a direct target server (no proxy) + let direct_server = MockProxyServer::new(move |req| { + // For direct connections, we should see only the path (origin form) + assert_eq!(req.method, "GET"); + // The URI should be just the path part, not the full URL + assert!( + req.uri == target_path || req.uri.ends_with(target_path), + "Expected origin form URI ending with '{}', got '{}'", + target_path, + req.uri + ); + + Response::builder() + .status(StatusCode::OK) + .body("direct response".to_string()) + .unwrap() + }) + .await; + + // Use disabled proxy to ensure direct connection + let proxy_config = ProxyConfig::disabled(); + + let target_url = format!("http://{}{}", direct_server.addr(), target_path); + let result = make_http_request_through_proxy(proxy_config, &target_url).await; + + let (status, body) = result.expect("Direct HTTP request should succeed"); + assert_eq!(status, StatusCode::OK); + assert_eq!(body, "direct response"); + + // Verify the server received the request + let requests = direct_server.requests(); + assert_eq!(requests.len(), 1); +} + +/// Tests URI form handling with different proxy configurations +/// Verifies that URI form changes based on proxy vs direct connection +#[tokio::test] +async fn test_uri_form_proxy_vs_direct() { + let target_host = "test.example.com"; + let target_path = "/api/test"; + let full_url = format!("http://{}{}", target_host, target_path); + + // Test 1: Through proxy - should use absolute form + { + // Clone for use in closure + let target_host_clone = target_host.to_string(); + let target_path_clone = target_path.to_string(); + + let mock_proxy = MockProxyServer::new(move |req| { + // Should receive absolute URI + assert!(req.uri.starts_with("http://")); + assert!(req.uri.contains(&target_host_clone)); + assert!(req.uri.contains(&target_path_clone)); + + Response::builder() + .status(StatusCode::OK) + .body("proxy response".to_string()) + .unwrap() + }) + .await; + + let proxy_config = ProxyConfig::http(format!("http://{}", mock_proxy.addr())).unwrap(); + let result = make_http_request_through_proxy(proxy_config, &full_url).await; + + assert!(result.is_ok(), "Proxy request should succeed"); + let requests = mock_proxy.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].uri, full_url); + } + + // Test 2: Direct connection - should use origin form + { + let target_path_clone = target_path.to_string(); + + let direct_server = MockProxyServer::new(move |req| { + // Should receive only the path part + assert!(!req.uri.starts_with("http://")); + assert!(req.uri == target_path_clone || req.uri.ends_with(&target_path_clone)); + + Response::builder() + .status(StatusCode::OK) + .body("direct response".to_string()) + .unwrap() + }) + .await; + + let proxy_config = ProxyConfig::disabled(); + let direct_url = format!("http://{}{}", direct_server.addr(), target_path); + let result = make_http_request_through_proxy(proxy_config, &direct_url).await; + + assert!(result.is_ok(), "Direct request should succeed"); + let requests = direct_server.requests(); + assert_eq!(requests.len(), 1); + } +} + +/// Generic test function for CONNECT URI form validation +/// Tests that CONNECT requests use the correct host:port format +async fn run_connect_uri_form_test(tls_provider: tls::Provider, provider_name: &str) { + let target_host = "secure.example.com"; + let target_port = 443; + let expected_connect_uri = format!("{}:{}", target_host, target_port); + + // Clone for use in closure + let expected_uri_clone = expected_connect_uri.clone(); + + let mock_proxy = MockProxyServer::new(move |req| { + if req.method == "CONNECT" { + // CONNECT should use host:port format + assert_eq!(req.uri, expected_uri_clone); + + // CONNECT requests should not have a Host header in the CONNECT line + // (the Host header is for the tunneled HTTP request, not the CONNECT) + + Response::builder() + .status(StatusCode::OK) + .body("Connection established".to_string()) + .unwrap() + } else { + // This shouldn't happen in our test, but handle it gracefully + Response::builder() + .status(StatusCode::BAD_REQUEST) + .body("Unexpected non-CONNECT request".to_string()) + .unwrap() + } + }) + .await; + + let proxy_config = ProxyConfig::all(format!("http://{}", mock_proxy.addr())).unwrap(); + + // Try to make an HTTPS request - this should trigger CONNECT + let target_url = format!("https://{}/api/secure", target_host); + + let _result = make_https_request_through_proxy(proxy_config, &target_url, tls_provider).await; + + // The request will likely fail due to our mock setup, but that's OK + // The important thing is that the CONNECT request was made with correct URI + let requests = mock_proxy.requests(); + assert_eq!( + requests.len(), + 1, + "Should have received exactly one CONNECT request for {}", + provider_name + ); + assert_eq!(requests[0].method, "CONNECT"); + assert_eq!(requests[0].uri, expected_connect_uri); +} + +/// Tests CONNECT method URI form for HTTPS tunneling - rustls provider +/// Verifies that CONNECT requests use the correct host:port format +#[cfg(feature = "rustls-ring")] +#[tokio::test] +async fn test_connect_uri_form_rustls() { + run_connect_uri_form_test( + tls::Provider::rustls(tls::rustls_provider::CryptoMode::Ring), + "rustls", + ) + .await; +} + +/// Tests CONNECT method URI form for HTTPS tunneling - s2n-tls provider +/// Verifies that CONNECT requests use the correct host:port format +#[cfg(feature = "s2n-tls")] +#[tokio::test] +async fn test_connect_uri_form_s2n_tls() { + run_connect_uri_form_test(tls::Provider::S2nTls, "s2n-tls").await; +} diff --git a/sources/aws-smithy-http-client/tests/regen-certificates.sh b/sources/aws-smithy-http-client/tests/regen-certificates.sh new file mode 100755 index 000000000..4c02f440d --- /dev/null +++ b/sources/aws-smithy-http-client/tests/regen-certificates.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +set -xe + +# The files generated by this script are used in unit tests that create TLS +# connections between a localhost server and client. Run this script if any +# certificates used for tests are expired. + +# Files generated by this script: +# server.rsa - private RSA server key +# server.pem - certificate chain containing certificates [server, intermediate, CA] + +# generate the root CA certificate and key +openssl req -nodes \ + -x509 \ + -days 3650 \ + -newkey rsa:4096 \ + -keyout ca.key \ + -out ca.cert \ + -sha256 \ + -batch \ + -subj "/CN=SDK RSA CA" + +# generate intermediate authority cert and key +openssl req -nodes \ + -newkey rsa:3072 \ + -keyout inter.key \ + -out inter.req \ + -sha256 \ + -batch \ + -subj "/CN=SDK RSA level 2 intermediate" + +# generate key and signing request for server +openssl req -nodes \ + -newkey rsa:2048 \ + -keyout server.key \ + -out server.req \ + -sha256 \ + -batch \ + -subj "/CN=sdktest.com" + +openssl rsa \ + -in server.key \ + -out server.rsa + +# sign intermediate cert with CA root +openssl x509 -req \ + -in inter.req \ + -out inter.cert \ + -CA ca.cert \ + -CAkey ca.key \ + -sha256 \ + -days 3650 \ + -set_serial 123 \ + -extensions v3_inter -extfile openssl.cnf + +# sign server cert with intermediate +openssl x509 -req \ + -in server.req \ + -out server.cert \ + -CA inter.cert \ + -CAkey inter.key \ + -sha256 \ + -days 2000 \ + -set_serial 456 \ + -extensions v3_end -extfile openssl.cnf + +cat server.cert inter.cert ca.cert > server.pem +rm *.key *.cert *.req diff --git a/sources/aws-smithy-http-client/tests/server.pem b/sources/aws-smithy-http-client/tests/server.pem new file mode 100644 index 000000000..ba429d34d --- /dev/null +++ b/sources/aws-smithy-http-client/tests/server.pem @@ -0,0 +1,80 @@ +-----BEGIN CERTIFICATE----- +MIID2TCCAkGgAwIBAgICAcgwDQYJKoZIhvcNAQELBQAwJzElMCMGA1UEAwwcU0RL +IFJTQSBsZXZlbCAyIGludGVybWVkaWF0ZTAeFw0yNTAyMTkxNDUzMDJaFw0zMDA4 +MTIxNDUzMDJaMBYxFDASBgNVBAMMC3Nka3Rlc3QuY29tMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAq0qAhLP/7uVfR5zv8d41eciCEco2kKjqoN9NlUy8 +xt8fNUFvFdSqEsrPMyycyfPg9HBhJHhCBtVQSxmr/y/IEkvbf5P5aq2CnVKKMjGm +xlqUm+mHmtDYZkJp/2nnK3ZFm9/b1oqs6XvfEJ3R2eqFf454a8SRHX1gVdj+LFnm +EVfUkVBSZCMJm9EOL9HL2I6tDPzXvJpEZMzu/pHouMNeSBBeQDGQN7vZo36cb2wG +PCTwHCh/DjZpKtJK8eKRDipIbjQr3upNbLxva36/V70xM6rTy9ac+GT/IUh0Dh0q +nXgjntez5grtlz4DrQBBhNhGAAliOTGvk74kPjzHYFy8FQIDAQABo4GfMIGcMAwG +A1UdEwEB/wQCMAAwCwYDVR0PBAQDAgbAMB0GA1UdDgQWBBTF0O0/x6KcdOoqlv0T +//HVFq2RODA9BgNVHSMENjA0gBQFk2G/rf1Iob6rw0H6lQ/uzbfLaKEZpBcwFTET +MBEGA1UEAwwKU0RLIFJTQSBDQYIBezAhBgNVHREEGjAYggtzZGt0ZXN0LmNvbYIJ +bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBgQAHDCVmgHAuUkJCyNJefN+GLj9s +hCzoSScWSSuyUTOP+N2cT9SOIyjdBb0uLaj4E/Casf/fEvqd/qSz2pBZtXXG3udP +NOfO7ZXOuAW/DqMKL7B/JYvybr2ugJefB7+fVqEdJ/lKZ6DT517nO+V9BRZRrcyR +vx5gM4j46p2qkN9yiAvimE37hBUIkjKCdejXA4IVr90Z6kG2yGo1XIFp8fDAEoSm +u5h5ftuyGvy3kXg8e1quVs38yeh3Vcxk60g8MpFrbUBZjYd3mBN0wnQAOmsK592k +3mEJIDN5NfFvuap/Tl0SAqsP7vfIy7t3Wu5aJoH5VlepmKOhHVnbwhd61pd2ZeHs +zvo8dLsfgowGiSItvdw+pg0PUsozXG0JH7o8igzEfIiRJ237pX8U1/Lr1v5fKYhL +pfZoGVfpH2oK6uUE4TzAK7bf9tGCKkvqx0MhvQ1Y6tjbp3l/+vxDTSgoo5Oh4XJ+ +gRxRIoosMVJffwwy+sbfMs5rqCTC7DrqHoFb+3I= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEtjCCAp6gAwIBAgIBezANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDDApTREsg +UlNBIENBMB4XDTI1MDIxOTE0NTMwMloXDTM1MDIxNzE0NTMwMlowJzElMCMGA1UE +AwwcU0RLIFJTQSBsZXZlbCAyIGludGVybWVkaWF0ZTCCAaIwDQYJKoZIhvcNAQEB +BQADggGPADCCAYoCggGBALpfp+sVFpcJxmK7v/aCL5R5sjWzfkwUABhYNWsk+26x +ioayPUZ3IMnKYvzGXmX2a6pjPJeP5HpFIvFnezac/tHHEk8zgATVGGbC1gO6aKT+ +o1UYKICcwHuhwG6BZ3hoFqjuBY8wk+zIgV9pd+907E0oZOy+TutCtk3dHPBYYbJO +NS8pQDw804ckKpEa34NQcLN9D3FBsgMXX6/pYyvzpD2fDgo6X6lDOWVZ31mtgmwP +jV6Be/y2r7Z/aA9fePfSTxSZnsNpwS5tkVwVG8v42ph7rVNJ/zaBsfXfCbbRV0q8 +HjJAzWGtXbPH4Ojj6MEOSjGK+rusIOZmqQAbvJ0fMH4soW5FjYWsngEuDYOrWRlB +bkv4ixeMfaz8rwC0fkLIJWFOZnHt0nmVtEVsGRc0ndTrQDlaqIJq87cL0cYkqYZE +vn2I0lmkIgna9OTCDbvBUaglYrxLe8e41OSvvcVrFmSxvx1GcWjuU1Q08P3JcBmP +9uhhjW53TYoarFhKjrO+WQIDAQABo38wfTAdBgNVHQ4EFgQUBZNhv639SKG+q8NB ++pUP7s23y2gwIAYDVR0lAQH/BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1Ud +EwQFMAMBAf8wCwYDVR0PBAQDAgH+MB8GA1UdIwQYMBaAFOCtnMCd873GJzp4HbZs +uuxho+wIMA0GCSqGSIb3DQEBCwUAA4ICAQBvCShvDdnp+pSRzK4sJS/AgElhIDVR +QrFfnhOxElVWxXSYEXVegwj1qKhgM+de9k9eKH/dM7mr5Ry9N6GfTjtRuf4fDp8W +Zj6bnqQoNz1vTPmX5ymnEipRwg5BeQtfmHVf++uor4Ch6Xx5lp9mlyCkZqEgHKkz +UietQNkZjalnIex/nBr+2kHD1dh2ZTefsTOHofnz+gsg3tNjjvzf+YMOw7aifQvI +2uoQVDjb2FBq24LSpBAc9r2sU2CKHD4/4Ei11QRI29gNgOJiiUArAV6P13mdCVMY +ue92chdLg3/AYctAUXE7NXMXeqZdOlST65GgEg36tays3l7JqlF+FaPqzuJnB+kt +pzYDdt1pFxV0Ep8RsyWm7TbYsRVqzx0eueReVKQQBBjM2+8uD8X3bZB/0QtPknBo +6x0S90g32BL4dPn/mHLLbKTNWPQ9dGmIBHvZocw5UxRYhiFLx+Wurw+V8CWmoU2g +kEiTAaA/5jiZNR02sz6H7CC70FN0vHYREiivqvNMhLbmWxORrCL/TdmwdwfEMCT4 +ozEzyGHFUVatCTUeiWxI2YXg1oNHLQU5gfy8gxfygNjG8r3A/L1SOoCsS2Cw/FcW +LJXtCyJoVzM1oLSzPbvUpe4fbnhQ/5hcM2+Tfm2DsNb9RVLjbl5t6EUTuVwKP8us +y0Qg67cAE/Grlw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFCzCCAvOgAwIBAgIUbOKEL90YNxD9QGi6wIP6zs5AzCMwDQYJKoZIhvcNAQEL +BQAwFTETMBEGA1UEAwwKU0RLIFJTQSBDQTAeFw0yNTAyMTkxNDUzMDFaFw0zNTAy +MTcxNDUzMDFaMBUxEzARBgNVBAMMClNESyBSU0EgQ0EwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDKHnrAedyOnngswWm1oeOgcfjUHo4EMHMgTn6b6vnj +KRV5idN9iIl0rI/QuyZeGcvEkarSZZMnkme1cnhOpLvelPEgCRqMI+02VodEpVer +7i2QIxqpCzf2VfZMAQ5cNyuBeN+IS+FDViPge71eypmBnF1CHu5d9lfI8y8PVVfJ +QRsP1M/bWCLIgGs7ZjHNt+cA+ZBTBHID66aXx4egW59MPBCq9SH9FC0PfvJqJjkU +QNfiSCpn4XKYRraucQ9LChNlaJqu58HPnCMX+B9/g3MuQ9qCNUq7m/p4ILre0rcJ +mz30RMGFXhyp+94DRwJBDUsWVny9fVhbMtSQKcGd51nBrrn1xcWC0+NNEWkf4VLt +sp3OPOGw/b+njgshxOdi2oPmSumRhHmMC98nl7iWT41Hy/uvLuTpd8KStN/2VIFt +7lsCRnPoIUezM34oZvsIiMsEm93AFPfQtWaKm/8GvSzGBHZ0ba8op9ZVL879yaSN +B1oHrR6OzgpfiekB8Jg3OLFSxwtgDuKJeAiorFXMiSIfqZxBfzsOQCU4UhlY38xR +SVYZMRxmHq06Mi9tVGak60RHV4opu9ixlm5sUbv0NliIzHbOHjry85z+iNSttjqu +Wv3w6jVeIHJcZeYHXZYNx8nGMWEI3JpVBNaF9ycvVpvV1hC5GM/Egn5qggpAjiL+ +uwIDAQABo1MwUTAdBgNVHQ4EFgQU4K2cwJ3zvcYnOngdtmy67GGj7AgwHwYDVR0j +BBgwFoAU4K2cwJ3zvcYnOngdtmy67GGj7AgwDwYDVR0TAQH/BAUwAwEB/zANBgkq +hkiG9w0BAQsFAAOCAgEAifwXv/ezBA8BbdV+7IcdA9R5Yz/fENQ8n5u8U0Mmpu2v +mcGpuqJOQlH6Yr5cs0zS9BRmNaMYlVWNnJxMz/3m+uaqohiFf98SgxhT+RGFRohx +jSPI6jE19vKbi1RMvv8yf8DLIjnc8d5e8kOEWNleOarVtkpAhze5nyKVyYkIyyER +zL2RKslmWzxsG0ae0HZ5T5U1voIUG5Y8WyE662egR7INLE3HMUklYljIUJNXClqe +sMfDyvl735z8S93r7sRFv3x+2YZpIzJZGk0ZbyWxfpwAVfr+ktf01BZ+lASXCmDX +94TPHswhoC4HQyxziyf2CLHvpnRczS57/bSVgatYELN4oU2sPicHqPTntrSA6Yhq ++qswrShiF47MXZeiXrddjcWTRQZOs30dmBvTj7CufAYek7r+9KeK0j+ETs/yr5Q4 +NQd7/7kkS/4GIKx0uf/+WJJhMDD8qJfWlZDV4kZ+laz49ksjES19IPKoalugst06 +SquAqOCKiOQxWeD+QQvdUu9Yac8jNEt3th1z29MoJ02QHSrKe1CmFu7jXcgvWTVI +1dhO2YJ4UNvGaFK3rWU8sDvgFnV6Hpw7y1YAgVYtterSPMAyAGdSFDvZKuo2Bq2N +juj9U9BUS1Xrl1cj8RjpZ0g0gYYF+BX2qwDCqi9qkxHel/oCzG80QAY0dnM+JT4= +-----END CERTIFICATE----- diff --git a/sources/aws-smithy-http-client/tests/server.rsa b/sources/aws-smithy-http-client/tests/server.rsa new file mode 100644 index 000000000..0ccbbf739 --- /dev/null +++ b/sources/aws-smithy-http-client/tests/server.rsa @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrSoCEs//u5V9H +nO/x3jV5yIIRyjaQqOqg302VTLzG3x81QW8V1KoSys8zLJzJ8+D0cGEkeEIG1VBL +Gav/L8gSS9t/k/lqrYKdUooyMabGWpSb6Yea0NhmQmn/aecrdkWb39vWiqzpe98Q +ndHZ6oV/jnhrxJEdfWBV2P4sWeYRV9SRUFJkIwmb0Q4v0cvYjq0M/Ne8mkRkzO7+ +kei4w15IEF5AMZA3u9mjfpxvbAY8JPAcKH8ONmkq0krx4pEOKkhuNCve6k1svG9r +fr9XvTEzqtPL1pz4ZP8hSHQOHSqdeCOe17PmCu2XPgOtAEGE2EYACWI5Ma+TviQ+ +PMdgXLwVAgMBAAECggEAIRG6myQXIpuE2BIL3vGr+E2GEwIro6UA2zsRQuQ0Q966 +qzDtTdUnWoZuFy6jPHgJc5eooX5YHPKc1ErqAlPKuAGOKGS8j+RcpZvUU1lznbLK +IwmkZPax+JgU17st5p2oVa4TxqkyNcd06sPVSlF1IYmzZPbnnz5f1WMVuyFGAg3W +CsFevfTtdpzgN8fWE6EnpSO1OY9Sa0xc+k94U6b1Coi2i6loUfKXP/iFStsGlDW3 +Dkl2egptPHi4oDcUuobJrOFr61XgOrTrYlFOrzZOgykgzAEyKN/nF3q2vAIbZSPq +MkJTw+TFGypVkMBmMTBb83CnaBWDmsicY1WJ4DjXAQKBgQDo8wgCNzNGDaqcFR2F +ZSYdqsGeSQzxjBnxnyMp/1ewb5cYFrE86ORXND7TfmJECJNEIhNLv5ytu8Z/cDsm +envwlotj8J+u0cVp3t916GEWyNBZaZM/40ivDxMCAWTeryleGbQ0ytmnDPOdPS93 +6tIImk6vKiyae3iM4eKx3o/hgQKBgQC8PZHcWrXt1OYOn/VrMeYfGp57HNeHddrA +noVIL5Ysx0pnAMhyZi69orJUYX9n1d0DwiXbagS1mTuqJK2t+0EUe5tKHalUY3Ew +CL0XGskN3526l1CKzD/vFfx28Q3lGdpxwFSNIil+85hk8BMDAu6nvzVTWd0ol/Ll +FmhIj2R8lQKBgQCMPsTzkIguBMxSa+P6C4uHEObAhPpl8hMshMKG5lPExWoR9pd+ +NDhEyA+LBOvkScVfn6Q+Vn3dms9GqiryX4t+rSP+Zz+74E8aDjQ6qEzLitrk+crN +C+kWe9zVuEEiUBJ3tMHphrbC++kvIUcSyyzoQP5eNuXjm7JYHyZ8L6IwAQKBgCir +KKsx0eJrP7TjLDOXmT8ZEipJBeuXM8avuN3qoghUmid8zNGib/C6iMqgMdVuKAza +0jEAS7osO+67c+aFxP624cr2c5Hu1hJpoOiv+cYMjNg2Pslt7VYrKttLoxPfLQms +YtgHDG4IFOGh2ImrBYKqVcwHOze89FY4LBmZGT2tAoGADKA9beP/o5vXu2BmdEPF +c4XUb5gvbI7Zv7xhF6CGOsryBkz8yF5WaAreNcG1uwPzqAyV6kk6Fgaa7XDa9Mhf +3QMe/ncGWI3NjQaB6E+7rJMQ/UDIr4Cs5ThZMkXvrR9GPhpULQi3iPepcIT79Jvn +o40e3/lYe5+59N2kVfwac0U= +-----END PRIVATE KEY----- diff --git a/sources/aws-smithy-http-client/tests/smoke_test_clients.rs b/sources/aws-smithy-http-client/tests/smoke_test_clients.rs new file mode 100644 index 000000000..1c87cf4cd --- /dev/null +++ b/sources/aws-smithy-http-client/tests/smoke_test_clients.rs @@ -0,0 +1,127 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#![cfg(any(feature = "__rustls", feature = "s2n-tls",))] + +use aws_smithy_async::time::SystemTimeSource; +use aws_smithy_http_client::{tls, Builder}; +use aws_smithy_runtime_api::client::dns::{DnsFuture, ResolveDns, ResolveDnsError}; +use aws_smithy_runtime_api::client::http::{HttpClient, HttpConnector, HttpConnectorSettings}; +use aws_smithy_runtime_api::client::orchestrator::HttpRequest; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponentsBuilder; +use hyper_util::client::legacy::connect::dns::{GaiResolver, Name}; +use std::error::Error; +use std::str::FromStr; +use std::sync::Arc; +use tower::Service; + +#[cfg(feature = "rustls-ring")] +#[tokio::test] +async fn ring_client() { + let client = Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::Ring, + )) + .build_https(); + smoke_test_client(&client).await.unwrap(); +} + +#[cfg(feature = "rustls-aws-lc-fips")] +#[tokio::test] +async fn aws_lc_fips_client() { + let client = Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::AwsLcFips, + )) + .build_https(); + smoke_test_client(&client).await.unwrap(); +} + +#[cfg(feature = "rustls-aws-lc")] +#[tokio::test] +async fn aws_lc_client() { + let client = Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::AwsLc, + )) + .build_https(); + smoke_test_client(&client).await.unwrap(); +} + +#[cfg(all(aws_sdk_unstable, feature = "rustls-ring",))] +#[tokio::test] +async fn custom_provider_client() { + let provider = rustls::crypto::ring::default_provider(); + let client = Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::Custom(provider), + )) + .build_https(); + smoke_test_client(&client).await.unwrap(); +} + +#[cfg(feature = "s2n-tls")] +#[tokio::test] +async fn s2n_tls_client() { + let client = Builder::new() + .tls_provider(tls::Provider::S2nTls) + .build_https(); + smoke_test_client(&client).await.unwrap(); +} + +#[cfg(any(feature = "rustls-ring", feature = "s2n-tls"))] +#[tokio::test] +async fn custom_dns_client() { + use std::sync::atomic::{AtomicUsize, Ordering}; + #[derive(Debug, Clone)] + struct PassThroughResolver { + inner: GaiResolver, + count: Arc, + } + impl ResolveDns for PassThroughResolver { + fn resolve_dns<'a>(&'a self, _name: &'a str) -> DnsFuture<'a> { + let mut inner = self.inner.clone(); + let name = Name::from_str(_name).unwrap(); + let count = self.count.clone(); + DnsFuture::new(async move { + count.fetch_add(1, Ordering::Relaxed); + let result = inner.call(name).await.map_err(ResolveDnsError::new)?; + Ok(result.map(|addr| addr.ip()).collect::>()) + }) + } + } + + let providers = [ + #[cfg(feature = "rustls-ring")] + tls::Provider::Rustls(tls::rustls_provider::CryptoMode::Ring), + #[cfg(feature = "s2n-tls")] + tls::Provider::S2nTls, + ]; + + for provider in providers { + let resolver = PassThroughResolver { + inner: GaiResolver::new(), + count: Default::default(), + }; + let client = Builder::new() + .tls_provider(provider) + .build_with_resolver(resolver.clone()); + smoke_test_client(&client).await.unwrap(); + assert_eq!(resolver.count.load(Ordering::Relaxed), 1); + } +} + +async fn smoke_test_client(client: &dyn HttpClient) -> Result<(), Box> { + let connector_settings = HttpConnectorSettings::builder().build(); + let runtime_components = RuntimeComponentsBuilder::for_tests() + .with_time_source(Some(SystemTimeSource::new())) + .build() + .unwrap(); + let connector = client.http_connector(&connector_settings, &runtime_components); + let _response = connector + .call(HttpRequest::get("https://amazon.com").unwrap()) + .await?; + Ok(()) +} diff --git a/sources/aws-smithy-http-client/tests/tls.rs b/sources/aws-smithy-http-client/tests/tls.rs new file mode 100644 index 000000000..7f310d32d --- /dev/null +++ b/sources/aws-smithy-http-client/tests/tls.rs @@ -0,0 +1,344 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#![cfg(any(feature = "__rustls", feature = "s2n-tls",))] + +use aws_smithy_async::time::SystemTimeSource; +use aws_smithy_http_client::tls; +use aws_smithy_http_client::tls::{TlsContext, TrustStore}; +use aws_smithy_runtime_api::box_error::BoxError; +use aws_smithy_runtime_api::client::http::{HttpClient, HttpConnector, HttpConnectorSettings}; +use aws_smithy_runtime_api::client::orchestrator::HttpRequest; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponentsBuilder; +use aws_smithy_types::byte_stream::ByteStream; +use http_1x::{Method, Request, Response, StatusCode}; +use http_body_util::{BodyExt, Full}; +use hyper::body::{Bytes, Incoming}; +use hyper::service::service_fn; +use hyper_util::rt::{TokioExecutor, TokioIo}; +use hyper_util::server::conn::auto::Builder; +use rustls::ServerConfig; +use rustls_pki_types::{CertificateDer, PrivateKeyDer}; +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; +use std::{fs, io}; +use tokio::net::TcpListener; +use tokio::task::JoinHandle; +use tokio_rustls::TlsAcceptor; +use tracing::{debug, error}; + +struct TestServer { + _handle: JoinHandle<()>, + listen_addr: SocketAddr, + conn_count: Arc<()>, +} + +impl TestServer { + /// Return the number of active connections to this server + fn conn_count(&self) -> usize { + // 1 reference for the struct MockProxyServer, 1 reference for the + // socket task. + Arc::strong_count(&self.conn_count) + .checked_sub(2) + .expect("de-count 2 refs") + } +} + +async fn server() -> Result { + // Set process wide crypto provider + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + // load public certificate. + let certs = load_certs("tests/server.pem")?; + + // load private key. + let key = load_private_key("tests/server.rsa")?; + + debug!("Starting to serve on https://{}", addr); + + // TLS config + let mut server_config = ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, key) + .map_err(|e| error(e.to_string()))?; + + server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()]; + let tls_acceptor = TlsAcceptor::from(Arc::new(server_config)); + let service = service_fn(echo); + + let conn_count = Arc::new(()); + let server_conn_count = conn_count.clone(); + + let server = async move { + loop { + let (tcp_stream, remote_addr) = listener.accept().await.unwrap(); + debug!("accepted connection from: {}", remote_addr); + + let tls_acceptor = tls_acceptor.clone(); + let connection_conn_count = server_conn_count.clone(); + tokio::spawn(async move { + let _connection_conn_count = connection_conn_count; + let tls_stream = match tls_acceptor.accept(tcp_stream).await { + Ok(tls_stream) => tls_stream, + Err(err) => { + error!("failed to perform tls handshake: {err:#}"); + return; + } + }; + if let Err(err) = Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(tls_stream), service) + .await + { + error!("failed to serve connection: {err:#}"); + } + }); + } + }; + + let server_task = tokio::spawn(server); + + Ok(TestServer { + _handle: server_task, + listen_addr: addr, + conn_count, + }) +} + +// Custom echo service, handling two different routes and a +// catch-all 404 responder. +async fn echo(req: Request) -> Result>, hyper::Error> { + let mut response = Response::new(Full::default()); + match (req.method(), req.uri().path()) { + // default route. + (&Method::GET, "/") => { + *response.body_mut() = Full::from("Hello TLS!"); + } + // echo service route. + (&Method::POST, "/echo") => { + *response.body_mut() = Full::from(req.into_body().collect().await?.to_bytes()); + } + // Catch-all 404. + _ => { + *response.status_mut() = StatusCode::NOT_FOUND; + } + }; + Ok(response) +} + +fn error(err: String) -> io::Error { + io::Error::new(io::ErrorKind::Other, err) +} + +// Load public certificate from file. +fn load_certs(filename: &str) -> io::Result>> { + let certfile = fs::File::open(filename) + .map_err(|e| error(format!("failed to open {}: {}", filename, e)))?; + let mut reader = io::BufReader::new(certfile); + rustls_pemfile::certs(&mut reader).collect() +} + +// Load private key from file. +fn load_private_key(filename: &str) -> io::Result> { + // Open keyfile. + let keyfile = fs::File::open(filename) + .map_err(|e| error(format!("failed to open {}: {}", filename, e)))?; + let mut reader = io::BufReader::new(keyfile); + + // Load and return a single private key. + rustls_pemfile::private_key(&mut reader).map(|key| key.unwrap()) +} + +fn tls_context_from_pem(filename: &str) -> TlsContext { + let pem_contents = fs::read(filename).unwrap(); + let trust_store = TrustStore::empty().with_pem_certificate(pem_contents); + TlsContext::builder() + .with_trust_store(trust_store) + .build() + .unwrap() +} + +#[cfg(feature = "rustls-aws-lc")] +#[should_panic(expected = "InvalidCertificate(UnknownIssuer)")] +#[tokio::test] +async fn test_rustls_aws_lc_native_ca() { + let client = aws_smithy_http_client::Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::AwsLc, + )) + .build_https(); + + run_tls_test(&client).await.unwrap() +} + +#[cfg(feature = "rustls-aws-lc")] +#[tokio::test] +async fn test_rustls_aws_lc_custom_ca() { + let client = aws_smithy_http_client::Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::AwsLc, + )) + .tls_context(tls_context_from_pem("tests/server.pem")) + .build_https(); + + run_tls_test(&client).await.unwrap() +} + +#[cfg(feature = "rustls-aws-lc")] +#[tokio::test(start_paused = false)] +// can't have paused clock due to +async fn test_rustls_aws_lc_custom_ca_with_timeout() { + const TIMEOUT: Duration = Duration::from_secs(10); + let client = aws_smithy_http_client::Builder::new() + .pool_idle_timeout(TIMEOUT) + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::AwsLc, + )) + .tls_context(tls_context_from_pem("tests/server.pem")) + .build_https(); + + run_tls_test_with_idle_timeout(&client, Some(TIMEOUT)) + .await + .unwrap() +} + +#[cfg(feature = "rustls-aws-lc-fips")] +#[should_panic(expected = "InvalidCertificate(UnknownIssuer)")] +#[tokio::test] +async fn test_rustls_aws_lc_fips_native_ca() { + let client = aws_smithy_http_client::Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::AwsLcFips, + )) + .build_https(); + + run_tls_test(&client).await.unwrap() +} + +#[cfg(feature = "rustls-aws-lc-fips")] +#[tokio::test] +async fn test_rustls_aws_lc_fips_custom_ca() { + let client = aws_smithy_http_client::Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::AwsLcFips, + )) + .tls_context(tls_context_from_pem("tests/server.pem")) + .build_https(); + + run_tls_test(&client).await.unwrap() +} + +#[cfg(feature = "rustls-ring")] +#[should_panic(expected = "InvalidCertificate(UnknownIssuer)")] +#[tokio::test] +async fn test_rustls_ring_native_ca() { + let client = aws_smithy_http_client::Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::Ring, + )) + .build_https(); + + run_tls_test(&client).await.unwrap() +} + +#[cfg(feature = "rustls-ring")] +#[tokio::test] +async fn test_rustls_ring_custom_ca() { + let client = aws_smithy_http_client::Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::Ring, + )) + .tls_context(tls_context_from_pem("tests/server.pem")) + .build_https(); + + run_tls_test(&client).await.unwrap() +} + +#[cfg(all(aws_sdk_unstable, feature = "rustls-ring"))] +#[should_panic(expected = "InvalidCertificate(UnknownIssuer)")] +#[tokio::test] +async fn test_rustls_custom_provider_native_ca() { + let provider = rustls::crypto::ring::default_provider(); + let client = aws_smithy_http_client::Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::Custom(provider), + )) + .build_https(); + + run_tls_test(&client).await.unwrap() +} + +#[cfg(all(aws_sdk_unstable, feature = "rustls-ring"))] +#[tokio::test] +async fn test_rustls_custom_provider_custom_ca() { + let ring_provider = rustls::crypto::ring::default_provider(); + let client = aws_smithy_http_client::Builder::new() + .tls_provider(tls::Provider::Rustls( + tls::rustls_provider::CryptoMode::Custom(ring_provider), + )) + .tls_context(tls_context_from_pem("tests/server.pem")) + .build_https(); + run_tls_test(&client).await.unwrap() +} + +#[cfg(feature = "s2n-tls")] +#[should_panic(expected = "Certificate is untrusted")] +#[tokio::test] +async fn test_s2n_native_ca() { + let client = aws_smithy_http_client::Builder::new() + .tls_provider(tls::Provider::S2nTls) + .build_https(); + + run_tls_test(&client).await.unwrap() +} + +#[cfg(feature = "s2n-tls")] +#[tokio::test] +async fn test_s2n_tls_custom_ca() { + let client = aws_smithy_http_client::Builder::new() + .tls_provider(tls::Provider::S2nTls) + .tls_context(tls_context_from_pem("tests/server.pem")) + .build_https(); + run_tls_test(&client).await.unwrap() +} + +async fn run_tls_test(client: &dyn HttpClient) -> Result<(), BoxError> { + run_tls_test_with_idle_timeout(client, None).await +} + +async fn run_tls_test_with_idle_timeout( + client: &dyn HttpClient, + pool_timeout: Option, +) -> Result<(), BoxError> { + let server = server().await?; + let start = tokio::time::Instant::now(); + assert_eq!(server.conn_count(), 0); // calibrate conn_count + let endpoint = format!("https://localhost:{}/", server.listen_addr.port()); + + let connector_settings = HttpConnectorSettings::builder().build(); + let runtime_components = RuntimeComponentsBuilder::for_tests() + .with_time_source(Some(SystemTimeSource::new())) + .build() + .unwrap(); + let connector = client.http_connector(&connector_settings, &runtime_components); + let mut response = connector.call(HttpRequest::get(endpoint).unwrap()).await?; + + let sdk_body = response.take_body(); + let body_stream = ByteStream::new(sdk_body); + let resp_bytes = body_stream.collect().await?.into_bytes(); + assert_eq!(b"Hello TLS!", &resp_bytes[..]); + + if let Some(pool_timeout) = pool_timeout { + assert_eq!(server.conn_count(), 1); + tokio::time::sleep_until(start + pool_timeout - Duration::from_secs(1)).await; + assert_eq!(server.conn_count(), 1); + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(server.conn_count(), 0); + } + Ok(()) +} diff --git a/sources/cfsignal/Cargo.toml b/sources/cfsignal/Cargo.toml index 52cb8b496..5b3b4a502 100644 --- a/sources/cfsignal/Cargo.toml +++ b/sources/cfsignal/Cargo.toml @@ -8,13 +8,13 @@ publish = false exclude = ["README.md"] [features] -fips = ["rustls/fips", "aws-lc-rs/fips", "aws-smithy-experimental/crypto-aws-lc-fips"] +fips = ["aws-lc-rs/fips", "aws-smithy-http-client/rustls-aws-lc-fips", "rustls/fips"] [dependencies] aws-config.workspace = true aws-lc-rs = { workspace = true, features = ["bindgen"] } aws-sdk-cloudformation.workspace = true -aws-smithy-experimental = { workspace = true, features = ["crypto-aws-lc"] } +aws-smithy-http-client = { workspace = true, features = ["rustls-aws-lc"] } aws-types.workspace = true imdsclient.workspace = true log.workspace = true diff --git a/sources/cfsignal/src/cloudformation.rs b/sources/cfsignal/src/cloudformation.rs index b5fc88556..484fd3a85 100644 --- a/sources/cfsignal/src/cloudformation.rs +++ b/sources/cfsignal/src/cloudformation.rs @@ -1,7 +1,7 @@ use crate::error::{self, Result}; - use aws_config::BehaviorVersion; -use aws_smithy_experimental::hyper_1_0::{CryptoMode, HyperClientBuilder}; +use aws_smithy_http_client::tls::rustls_provider::CryptoMode; +use aws_smithy_http_client::{proxy::ProxyConfig, tls, Builder as HttpClientBuilder, Connector}; use aws_types::region::Region; use imdsclient::ImdsClient; use log::info; @@ -45,13 +45,20 @@ pub async fn signal_resource( }; let http_client = if let Some(https_proxy) = https_proxy { - let no_proxy = no_proxy.as_deref(); - HyperClientBuilder::new() - .crypto_mode(crypto_mode) - .build_with_proxy(https_proxy, no_proxy) + let mut proxy = ProxyConfig::https(&https_proxy).context(error::ProxyConfigSnafu)?; + if let Some(ref no_proxy) = no_proxy { + proxy = proxy.no_proxy(no_proxy.join(",")); + } + HttpClientBuilder::new().build_with_connector_fn(move |settings, _runtime_components| { + let mut builder = Connector::builder() + .proxy_config(proxy.clone()) + .tls_provider(tls::Provider::Rustls(crypto_mode.clone())); + builder.set_connector_settings(settings.cloned()); + builder.build() + }) } else { - HyperClientBuilder::new() - .crypto_mode(crypto_mode) + HttpClientBuilder::new() + .tls_provider(tls::Provider::Rustls(crypto_mode)) .build_https() }; diff --git a/sources/cfsignal/src/error.rs b/sources/cfsignal/src/error.rs index a959a1de6..99109c262 100644 --- a/sources/cfsignal/src/error.rs +++ b/sources/cfsignal/src/error.rs @@ -42,4 +42,9 @@ pub enum Error { >, >, }, + + #[snafu(display("Invalid proxy URL: {}", source))] + ProxyConfig { + source: aws_smithy_http_client::proxy::ProxyError, + }, } diff --git a/sources/deny.toml b/sources/deny.toml index 52d51fbd0..7ffe38176 100644 --- a/sources/deny.toml +++ b/sources/deny.toml @@ -49,7 +49,14 @@ wildcards = "deny" deny = [{ name = "structopt" }, { name = "clap", wrappers = ["cargo-readme"] }] -skip = [] +skip = [ + # Older crates (gptman, handlebars, tungstenite) use thiserror 1.x while + # newer crates (asn1-rs, x509-parser, cached) use thiserror 2.x + { name = "thiserror", version = "=1" }, + { name = "thiserror-impl", version = "=1" }, + # aws-sigv4 uses sha1 0.11 while aws-config still uses sha1 0.10 + { name = "sha1", version = "=0.10" }, +] skip-tree = [ # windows-sys is not a direct dependency. mio and schannel # are using different versions of windows-sys. we skip the @@ -58,11 +65,11 @@ skip-tree = [ { name = "windows-sys" }, # httptest uses an older version of bstr { name = "httptest", version = "=0.15" }, - # aws-smithy-experimental brings a set of new dependencies that - # the aws-sdk doesn't support - { name = "aws-smithy-experimental", version = "=0.1" }, # tough 0.24 uses reqwest 0.13 while the rest of the workspace uses 0.12 { name = "tough", version = "=0.24" }, + # aws-smithy-http-client (local carry) brings newer TLS dependencies + # that coexist with older versions in the aws-sdk dependency tree + { name = "aws-smithy-http-client", version = "=1.2.0" }, # actix-http has upgraded to v0.9 of 'rand' but upstream has called out some # breaking changes which will need to be explored for our first party code # https://rust-random.github.io/book/update-0.9.html