Replace rusoto with custom implementation for AWS (#2176) (#2352)

* Replace rusoto (#2176)

* Add integration test for metadata endpoint

* Fix WebIdentity

* Fix doc

* Fix handling of multipart errors

* Use separate client for credentials

* Include port in Host header canonical request

* Fix doc link

* Review feedback

.github/workflows/object_store.yml

@@ -59,6 +59,13 @@ jobs:
         image: localstack/localstack:0.14.4
         ports:
           - 4566:4566
+      ec2-metadata:
+        image: amazon/amazon-ec2-metadata-mock:v1.9.2
+        ports:
+          - 1338:1338
+        env:
+          # Only allow IMDSv2
+          AEMM_IMDSV2: "1"
       azurite:
         image: mcr.microsoft.com/azure-storage/azurite
         ports:
@@ -78,6 +85,7 @@ jobs:
         AWS_ACCESS_KEY_ID: test
         AWS_SECRET_ACCESS_KEY: test
         AWS_ENDPOINT: http://localstack:4566
+        EC2_METADATA_ENDPOINT: http://ec2-metadata:1338
         AZURE_USE_EMULATOR: "1"
         AZURITE_BLOB_STORAGE_URL: "http://azurite:10000"
         AZURITE_QUEUE_STORAGE_URL: "http://azurite:10001"
@@ -101,8 +109,8 @@ jobs:
           aws --endpoint-url=http://localstack:4566 s3 mb s3://test-bucket
 
       - name: Configure Azurite (Azure emulation)
-          # the magical connection string is from
-          # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
+        # the magical connection string is from
+        # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
         run: |
           curl -sL https://aka.ms/InstallAzureCLIDeb | bash
           az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;'

object_store/Cargo.toml

@@ -46,17 +46,8 @@ rustls-pemfile = { version = "1.0", default-features = false, optional = true }
 ring = { version = "0.16", default-features = false, features = ["std"], optional = true }
 base64 = { version = "0.13", default-features = false, optional = true }
 rand = { version = "0.8", default-features = false, features = ["std", "std_rng"], optional = true }
-# for rusoto
-hyper = { version = "0.14", optional = true, default-features = false }
-# for rusoto
-hyper-rustls = { version = "0.23.0", optional = true, default-features = false, features = ["webpki-tokio", "http1", "http2", "tls12"] }
 itertools = "0.10.1"
 percent-encoding = "2.1"
-# rusoto crates are for Amazon S3 integration
-rusoto_core = { version = "0.48.0", optional = true, default-features = false, features = ["rustls"] }
-rusoto_credential = { version = "0.48.0", optional = true, default-features = false }
-rusoto_s3 = { version = "0.48.0", optional = true, default-features = false, features = ["rustls"] }
-rusoto_sts = { version = "0.48.0", optional = true, default-features = false, features = ["rustls"] }
 snafu = "0.7"
 tokio = { version = "1.18", features = ["sync", "macros", "parking_lot", "rt-multi-thread", "time", "io-util"] }
 tracing = { version = "0.1" }
@@ -70,7 +61,7 @@ walkdir = "2"
 azure = ["azure_core", "azure_storage_blobs", "azure_storage", "reqwest", "azure_identity"]
 azure_test = ["azure", "azure_core/azurite_workaround", "azure_storage/azurite_workaround", "azure_storage_blobs/azurite_workaround"]
 gcp = ["serde", "serde_json", "quick-xml", "reqwest", "reqwest/json", "reqwest/stream", "chrono/serde", "rustls-pemfile", "base64", "rand", "ring"]
-aws = ["rusoto_core", "rusoto_credential", "rusoto_s3", "rusoto_sts", "hyper", "hyper-rustls"]
+aws = ["serde", "serde_json", "quick-xml", "reqwest", "reqwest/json", "reqwest/stream", "chrono/serde", "rustls-pemfile", "base64", "rand", "ring"]
 
 [dev-dependencies] # In alphabetical order
 dotenv = "0.15.0"