AWS Lambda attached to the ALB producing thread 'main' panicked at 'failed to build request: http::Error(InvalidUri(InvalidFormat))'

[yury-sannikov]

Hi folks,
I have AWS Lambda attached to the ALB. ALB has a health check set up on it. The health check request is pretty simple and looks like this:

{
  "body": "",
  "headers": {
    "user-agent": "ELB-HealthChecker/2.0"
  },
  "httpMethod": "GET",

  "isBase64Encoded": false,

  "path": "/v1/health/",
  "queryStringParameters": {},
  "requestContext": {
    "elb": {
      "targetGroupArn": "arn:aws:elasticloadbalancing:us-east-1:111111111:targetgroup/cdn-ingestor-tg-usea1-dev/3fe2aca58c0da101"
    }
  }
}

Using the example code

use lambda_http::{run, service_fn, Error, IntoResponse, Request, Response};

async fn function_handler(_event: Request) -> Result<impl IntoResponse, Error> {
    let resp = Response::builder()
        .status(200)
        .header("content-type", "text/html")
        .body("Hello AWS Lambda HTTP request")
        .map_err(Box::new)?;
    Ok(resp)
}

#[tokio::main]
async fn main() -> Result<(), Error> {
    tracing_subscriber::fmt()
        .with_max_level(tracing::Level::INFO)
        .without_time()
        .init();
    run(service_fn(function_handler)).await
}

I'm getting this error:

START RequestId: 08363f5a-a246-4caf-b814-93b5243ac915 Version: $LATEST
--
  | thread 'main' panicked at 'failed to build request: http::Error(InvalidUri(InvalidFormat))', /root/.cargo/registry/src/github.com-1ecc6299db9ec823/lambda_http-0.5.2/src/request.rs:239:10
  | stack backtrace:
  | 0: 0x55d1edf5df2d - std::backtrace_rs::backtrace::libunwind::trace::h22893a5306c091b4
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
  | 1: 0x55d1edf5df2d - std::backtrace_rs::backtrace::trace_unsynchronized::h29c3bc6f9e91819d
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
  | 2: 0x55d1edf5df2d - std::sys_common::backtrace::_print_fmt::he497d8a0ec903793
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/sys_common/backtrace.rs:66:5
  | 3: 0x55d1edf5df2d - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h9c2a9d2774d81873
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/sys_common/backtrace.rs:45:22
  | 4: 0x55d1edf7f07c - core::fmt::write::hba4337c43d992f49
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/core/src/fmt/mod.rs:1194:17
  | 5: 0x55d1edf58f51 - std::io::Write::write_fmt::heb73de6e02cfabed
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/io/mod.rs:1655:15
  | 6: 0x55d1edf5fa55 - std::sys_common::backtrace::_print::h63c8b24acdd8e8ce
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/sys_common/backtrace.rs:48:5
  | 7: 0x55d1edf5fa55 - std::sys_common::backtrace::print::h426700d6240cdcc2
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/sys_common/backtrace.rs:35:9
  | 8: 0x55d1edf5fa55 - std::panicking::default_hook::{{closure}}::hc9a76eed0b18f82b
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/panicking.rs:295:22
  | 9: 0x55d1edf5f709 - std::panicking::default_hook::h2e88d02087fae196
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/panicking.rs:314:9
  | 10: 0x55d1edf600b2 - std::panicking::rust_panic_with_hook::habfdcc2e90f9fd4c
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/panicking.rs:698:17
  | 11: 0x55d1edf5ff97 - std::panicking::begin_panic_handler::{{closure}}::he054b2a83a51d2cd
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/panicking.rs:588:13
  | 12: 0x55d1edf5e3e4 - std::sys_common::backtrace::__rust_end_short_backtrace::ha48b94ab49b30915
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/sys_common/backtrace.rs:138:18
  | 13: 0x55d1edf5fcc9 - rust_begin_unwind
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/panicking.rs:584:5
  | 14: 0x55d1edf7d403 - core::panicking::panic_fmt::h366d3a309ae17c94
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/core/src/panicking.rs:143:14
  | 15: 0x55d1edf7d6b3 - core::result::unwrap_failed::hddd78f4658ac7d0f
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/core/src/result.rs:1785:5
  | 16: 0x55d1edd2aa70 - core::result::Result<T,E>::expect::h8bf945eb572da57b
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/core/src/result.rs:1035:23
  | 17: 0x55d1edcfee0c - lambda_http::request::into_alb_request::hd1427a9c8aad13c2
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/lambda_http-0.5.2/src/request.rs:233:19
  | 18: 0x55d1edd16e84 - lambda_http::request::<impl core::convert::From<lambda_http::request::LambdaRequest> for http::request::Request<aws_lambda_events::encodings::Body>>::from::hb6c31a62677bdbbd
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/lambda_http-0.5.2/src/request.rs:340:40
  | 19: 0x55d1edcfa770 - <T as core::convert::Into<U>>::into::hd711d4f05e8e6a89
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/core/src/convert/mod.rs:550:9
  | 20: 0x55d1edb65fbd - <lambda_http::Adapter<R,S> as tower_service::Service<lambda_runtime::types::LambdaEvent<lambda_http::request::LambdaRequest>>>::call::hda07f6ea62d827e2
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/lambda_http-0.5.2/src/lib.rs:152:30
  | 21: 0x55d1edbacfb3 - lambda_runtime::Runtime<C>::run::{{closure}}::{{closure}}::h52e361f378c8f567
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/lambda_runtime-0.5.1/src/lib.rs:133:72
  | 22: 0x55d1edb1f57f - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::hd10870f73a0a4fe9
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/core/src/panic/unwind_safe.rs:271:9
  | 23: 0x55d1edb6b05c - std::panicking::try::do_call::h0a38e5d6caf73237
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/panicking.rs:492:40
  | 24: 0x55d1edb6f0fb - __rust_try
  | 25: 0x55d1edb6a684 - std::panicking::try::hda9e4d52b2ed2c10
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/panicking.rs:456:19
  | 26: 0x55d1edbd3d9e - std::panic::catch_unwind::h193413935d9e5422
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/panic.rs:137:14
  | 27: 0x55d1edba9057 - lambda_runtime::Runtime<C>::run::{{closure}}::h24dc46012de9df5e
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/lambda_runtime-0.5.1/src/lib.rs:133:25
  | 28: 0x55d1edb26dea - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::hb4f4f648df04ee55
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/core/src/future/mod.rs:91:19
  | 29: 0x55d1edbaecd1 - lambda_runtime::run::{{closure}}::h34081601356b1b27
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/lambda_runtime-0.5.1/src/lib.rs:243:44
  | 30: 0x55d1edb26fba - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::hbfb303e4dbe93d34
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/core/src/future/mod.rs:91:19
  | 31: 0x55d1edb65de5 - lambda_http::run::{{closure}}::h809b36faf356d724
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/lambda_http-0.5.2/src/lib.rs:169:48
  | 32: 0x55d1edb261ea - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::h10e3a25b79dba502
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/core/src/future/mod.rs:91:19
  | 33: 0x55d1edb627d7 - ringestor::main::{{closure}}::hc5b1e86af1879dce
  | at /var/task/src/main.rs:24:38
  | 34: 0x55d1edb2719a - <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll::hd020f002937f3450
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/core/src/future/mod.rs:91:19
  | 35: 0x55d1edba195c - tokio::park::thread::CachedParkThread::block_on::{{closure}}::hc3a1b9bbab2c0531
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/park/thread.rs:263:54
  | 36: 0x55d1edb9282a - tokio::coop::with_budget::{{closure}}::h8354f747225cb609
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:102:9
  | 37: 0x55d1edb63977 - std::thread::local::LocalKey<T>::try_with::h70fc753c271d7399
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/thread/local.rs:442:16
  | 38: 0x55d1edb6314b - std::thread::local::LocalKey<T>::with::hf0eba7c3c8b1601b
  | at /rustc/fe5b13d681f25ee6474be29d748c65adcd91f69e/library/std/src/thread/local.rs:418:9
  | 39: 0x55d1edba1285 - tokio::coop::with_budget::hc7fa3562009732fa
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:95:5
  | 40: 0x55d1edba1285 - tokio::coop::budget::h1f4553430d787120
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:72:5
  | 41: 0x55d1edba1285 - tokio::park::thread::CachedParkThread::block_on::h4dc21304d5adca80
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/park/thread.rs:263:31
  | 42: 0x55d1edb8fd94 - tokio::runtime::enter::Enter::block_on::h8084a4290ab933b5
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/enter.rs:151:13
  | 43: 0x55d1edbe5846 - tokio::runtime::thread_pool::ThreadPool::block_on::hb557abb31ad1a7c2
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/mod.rs:81:9
  | 44: 0x55d1edb8fc56 - tokio::runtime::Runtime::block_on::hc9d7fef4cff55db5
  | at /root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/mod.rs:477:43
  | 45: 0x55d1edb1303e - ringestor::main::h92ebe4130aa5f530
  | at /var/task/src/main.rs:24:5
  | ERROR lambda_runtime: Any { .. }

If I'm using lambda_runtime everything is ok, though I need to massage data structures manually. Any thoughts? I'm pretty new to Rust so I might miss something.

Thanks.

[yury-sannikov]

CFN template looks like

CDNMetricsIngestor:
    Type: AWS::Serverless::Function
    Auth:
      Authorizer: "NONE"
    Properties:
      AutoPublishAlias: Live
      Timeout: 300
      Environment:
        Variables:
          DD_API_KEY: !FindInMap [DatadogApiKey, Common, !Ref Environment]
          DD_APP_KEY: !FindInMap [DatadogAppKey, Common, !Ref Environment]
          DD_LAMBDA_HANDLER: function.handler
          DD_FLUSH_TO_LOG: false
          DD_FORWARD_LOG: false
          DD_TRACE_ENABLED: true
          ENV: !Ref Environment
          REGION: !Ref Region
          RUST_BACKTRACE: full
          RUST_LOG: trace
      Tags:
        Env: !Ref Environment
      Role: !GetAtt CDNMetricsIngestorRole.Arn
      MemorySize: 128
      Architectures: ["x86_64"]
      PackageType: Image
      ImageUri: !Ref ImageUri
      FunctionName: !Sub arc-cdn-metrics-ingestor-${RegionShort}-${Environment}

the Dockerfile (pretty raw) looks like this:

FROM  public.ecr.aws/lambda/provided:al2 as builder
RUN yum update -y && \
  yum install -y make glibc-devel gcc patch curl tar wget xz git && \
  yum update -y

RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"

RUN cargo install cargo-lambda

RUN cd ${LAMBDA_TASK_ROOT} && wget -q https://ziglang.org/download/0.9.1/zig-linux-x86_64-0.9.1.tar.xz
RUN tar -xf ./zig-linux-x86_64-0.9.1.tar.xz
ENV PATH="${LAMBDA_TASK_ROOT}/zig-linux-x86_64-0.9.1:${PATH}"
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${LAMBDA_TASK_ROOT}/zig-linux-x86_64-0.9.1/lib

RUN ls ${LAMBDA_TASK_ROOT}/zig-linux-x86_64-0.9.1/lib

COPY src/ ${LAMBDA_TASK_ROOT}/src
COPY Cargo.* ${LAMBDA_TASK_ROOT}
RUN cd ${LAMBDA_TASK_ROOT} && cargo lambda build
# --target aarch64-unknown-linux-gnu

FROM public.ecr.aws/lambda/provided:al2 as app
COPY --from=public.ecr.aws/datadog/lambda-extension:latest /opt/extensions/ /opt/extensions
COPY --from=builder ${LAMBDA_TASK_ROOT}/target/lambda/ringestor/bootstrap /var/runtime
CMD [ "datadog_lambda.handler.handler" ]

[calavera]

This is because the payload doesn't have enough information to create a valid http URI. We'll need to modify the code here to support that payload:

https://github.com/awslabs/aws-lambda-rust-runtime/blob/v0.5.2-lambda-http/lambda-http/src/request.rs#L233

[yury-sannikov]

I dug into it a little bit further and here is what I found:
Since ALB health check request does not set a host header value, the URL generated at https://github.com/awslabs/aws-lambda-rust-runtime/blob/v0.5.2-lambda-http/lambda-http/src/request.rs#L81 looks like the https:///v1/health/ because of the unwrap_or_default return an empty string.

@calavera, what do you think would be the fix for that? I assume the best would be not to call uri method for the http::Request::builder() if there is no host. By default uri will contain the / per https://docs.rs/http/0.1.1/http/request/struct.Builder.html#method.uri

[calavera]

@yury-sannikov I would construct the URI without scheme and host if there is no host. The path can still be relevant, and it looks like the payload includes it, so it'd be good to have it in the URI.