Paginators #391

tschuett · 2022-01-09T15:07:08Z

What is the problem?

The following does not terminate:

use aws_config::meta::region::RegionProviderChain;
use aws_sdk_ec2::model::InstanceType;
use aws_types::region::Region;
use tokio_stream::StreamExt;

async fn get_region_provider(region: &str) -> aws_sdk_ec2::client::Client {
    let region_provider =
        RegionProviderChain::first_try(Region::new(region.to_string())).or_default_provider();
    let shared_config = aws_config::from_env().region(region_provider).load().await;

    aws_sdk_ec2::Client::new(&shared_config)
}

#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Hello, world!");

    let region = "eu-north-1";
    let availibility_zone = "eu-north-1a";
    let instance_type = InstanceType::from("g5.48xlarge");

    let client = get_region_provider(region).await;

    println!("call describe_spot_price_history");

    let paginator = client
        .describe_spot_price_history()
        .set_instance_types(Some(vec![instance_type.clone()]))
        .set_product_descriptions(Some(vec!["Linux/UNIX".to_string()]))
        .set_availability_zone(Some(availibility_zone.to_string()))
        .into_paginator()
        .items()
        .send();

    println!("called describe_spot_price_history");
    let _prices = paginator.collect::<Result<Vec<_>, _>>().await?;

    println!("called describe_spot_price_history: collect");

    Ok(())
}

pre paginators, I used:

fn get_done(token: &Option<String>) -> bool {
   token.as_ref().map_or(true, |v| v.is_empty())
}

fn get_token(token: &Option<String>) -> Option<String> {
    token.as_ref().and_then(
        |tok: &String| {
            if tok.is_empty() { None } else { token.clone() }
        },
    )
}

Version

├── aws-config v0.4.0
│   ├── aws-http v0.4.0
│   │   ├── aws-smithy-http v0.34.0
│   │   │   ├── aws-smithy-eventstream v0.34.0
│   │   │   │   ├── aws-smithy-types v0.34.0
│   │   │   ├── aws-smithy-types v0.34.0 (*)
│   │   ├── aws-smithy-types v0.34.0 (*)
│   │   ├── aws-types v0.4.0
│   │   │   ├── aws-smithy-async v0.34.0
│   │   │   ├── aws-smithy-types v0.34.0 (*)
│   ├── aws-sdk-sts v0.4.0
│   │   ├── aws-endpoint v0.4.0
│   │   │   ├── aws-smithy-http v0.34.0 (*)
│   │   │   ├── aws-types v0.4.0 (*)
│   │   ├── aws-http v0.4.0 (*)
│   │   ├── aws-sig-auth v0.4.0
│   │   │   ├── aws-sigv4 v0.4.0
│   │   │   │   ├── aws-smithy-eventstream v0.34.0 (*)
│   │   │   │   ├── aws-smithy-http v0.34.0 (*)
│   │   │   ├── aws-smithy-eventstream v0.34.0 (*)
│   │   │   ├── aws-smithy-http v0.34.0 (*)
│   │   │   ├── aws-types v0.4.0 (*)
│   │   ├── aws-smithy-async v0.34.0 (*)
│   │   ├── aws-smithy-client v0.34.0
│   │   │   ├── aws-smithy-async v0.34.0 (*)
│   │   │   ├── aws-smithy-http v0.34.0 (*)
│   │   │   ├── aws-smithy-http-tower v0.34.0
│   │   │   │   ├── aws-smithy-http v0.34.0 (*)
│   │   │   ├── aws-smithy-types v0.34.0 (*)
│   │   ├── aws-smithy-http v0.34.0 (*)
│   │   ├── aws-smithy-http-tower v0.34.0 (*)
│   │   ├── aws-smithy-query v0.34.0
│   │   │   ├── aws-smithy-types v0.34.0 (*)
│   │   ├── aws-smithy-types v0.34.0 (*)
│   │   ├── aws-smithy-xml v0.34.0
│   │   ├── aws-types v0.4.0 (*)
│   ├── aws-smithy-async v0.34.0 (*)
│   ├── aws-smithy-client v0.34.0 (*)
│   ├── aws-smithy-http v0.34.0 (*)
│   ├── aws-smithy-http-tower v0.34.0 (*)
│   ├── aws-smithy-json v0.34.0
│   │   └── aws-smithy-types v0.34.0 (*)
│   ├── aws-smithy-types v0.34.0 (*)
│   ├── aws-types v0.4.0 (*)
├── aws-sdk-s3 v0.4.0
│   ├── aws-endpoint v0.4.0 (*)
│   ├── aws-http v0.4.0 (*)
│   ├── aws-sig-auth v0.4.0 (*)
│   ├── aws-sigv4 v0.4.0 (*)
│   ├── aws-smithy-async v0.34.0 (*)
│   ├── aws-smithy-client v0.34.0 (*)
│   ├── aws-smithy-eventstream v0.34.0 (*)
│   ├── aws-smithy-http v0.34.0 (*)
│   ├── aws-smithy-http-tower v0.34.0 (*)
│   ├── aws-smithy-types v0.34.0 (*)
│   ├── aws-smithy-xml v0.34.0 (*)
│   ├── aws-types v0.4.0 (*)
├── aws-config v0.4.0 (*)
├── aws-sdk-dynamodb v0.4.0
│   ├── aws-endpoint v0.4.0 (*)
│   ├── aws-http v0.4.0 (*)
│   ├── aws-sig-auth v0.4.0 (*)
│   ├── aws-smithy-async v0.34.0 (*)
│   ├── aws-smithy-client v0.34.0 (*)
│   ├── aws-smithy-http v0.34.0 (*)
│   ├── aws-smithy-http-tower v0.34.0 (*)
│   ├── aws-smithy-json v0.34.0 (*)
│   ├── aws-smithy-types v0.34.0 (*)
│   ├── aws-types v0.4.0 (*)
├── aws-smithy-http v0.34.0 (*)
├── aws-config v0.4.0 (*)
├── aws-sdk-ec2 v0.4.0
│   ├── aws-endpoint v0.4.0 (*)
│   ├── aws-http v0.4.0 (*)
│   ├── aws-sig-auth v0.4.0 (*)
│   ├── aws-smithy-async v0.34.0 (*)
│   ├── aws-smithy-client v0.34.0 (*)
│   ├── aws-smithy-http v0.34.0 (*)
│   ├── aws-smithy-http-tower v0.34.0 (*)
│   ├── aws-smithy-query v0.34.0 (*)
│   ├── aws-smithy-types v0.34.0 (*)
│   ├── aws-smithy-xml v0.34.0 (*)
│   ├── aws-types v0.4.0 (*)
├── aws-sdk-pricing v0.4.0
│   ├── aws-endpoint v0.4.0 (*)
│   ├── aws-http v0.4.0 (*)
│   ├── aws-sig-auth v0.4.0 (*)
│   ├── aws-smithy-async v0.34.0 (*)
│   ├── aws-smithy-client v0.34.0 (*)
│   ├── aws-smithy-http v0.34.0 (*)
│   ├── aws-smithy-http-tower v0.34.0 (*)
│   ├── aws-smithy-json v0.34.0 (*)
│   ├── aws-smithy-types v0.34.0 (*)
│   ├── aws-types v0.4.0 (*)
├── aws-smithy-types-convert v0.34.0
│   ├── aws-smithy-types v0.34.0 (*)
├── aws-types v0.4.0 (*)
├── aws-config v0.4.0 (*)
├── aws-hyper v0.4.0
├── aws-sdk-s3 v0.4.0 (*)
├── aws-sdk-sts v0.4.0 (*)
├── aws-types v0.4.0 (*)
├── aws-config v0.4.0 (*)
├── aws-sdk-codecommit v0.4.0
│   ├── aws-endpoint v0.4.0 (*)
│   ├── aws-http v0.4.0 (*)
│   ├── aws-sig-auth v0.4.0 (*)
│   ├── aws-smithy-async v0.34.0 (*)
│   ├── aws-smithy-client v0.34.0 (*)
│   ├── aws-smithy-http v0.34.0 (*)
│   ├── aws-smithy-http-tower v0.34.0 (*)
│   ├── aws-smithy-json v0.34.0 (*)
│   ├── aws-smithy-types v0.34.0 (*)
│   ├── aws-types v0.4.0 (*)
├── aws-sdk-eventbridge v0.4.0
│   ├── aws-endpoint v0.4.0 (*)
│   ├── aws-http v0.4.0 (*)
│   ├── aws-sig-auth v0.4.0 (*)
│   ├── aws-smithy-async v0.34.0 (*)
│   ├── aws-smithy-client v0.34.0 (*)
│   ├── aws-smithy-http v0.34.0 (*)
│   ├── aws-smithy-http-tower v0.34.0 (*)
│   ├── aws-smithy-json v0.34.0 (*)
│   ├── aws-smithy-types v0.34.0 (*)
│   ├── aws-types v0.4.0 (*)
├── aws-sdk-lambda v0.4.0
│   ├── aws-endpoint v0.4.0 (*)
│   ├── aws-http v0.4.0 (*)
│   ├── aws-sig-auth v0.4.0 (*)
│   ├── aws-smithy-async v0.34.0 (*)
│   ├── aws-smithy-client v0.34.0 (*)
│   ├── aws-smithy-http v0.34.0 (*)
│   ├── aws-smithy-http-tower v0.34.0 (*)
│   ├── aws-smithy-json v0.34.0 (*)
│   ├── aws-smithy-types v0.34.0 (*)
│   ├── aws-types v0.4.0 (*)
├── aws-sdk-resourcegroupstagging v0.4.0
│   ├── aws-endpoint v0.4.0 (*)
│   ├── aws-http v0.4.0 (*)
│   ├── aws-sig-auth v0.4.0 (*)
│   ├── aws-smithy-async v0.34.0 (*)
│   ├── aws-smithy-client v0.34.0 (*)
│   ├── aws-smithy-http v0.34.0 (*)
│   ├── aws-smithy-http-tower v0.34.0 (*)
│   ├── aws-smithy-json v0.34.0 (*)
│   ├── aws-smithy-types v0.34.0 (*)
│   ├── aws-types v0.4.0 (*)
├── aws-sdk-s3 v0.4.0 (*)
├── aws-types v0.4.0 (*)
├── aws-config v0.4.0 (*)
├── aws-sdk-ec2 v0.4.0 (*)
├── aws-types v0.4.0 (*)

Platform

OSX

AWS Services

ec2

Description

No response

Logs

Jan 09 16:07:55.783 TRACE send_operation{operation="DescribeSpotPriceHistory" service="ec2"}:load_response: aws_smithy_http::middleware: http_response=Response { status: 200, version: HTTP/1.1, headers: {"x-amzn-requestid": "edf3e86c-4baf-47c1-9228-9a5ea09542e8", "cache-control": "no-cache, no-store", "strict-transport-security": "max-age=31536000; includeSubDomains", "content-type": "text/xml;charset=UTF-8", "content-length": "278", "date": "Sun, 09 Jan 2022 15:07:55 GMT", "server": "AmazonEC2"}, body: b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<DescribeSpotPriceHistoryResponse xmlns=\"http://ec2.amazonaws.com/doc/2016-11-15/\">\n    <requestId>edf3e86c-4baf-47c1-9228-9a5ea09542e8</requestId>\n    <spotPriceHistorySet/>\n    <nextToken></nextToken>\n</DescribeSpotPriceHistoryResponse>\n" }
Jan 09 16:07:55.783 TRACE send_operation{operation="DescribeSpotPriceHistory" service="ec2"}: aws_smithy_http_tower::dispatch: request=Request { method: POST, uri: https://ec2.eu-north-1.amazonaws.com/, version: HTTP/1.1, headers: {"content-type": "application/x-www-form-urlencoded", "content-length": "151", "user-agent": "aws-sdk-rust/0.4.0 os/macos lang/rust/1.57.0", "x-amz-user-agent": "aws-sdk-rust/0.4.0 api/ec2/0.4.0 os/macos lang/rust/1.57.0", "x-amz-date": "20220109T150755Z", "authorization": Sensitive}, body: SdkBody { inner: Once(Some(b"Action=DescribeSpotPriceHistory&Version=2016-11-15&AvailabilityZone=eu-north-1a&InstanceType.1=g5.48xlarge&NextToken=&ProductDescription.1=Linux%2FUNIX")), retryable: true } }

The next token looks odd.

rcoh · 2022-01-09T15:47:23Z

ah nice catch! Looks like we need to consider empty tokens and not just unset tokens. cc @kggilmer services can reply with the nextToken field present but empty apparently.

rcoh · 2022-01-09T16:18:48Z

I have a fix for this, we will release a patch ASAP.

rcoh · 2022-01-10T14:48:37Z

@tschuett as an aside, you can simplify that code a little bit by using the non-setter versions that accept explicit values instead of options/vecs:

    let paginator = client
        .describe_spot_price_history()
        .instance_types(instance_type)
        .product_descriptions("Linux/UNIX")
        .availability_zone("eu-north-1a")
        .into_paginator()
        .items()
        .send();

rcoh · 2022-01-10T20:14:09Z

fixed in 0.4.1

github-actions · 2022-01-10T20:14:40Z

⚠️COMMENT VISIBILITY WARNING⚠️

Comments on closed issues are hard for our team to see.
If you need more assistance, please either tag a team member or open a new issue that references this one.
If you wish to keep having a conversation with other community members under this issue feel free to do so.

tschuett added bug This issue is a bug. needs-triage This issue or PR still needs to be triaged. labels Jan 9, 2022

rcoh mentioned this issue Jan 9, 2022

Clarify Pagination behavior when tokens are empty vs. unset smithy-lang/smithy#1045

Closed

rcoh removed the needs-triage This issue or PR still needs to be triaged. label Jan 9, 2022

rcoh added this to We're Working On It in SDK for Rust Public Roadmap Jan 9, 2022

rcoh mentioned this issue Jan 10, 2022

Fix bug where paginators looped forever on empty string next token smithy-lang/smithy-rs#1054

Merged

5 tasks

rcoh closed this as completed Jan 10, 2022

SDK for Rust Public Roadmap automation moved this from We're Working On It to Just Shipped Jan 10, 2022

DavidSouther mentioned this issue Aug 30, 2023

Update Rust SDK Pagination guide to use paginators. awsdocs/aws-doc-sdk-examples#4023

Closed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Paginators #391

Paginators #391

tschuett commented Jan 9, 2022 •

edited

rcoh commented Jan 9, 2022

rcoh commented Jan 9, 2022

rcoh commented Jan 10, 2022 •

edited

rcoh commented Jan 10, 2022

github-actions bot commented Jan 10, 2022

Paginators #391

Paginators #391

Comments

tschuett commented Jan 9, 2022 • edited

What is the problem?

Version

Platform

AWS Services

Description

Logs

rcoh commented Jan 9, 2022

rcoh commented Jan 9, 2022

rcoh commented Jan 10, 2022 • edited

rcoh commented Jan 10, 2022

github-actions bot commented Jan 10, 2022

⚠️COMMENT VISIBILITY WARNING⚠️

tschuett commented Jan 9, 2022 •

edited

rcoh commented Jan 10, 2022 •

edited