From 1abcbc41370c63114fea9db3cbb5cf61c7bc657a Mon Sep 17 00:00:00 2001 From: Michael Nelson Date: Wed, 9 Aug 2023 13:28:17 +1000 Subject: [PATCH] Add DockerHub implementation for send_tags (#6580) ### Description of the change Adds the implementation for retrieving and sending tags for dockerhub repositories. ### Benefits Enables an initial integration to be tested. ### Possible drawbacks ### Applicable issues - ref #6263 ### Additional information IRL test: ```console grpcurl -plaintext -proto ./proto/ocicatalog.proto -d '{ "repository": {"registry": "registry-1.docker.io", "namespace": "bitnamicharts", "name": "zookeeper" }}' 0.0.0.0:50001 ocicatalog.OCICatalog.ListTagsForRepository { "name": "11.4.10" } { "name": "11.4.9" } { "name": "11.4.8" } { "name": "11.4.7" } { "name": "11.4.6" } { "name": "11.4.5" } { "name": "11.4.4" } { "name": "11.4.3" } { "name": "11.4.2" } { "name": "11.4.1" } { "name": "11.3.2" } { "name": "11.3.1" } { "name": "11.2.1" } { "name": "11.1.6" } { "name": "11.1.5" } { "name": "11.1.4" } { "name": "11.1.3" } { "name": "11.1.2" } { "name": "11.1.1" } { "name": "11.0.3" } { "name": "11.0.2" } { "name": "11.0.1" } { "name": "11.0.0" } { "name": "10.2.5" } { "name": "10.2.4" } { "name": "10.2.3" } ``` --------- Signed-off-by: Michael Nelson --- cmd/oci-catalog/src/providers/dockerhub.rs | 151 +++++++++++++++++++-- cmd/oci-catalog/src/providers/mod.rs | 13 +- 2 files changed, 148 insertions(+), 16 deletions(-) diff --git a/cmd/oci-catalog/src/providers/dockerhub.rs b/cmd/oci-catalog/src/providers/dockerhub.rs index 2a71736baf3..80bfc082ec9 100644 --- a/cmd/oci-catalog/src/providers/dockerhub.rs +++ b/cmd/oci-catalog/src/providers/dockerhub.rs @@ -10,8 +10,10 @@ use tokio::sync::mpsc; use tonic::Status; /// The default page size with which requests are sent to docker hub. +/// We fetch all results in batches of this page size. const DEFAULT_PAGE_SIZE: u8 = 100; pub const PROVIDER_NAME: &str = "DockerHubAPI"; +pub const DOCKERHUB_URI: &str = "https://hub.docker.com"; #[derive(Serialize, Deserialize)] struct DockerHubV2Repository { @@ -29,6 +31,21 @@ struct DockerHubV2RepositoriesResult { results: Vec, } +#[derive(Serialize, Deserialize)] +struct DockerHubV2Tag { + name: String, + repository_type: Option, + content_type: String, +} + +#[derive(Serialize, Deserialize)] +struct DockerHubV2TagsResult { + count: u16, + next: Option, + previous: Option, + results: Vec, +} + #[derive(Debug, Default)] pub struct DockerHubAPI {} @@ -38,16 +55,17 @@ impl OCICatalogSender for DockerHubAPI { PROVIDER_NAME } - // Update to return a result so errors are handled properly. async fn send_repositories( &self, tx: mpsc::Sender>, request: &ListRepositoriesRequest, ) { - let mut url = url_for_request(request); + let mut url = url_for_request_repositories(request); let client = reqwest::Client::builder().build().unwrap(); + // We continue making the request until there is no `next` url + // in the result. loop { log::debug!("requesting: {}", url); let response = match client.get(url.clone()).send().await { @@ -105,25 +123,81 @@ impl OCICatalogSender for DockerHubAPI { } } - async fn send_tags(&self, tx: mpsc::Sender>, _request: &ListTagsRequest) { - for count in 0..10 { - tx.send(Ok(Tag { - name: format!("tag-{}", count), - })) - .await - .unwrap(); + async fn send_tags(&self, tx: mpsc::Sender>, request: &ListTagsRequest) { + let mut url = match url_for_request_tags(request) { + Ok(u) => u, + Err(e) => { + tx.send(Err(e)).await.unwrap(); + return; + } + }; + + let client = reqwest::Client::builder().build().unwrap(); + + // We continue making the request until there is no `next` url + // in the result. + loop { + log::debug!("requesting: {}", url); + let response = match client.get(url.clone()).send().await { + Ok(r) => r, + Err(e) => { + tx.send(Err(Status::failed_precondition(e.to_string()))) + .await + .unwrap(); + return; + } + }; + + if response.status() != StatusCode::OK { + tx.send(Err(Status::failed_precondition(format!( + "unexpected status code when requesting {}: {}", + url, + response.status() + )))) + .await + .unwrap(); + return; + } + + let body = match response.text().await { + Ok(b) => b, + Err(e) => { + tx.send(Err(Status::failed_precondition(format!( + "unable to extract body from response: {}", + e.to_string() + )))) + .await + .unwrap(); + return; + } + }; + log::trace!("response body: {}", body); + + let response: DockerHubV2TagsResult = serde_json::from_str(&body).unwrap(); + + for tag in response.results { + tx.send(Ok(Tag { name: tag.name })).await.unwrap(); + } + + if response.next.is_some() { + url = reqwest::Url::parse(&response.next.unwrap()).unwrap(); + } else { + break; + } } } } -fn url_for_request(request: &ListRepositoriesRequest) -> Url { - let mut url = reqwest::Url::parse("https://hub.docker.com/v2/repositories/").unwrap(); +fn url_for_request_repositories(request: &ListRepositoriesRequest) -> Url { + let mut url = reqwest::Url::parse(DOCKERHUB_URI).unwrap(); if !request.namespace.is_empty() { url.set_path(&format!( "/v2/namespaces/{}/repositories/", request.namespace )); + } else { + url.set_path("/v2/repositories/"); } // For now we use a default page size and default ordering. url.query_pairs_mut() @@ -137,6 +211,30 @@ fn url_for_request(request: &ListRepositoriesRequest) -> Url { url } +fn url_for_request_tags(request: &ListTagsRequest) -> Result { + let mut url = reqwest::Url::parse(DOCKERHUB_URI).unwrap(); + + let repo = match request.repository.clone() { + Some(r) => r, + None => { + return Err(Status::invalid_argument(format!( + "repository not set in request" + ))) + } + }; + + url.set_path(&format!( + "/v2/repositories/{}/{}/tags", + repo.namespace, repo.name + )); + + // For now we use a default page size. + url.query_pairs_mut() + .append_pair("page_size", &format!("{}", DEFAULT_PAGE_SIZE)); + + Ok(url) +} + #[cfg(test)] mod tests { use super::*; @@ -165,6 +263,35 @@ mod tests { ..Default::default() }, "https://hub.docker.com/v2/namespaces/bitnamicharts/repositories/?page_size=100&ordering=name&content_types=helm&content_types=image")] fn test_url_for_request(#[case] request: ListRepositoriesRequest, #[case] expected_url: Url) { - assert_eq!(url_for_request(&request), expected_url); + assert_eq!(url_for_request_repositories(&request), expected_url); + } + + #[rstest] + #[case::without_repository(ListTagsRequest{ + ..Default::default() + }, Err(Status::invalid_argument("bang")))] + #[case::with_repository(ListTagsRequest{ + repository: Some(Repository{ + namespace: "bitnamicharts".to_string(), + name: "apache".to_string(), + ..Default::default() + }), + ..Default::default() + }, Ok(reqwest::Url::parse("https://hub.docker.com/v2/repositories/bitnamicharts/apache/tags?page_size=100").unwrap()))] + fn test_url_for_request_tags( + #[case] request: ListTagsRequest, + #[case] expected_result: Result, + ) { + match expected_result { + Ok(url) => { + assert_eq!(url_for_request_tags(&request).unwrap(), url); + } + Err(e) => { + assert_eq!( + url_for_request_tags(&request).err().unwrap().code(), + e.code() + ) + } + } } } diff --git a/cmd/oci-catalog/src/providers/mod.rs b/cmd/oci-catalog/src/providers/mod.rs index f59cb0f6975..4546379c4fa 100644 --- a/cmd/oci-catalog/src/providers/mod.rs +++ b/cmd/oci-catalog/src/providers/mod.rs @@ -9,17 +9,22 @@ use tonic::Status; pub mod dockerhub; -// OCICatalogSender is a trait that can be implemented by different providers. -// -// Initially we'll provide a DockerHub implementation followed by Harbor and -// others. +/// OCICatalogSender is a trait that can be implemented by different providers. +/// +/// Initially we'll provide a DockerHub implementation followed by Harbor and +/// others. #[tonic::async_trait] pub trait OCICatalogSender { + /// send_repositories requests repositories from the provider and sends + /// them down a channel for our API to return. async fn send_repositories( &self, tx: mpsc::Sender>, request: &ListRepositoriesRequest, ); + + /// send_tags requests tags for a repository of a provider and sends + /// them down a channel for our API to return. async fn send_tags(&self, tx: mpsc::Sender>, request: &ListTagsRequest); // The id simply gives a way in tests to determine which provider