Skip to content

Commit

Permalink
Add DockerHub implementation for send_tags (#6580)
Browse files Browse the repository at this point in the history 
### Description of the change

Adds the implementation for retrieving and sending tags for dockerhub
repositories.

### Benefits

Enables an initial integration to be tested.

### Possible drawbacks

<!-- Describe any known limitations with your change -->

### Applicable issues

<!-- Enter any applicable Issues here (You can reference an issue using
#) -->

- ref #6263 

### Additional information

IRL test:

```console
grpcurl -plaintext -proto ./proto/ocicatalog.proto -d '{ "repository": {"registry": "registry-1.docker.io", "namespace": "bitnamicharts", "name": "zookeeper" }}' 0.0.0.0:50001 ocicatalog.OCICatalog.ListTagsForRepository
{
  "name": "11.4.10"
}
{
  "name": "11.4.9"
}
{
  "name": "11.4.8"
}
{
  "name": "11.4.7"
}
{
  "name": "11.4.6"
}
{
  "name": "11.4.5"
}
{
  "name": "11.4.4"
}
{
  "name": "11.4.3"
}
{
  "name": "11.4.2"
}
{
  "name": "11.4.1"
}
{
  "name": "11.3.2"
}
{
  "name": "11.3.1"
}
{
  "name": "11.2.1"
}
{
  "name": "11.1.6"
}
{
  "name": "11.1.5"
}
{
  "name": "11.1.4"
}
{
  "name": "11.1.3"
}
{
  "name": "11.1.2"
}
{
  "name": "11.1.1"
}
{
  "name": "11.0.3"
}
{
  "name": "11.0.2"
}
{
  "name": "11.0.1"
}
{
  "name": "11.0.0"
}
{
  "name": "10.2.5"
}
{
  "name": "10.2.4"
}
{
  "name": "10.2.3"
}
```

---------

Signed-off-by: Michael Nelson <minelson@vmware.com>
  • Loading branch information
@absoludity
absoludity committed Aug 9, 2023
1 parent ae84b08 commit 1abcbc4
Show file tree
Hide file tree
Showing 2 changed files with 148 additions and 16 deletions.
151 changes: 139 additions & 12 deletions cmd/oci-catalog/src/providers/dockerhub.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@ use tokio::sync::mpsc;
use tonic::Status;

/// The default page size with which requests are sent to docker hub.
/// We fetch all results in batches of this page size.
const DEFAULT_PAGE_SIZE: u8 = 100;
pub const PROVIDER_NAME: &str = "DockerHubAPI";
pub const DOCKERHUB_URI: &str = "https://hub.docker.com";

#[derive(Serialize, Deserialize)]
struct DockerHubV2Repository {
Expand All @@ -29,6 +31,21 @@ struct DockerHubV2RepositoriesResult {
results: Vec<DockerHubV2Repository>,
}

#[derive(Serialize, Deserialize)]
struct DockerHubV2Tag {
name: String,
repository_type: Option<String>,
content_type: String,
}

#[derive(Serialize, Deserialize)]
struct DockerHubV2TagsResult {
count: u16,
next: Option<String>,
previous: Option<String>,
results: Vec<DockerHubV2Tag>,
}

#[derive(Debug, Default)]
pub struct DockerHubAPI {}

Expand All @@ -38,16 +55,17 @@ impl OCICatalogSender for DockerHubAPI {
PROVIDER_NAME
}

// Update to return a result so errors are handled properly.
async fn send_repositories(
&self,
tx: mpsc::Sender<Result<Repository, Status>>,
request: &ListRepositoriesRequest,
) {
let mut url = url_for_request(request);
let mut url = url_for_request_repositories(request);

let client = reqwest::Client::builder().build().unwrap();

// We continue making the request until there is no `next` url
// in the result.
loop {
log::debug!("requesting: {}", url);
let response = match client.get(url.clone()).send().await {
Expand Down Expand Up @@ -105,25 +123,81 @@ impl OCICatalogSender for DockerHubAPI {
}
}

async fn send_tags(&self, tx: mpsc::Sender<Result<Tag, Status>>, _request: &ListTagsRequest) {
for count in 0..10 {
tx.send(Ok(Tag {
name: format!("tag-{}", count),
}))
.await
.unwrap();
async fn send_tags(&self, tx: mpsc::Sender<Result<Tag, Status>>, request: &ListTagsRequest) {
let mut url = match url_for_request_tags(request) {
Ok(u) => u,
Err(e) => {
tx.send(Err(e)).await.unwrap();
return;
}
};

let client = reqwest::Client::builder().build().unwrap();

// We continue making the request until there is no `next` url
// in the result.
loop {
log::debug!("requesting: {}", url);
let response = match client.get(url.clone()).send().await {
Ok(r) => r,
Err(e) => {
tx.send(Err(Status::failed_precondition(e.to_string())))
.await
.unwrap();
return;
}
};

if response.status() != StatusCode::OK {
tx.send(Err(Status::failed_precondition(format!(
"unexpected status code when requesting {}: {}",
url,
response.status()
))))
.await
.unwrap();
return;
}

let body = match response.text().await {
Ok(b) => b,
Err(e) => {
tx.send(Err(Status::failed_precondition(format!(
"unable to extract body from response: {}",
e.to_string()
))))
.await
.unwrap();
return;
}
};
log::trace!("response body: {}", body);

let response: DockerHubV2TagsResult = serde_json::from_str(&body).unwrap();

for tag in response.results {
tx.send(Ok(Tag { name: tag.name })).await.unwrap();
}

if response.next.is_some() {
url = reqwest::Url::parse(&response.next.unwrap()).unwrap();
} else {
break;
}
}
}
}

fn url_for_request(request: &ListRepositoriesRequest) -> Url {
let mut url = reqwest::Url::parse("https://hub.docker.com/v2/repositories/").unwrap();
fn url_for_request_repositories(request: &ListRepositoriesRequest) -> Url {
let mut url = reqwest::Url::parse(DOCKERHUB_URI).unwrap();

if !request.namespace.is_empty() {
url.set_path(&format!(
"/v2/namespaces/{}/repositories/",
request.namespace
));
} else {
url.set_path("/v2/repositories/");
}
// For now we use a default page size and default ordering.
url.query_pairs_mut()
Expand All @@ -137,6 +211,30 @@ fn url_for_request(request: &ListRepositoriesRequest) -> Url {
url
}

fn url_for_request_tags(request: &ListTagsRequest) -> Result<Url, Status> {
let mut url = reqwest::Url::parse(DOCKERHUB_URI).unwrap();

let repo = match request.repository.clone() {
Some(r) => r,
None => {
return Err(Status::invalid_argument(format!(
"repository not set in request"
)))
}
};

url.set_path(&format!(
"/v2/repositories/{}/{}/tags",
repo.namespace, repo.name
));

// For now we use a default page size.
url.query_pairs_mut()
.append_pair("page_size", &format!("{}", DEFAULT_PAGE_SIZE));

Ok(url)
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down Expand Up @@ -165,6 +263,35 @@ mod tests {
..Default::default()
}, "https://hub.docker.com/v2/namespaces/bitnamicharts/repositories/?page_size=100&ordering=name&content_types=helm&content_types=image")]
fn test_url_for_request(#[case] request: ListRepositoriesRequest, #[case] expected_url: Url) {
assert_eq!(url_for_request(&request), expected_url);
assert_eq!(url_for_request_repositories(&request), expected_url);
}

#[rstest]
#[case::without_repository(ListTagsRequest{
..Default::default()
}, Err(Status::invalid_argument("bang")))]
#[case::with_repository(ListTagsRequest{
repository: Some(Repository{
namespace: "bitnamicharts".to_string(),
name: "apache".to_string(),
..Default::default()
}),
..Default::default()
}, Ok(reqwest::Url::parse("https://hub.docker.com/v2/repositories/bitnamicharts/apache/tags?page_size=100").unwrap()))]
fn test_url_for_request_tags(
#[case] request: ListTagsRequest,
#[case] expected_result: Result<Url, Status>,
) {
match expected_result {
Ok(url) => {
assert_eq!(url_for_request_tags(&request).unwrap(), url);
}
Err(e) => {
assert_eq!(
url_for_request_tags(&request).err().unwrap().code(),
e.code()
)
}
}
}
}
13 changes: 9 additions & 4 deletions cmd/oci-catalog/src/providers/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,22 @@ use tonic::Status;

pub mod dockerhub;

// OCICatalogSender is a trait that can be implemented by different providers.
//
// Initially we'll provide a DockerHub implementation followed by Harbor and
// others.
/// OCICatalogSender is a trait that can be implemented by different providers.
///
/// Initially we'll provide a DockerHub implementation followed by Harbor and
/// others.
#[tonic::async_trait]
pub trait OCICatalogSender {
/// send_repositories requests repositories from the provider and sends
/// them down a channel for our API to return.
async fn send_repositories(
&self,
tx: mpsc::Sender<Result<Repository, Status>>,
request: &ListRepositoriesRequest,
);

/// send_tags requests tags for a repository of a provider and sends
/// them down a channel for our API to return.
async fn send_tags(&self, tx: mpsc::Sender<Result<Tag, Status>>, request: &ListTagsRequest);

// The id simply gives a way in tests to determine which provider
Expand Down

0 comments on commit 1abcbc4

Please sign in to comment.