Skip to content

Commit

Permalink
Expose pagination API in Java Jobs client
Browse files Browse the repository at this point in the history
  • Loading branch information
@mprimi
rboorgapally authored and mprimi committed Aug 11, 2020
1 parent 6aaeafa commit fb8a2d8
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 4 deletions.
86 changes: 86 additions & 0 deletions genie-client/src/integTest/java/com/netflix/genie/client/JobClientIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.netflix.genie.client.apis.SortAttribute;
import com.netflix.genie.client.apis.SortDirection;
import com.netflix.genie.common.dto.Cluster;
import com.netflix.genie.common.dto.ClusterCriteria;
import com.netflix.genie.common.dto.ClusterStatus;
Expand All @@ -37,6 +39,7 @@
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
Expand Down Expand Up @@ -255,6 +258,89 @@ void canSubmitJob() throws Exception {
)
.extracting(JobSearchResult::getId)
.containsExactlyInAnyOrder(killJobId, timeoutJobId);

final List<String> ids = Lists.newArrayList(sleepJobId, killJobId, timeoutJobId, dateJobId, echoJobId);
// Paginate, 1 result per page
for (int i = 0; i < ids.size(); i++) {
final List<JobSearchResult> page = this.jobClient.getJobs(
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
1,
SortAttribute.CREATED,
SortDirection.ASC,
i
);

Assertions.assertThat(page.size()).isEqualTo(1);
Assertions.assertThat(page.get(0).getId()).isEqualTo(ids.get(i));
}

// Paginate, 1 result per page, reverse order
Collections.reverse(ids);
for (int i = 0; i < ids.size(); i++) {
final List<JobSearchResult> page = this.jobClient.getJobs(
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
1,
SortAttribute.CREATED,
SortDirection.DESC,
i
);

Assertions.assertThat(page.size()).isEqualTo(1);
Assertions.assertThat(page.get(0).getId()).isEqualTo(ids.get(i));
}

// Ask for page beyond end of results
Assertions.assertThat(
this.jobClient.getJobs(
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
10,
SortAttribute.CREATED,
SortDirection.DESC,
1
)
).isEmpty();
}

private String createDummyCluster() throws Exception {
Expand Down
84 changes: 82 additions & 2 deletions genie-client/src/main/java/com/netflix/genie/client/JobClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import com.fasterxml.jackson.databind.node.JsonNodeType;
import com.google.common.io.ByteStreams;
import com.netflix.genie.client.apis.JobService;
import com.netflix.genie.client.apis.SortAttribute;
import com.netflix.genie.client.apis.SortDirection;
import com.netflix.genie.client.configs.GenieNetworkConfiguration;
import com.netflix.genie.client.exceptions.GenieClientException;
import com.netflix.genie.common.dto.Application;
Expand All @@ -44,6 +46,7 @@
import retrofit2.Retrofit;

import javax.annotation.Nullable;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import java.io.IOException;
Expand Down Expand Up @@ -289,6 +292,8 @@ public List<JobSearchResult> getJobs(
* @return A list of jobs.
* @throws GenieClientException If the response received is not 2xx.
* @throws IOException For Network and other IO issues.
* @deprecated Use {@link #getJobs(String, String, String, Set, Set, String, String, String, String, Long, Long,
* Long, Long, String, String, Integer, SortAttribute, SortDirection, Integer)}
*/
public List<JobSearchResult> getJobs(
@Nullable final String id,
Expand All @@ -306,6 +311,78 @@ public List<JobSearchResult> getJobs(
@Nullable final Long maxFinished,
@Nullable final String grouping,
@Nullable final String groupingInstance
) throws IOException, GenieClientException {
return this.getJobs(
id,
name,
user,
statuses,
tags,
clusterName,
clusterId,
commandName,
commandId,
minStarted,
maxStarted,
minFinished,
maxFinished,
grouping,
groupingInstance,
null,
null,
null,
null
);
}


/**
* Method to get a list of all the jobs from Genie for the query parameters specified.
*
* @param id id for job
* @param name name of job (can be a SQL-style pattern such as HIVE%)
* @param user user who submitted job
* @param statuses statuses of jobs to find
* @param tags tags for the job
* @param clusterName the name of the cluster
* @param clusterId the id of the cluster
* @param commandName the name of the command run by the job
* @param commandId the id of the command run by the job
* @param minStarted The time which the job had to start after in order to be return (inclusive)
* @param maxStarted The time which the job had to start before in order to be returned (exclusive)
* @param minFinished The time which the job had to finish after in order to be return (inclusive)
* @param maxFinished The time which the job had to finish before in order to be returned (exclusive)
* @param grouping The grouping the job should be a member of
* @param groupingInstance The grouping instance the job should be a member of
* @param pageSize The maximum number of results returned
* @param sortAttribute The entity attribute used to sort
* @param sortDirection The sort direction
* @param pageIndex The page index
* @return A list of jobs.
* @throws GenieClientException If the response received is not 2xx.
* @throws IOException For Network and other IO issues.
*/
@SuppressWarnings("checkstyle:ParameterNumber")
public List<JobSearchResult> getJobs(
@Nullable final String id,
@Nullable final String name,
@Nullable final String user,
@Nullable final Set<String> statuses,
@Nullable final Set<String> tags,
@Nullable final String clusterName,
@Nullable final String clusterId,
@Nullable final String commandName,
@Nullable final String commandId,
@Nullable final Long minStarted,
@Nullable final Long maxStarted,
@Nullable final Long minFinished,
@Nullable final Long maxFinished,
@Nullable final String grouping,
@Nullable final String groupingInstance,
@Nullable @Min(1) final Integer pageSize,
@Nullable final SortAttribute sortAttribute,
@Nullable final SortDirection sortDirection,
@Nullable @Min(0) final Integer pageIndex
) throws IOException, GenieClientException {
return GenieClientUtils.parseSearchResultsResponse(
this.jobService.getJobs(
Expand All @@ -323,7 +400,10 @@ public List<JobSearchResult> getJobs(
minFinished,
maxFinished,
grouping,
groupingInstance
groupingInstance,
pageSize,
GenieClientUtils.getSortParameter(sortAttribute, sortDirection),
pageIndex
).execute(),
"jobSearchResultList",
JobSearchResult.class
Expand Down Expand Up @@ -481,7 +561,7 @@ public InputStream getJobStderr(
* manifest is returned.
* </p>
*
* @param jobId The id of the job whose output file is desired.
* @param jobId The id of the job whose output file is desired.
* @param outputFilePath The path to the file within output directory.
* @return An input stream to the output file contents.
* @throws GenieClientException If the response received is not 2xx.
Expand Down
11 changes: 9 additions & 2 deletions genie-client/src/main/java/com/netflix/genie/client/apis/JobService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,12 @@ Call<Void> submitJobWithAttachments(
* @param maxFinished The time which the job had to finish before in order to be returned (exclusive)
* @param grouping The grouping the job should be a member of
* @param groupingInstance The grouping instance the job should be a member of
* @param size The maximum number of results in the page
* @param sort The sort order
* @param page The page index
* @return A callable object.
*/
@SuppressWarnings("checkstyle:parameternumber")
@GET(JOBS_URL_SUFFIX)
Call<JsonNode> getJobs(
@Query("id") String id,
Expand All @@ -112,7 +116,10 @@ Call<JsonNode> getJobs(
@Query("minFinished") Long minFinished,
@Query("maxFinished") Long maxFinished,
@Query("grouping") String grouping,
@Query("groupingInstance") String groupingInstance
@Query("groupingInstance") String groupingInstance,
@Query("size") Integer size,
@Query("sort") String sort,
@Query("page") Integer page
);

/**
Expand All @@ -132,7 +139,7 @@ Call<JsonNode> getJobs(
* manifest is returned.
* </p>
*
* @param jobId The id of the job whose output file is desired.
* @param jobId The id of the job whose output file is desired.
* @param outputFilePath The path to the file within output directory.
* @return A callable object.
*/
Expand Down

0 comments on commit fb8a2d8

Please sign in to comment.