Skip to content

use new jql search API #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 24, 2025
Merged

use new jql search API #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/org/embulk/input/jira/Constant.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ public final class Constant
public static final String DEFAULT_TIMESTAMP_PATTERN = "%Y-%m-%dT%H:%M:%S.%L%z";

public static final String CREDENTIAL_URI_PATH = "rest/api/latest/myself";
public static final String SEARCH_URI_PATH = "rest/api/latest/search";
public static final String SEARCH_URI_PATH = "rest/api/latest/search/jql";

private Constant(){}
}
21 changes: 12 additions & 9 deletions src/main/java/org/embulk/input/jira/JiraInputPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import org.apache.commons.lang3.tuple.Pair;
import org.embulk.config.ConfigDiff;
import org.embulk.config.ConfigException;
import org.embulk.config.ConfigSource;
Expand Down Expand Up @@ -159,20 +160,21 @@ public TaskReport run(final TaskSource taskSource,
final int maxResults = task.getMaxResults();
try (final PageBuilder pageBuilder = getPageBuilder(schema, output)) {
if (isPreview()) {
final List<Issue> issues = jiraClient.searchIssues(task, 0, Math.min(maxResults, PREVIEW_RECORDS_COUNT));
final Pair<List<Issue>, String> result = jiraClient.searchIssues(task, null, Math.min(maxResults, PREVIEW_RECORDS_COUNT));
final List<Issue> issues = result.getLeft();
issues.forEach(issue -> JiraUtil.addRecord(issue, schema, task, pageBuilder));
}
else {
int currentPage = 0;
final int totalCount = jiraClient.getTotalCount(task);
final int totalPage = JiraUtil.calculateTotalPage(totalCount, maxResults);
LOGGER.info(String.format("Total pages (%d)", totalPage));
while (currentPage < totalPage) {
LOGGER.info(String.format("Fetching page %d/%d", (currentPage + 1), totalPage));
final List<Issue> issues = jiraClient.searchIssues(task, (currentPage * maxResults), maxResults);
String nextPageToken = null;
do {
LOGGER.info(String.format("Fetching page %d", (currentPage + 1)));
final Pair<List<Issue>, String> result = jiraClient.searchIssues(task, nextPageToken, maxResults);
final List<Issue> issues = result.getLeft();
nextPageToken = result.getRight();
issues.forEach(issue -> JiraUtil.addRecord(issue, schema, task, pageBuilder));
currentPage++;
}
} while(nextPageToken != null);
}
pageBuilder.finish();
}
Expand All @@ -193,7 +195,8 @@ public ConfigDiff guess(final ConfigSource config)

private List<ConfigDiff> getGuessedColumns(final JiraClient jiraClient, final PluginTask task)
{
final List<Issue> issues = jiraClient.searchIssues(task, 0, GUESS_RECORDS_COUNT);
final Pair<List<Issue>, String> result = jiraClient.searchIssues(task, null, GUESS_RECORDS_COUNT);
final List<Issue> issues = result.getLeft();
if (issues.isEmpty()) {
throw new ConfigException("Could not guess schema due to empty data set");
}
Expand Down
39 changes: 22 additions & 17 deletions src/main/java/org/embulk/input/jira/client/JiraClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
Expand Down Expand Up @@ -64,11 +65,18 @@ public void checkUserCredentials(final PluginTask task)
}
}

public List<Issue> searchIssues(final PluginTask task, final int startAt, final int maxResults)
public Pair<List<Issue>, String> searchIssues(final PluginTask task, final String nextPageToken, final int maxResults)
{
final String response = searchJiraAPI(task, startAt, maxResults);
final String response = searchJiraAPI(task, nextPageToken, maxResults);
final JsonObject result = new JsonParser().parse(response).getAsJsonObject();
return StreamSupport.stream(result.get("issues").getAsJsonArray().spliterator(), false)
final JsonElement newNextPageTokenJson = result.get("nextPageToken");
final String newNextPageToken;
if (newNextPageTokenJson == null) {
newNextPageToken = null;
} else {
newNextPageToken = newNextPageTokenJson.getAsString();
}
final List<Issue> issues = StreamSupport.stream(result.get("issues").getAsJsonArray().spliterator(), false)
.map(jsonElement -> {
final JsonObject json = jsonElement.getAsJsonObject();
final JsonObject fields = json.get("fields").getAsJsonObject();
Expand All @@ -81,14 +89,10 @@ public List<Issue> searchIssues(final PluginTask task, final int startAt, final
return new Issue(json);
})
.collect(Collectors.toList());
return Pair.of(issues, newNextPageToken);
}

public int getTotalCount(final PluginTask task)
{
return new JsonParser().parse(searchJiraAPI(task, 0, MIN_RESULTS)).getAsJsonObject().get("total").getAsInt();
}

private String searchJiraAPI(final PluginTask task, final int startAt, final int maxResults)
private String searchJiraAPI(final PluginTask task, final String nextPageToken, final int maxResults)
{
try {
return RetryExecutor.builder()
Expand All @@ -101,7 +105,7 @@ private String searchJiraAPI(final PluginTask task, final int startAt, final int
@Override
public String call() throws Exception
{
return authorizeAndRequest(task, JiraUtil.buildSearchUrl(task.getUri()), createSearchIssuesBody(task, startAt, maxResults));
return authorizeAndRequest(task, JiraUtil.buildSearchUrl(task.getUri()), createSearchIssuesBody(task, nextPageToken, maxResults));
}

@Override
Expand Down Expand Up @@ -243,21 +247,22 @@ private HttpRequestBase createGetRequest(final PluginTask task, final String url
return request;
}

private String createSearchIssuesBody(final PluginTask task, final int startAt, final int maxResults)
private String createSearchIssuesBody(final PluginTask task, final String nextPageToken, final int maxResults)
{
final JsonObject body = new JsonObject();
final Optional<String> jql = task.getJQL();
body.add("jql", new JsonPrimitive(jql.orElse("")));
body.add("startAt", new JsonPrimitive(startAt));
if (nextPageToken != null) {
body.add("nextPageToken", new JsonPrimitive(nextPageToken));
}
body.add("maxResults", new JsonPrimitive(maxResults));
final JsonArray fields = new JsonArray();
fields.add("*all");
body.add("fields", fields);
final JsonArray expand = new JsonArray();
task.getExpand().forEach(e -> {
expand.add(e);
});
body.add("expand", expand);
final String expands = task.getExpand().stream().collect(Collectors.joining(","));
if (!expands.isEmpty()) {
body.add("expand", new JsonPrimitive(expands));
}
return body.toString();
}
}