From 483abd21bcf8a707d3efd4c86a47a704b7dbea38 Mon Sep 17 00:00:00 2001 From: Yas Okada Date: Fri, 14 Mar 2025 09:42:17 +0000 Subject: [PATCH] use new jql search API --- .../java/org/embulk/input/jira/Constant.java | 2 +- .../embulk/input/jira/JiraInputPlugin.java | 21 +++++----- .../embulk/input/jira/client/JiraClient.java | 39 +++++++++++-------- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/embulk/input/jira/Constant.java b/src/main/java/org/embulk/input/jira/Constant.java index 1440418..bf68cfc 100644 --- a/src/main/java/org/embulk/input/jira/Constant.java +++ b/src/main/java/org/embulk/input/jira/Constant.java @@ -11,7 +11,7 @@ public final class Constant public static final String DEFAULT_TIMESTAMP_PATTERN = "%Y-%m-%dT%H:%M:%S.%L%z"; public static final String CREDENTIAL_URI_PATH = "rest/api/latest/myself"; - public static final String SEARCH_URI_PATH = "rest/api/latest/search"; + public static final String SEARCH_URI_PATH = "rest/api/latest/search/jql"; private Constant(){} } diff --git a/src/main/java/org/embulk/input/jira/JiraInputPlugin.java b/src/main/java/org/embulk/input/jira/JiraInputPlugin.java index 1140836..6a673c1 100644 --- a/src/main/java/org/embulk/input/jira/JiraInputPlugin.java +++ b/src/main/java/org/embulk/input/jira/JiraInputPlugin.java @@ -4,6 +4,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonNull; import com.google.gson.JsonObject; +import org.apache.commons.lang3.tuple.Pair; import org.embulk.config.ConfigDiff; import org.embulk.config.ConfigException; import org.embulk.config.ConfigSource; @@ -159,20 +160,21 @@ public TaskReport run(final TaskSource taskSource, final int maxResults = task.getMaxResults(); try (final PageBuilder pageBuilder = getPageBuilder(schema, output)) { if (isPreview()) { - final List issues = jiraClient.searchIssues(task, 0, Math.min(maxResults, PREVIEW_RECORDS_COUNT)); + final Pair, String> result = jiraClient.searchIssues(task, null, Math.min(maxResults, PREVIEW_RECORDS_COUNT)); + final List issues = result.getLeft(); issues.forEach(issue -> JiraUtil.addRecord(issue, schema, task, pageBuilder)); } else { int currentPage = 0; - final int totalCount = jiraClient.getTotalCount(task); - final int totalPage = JiraUtil.calculateTotalPage(totalCount, maxResults); - LOGGER.info(String.format("Total pages (%d)", totalPage)); - while (currentPage < totalPage) { - LOGGER.info(String.format("Fetching page %d/%d", (currentPage + 1), totalPage)); - final List issues = jiraClient.searchIssues(task, (currentPage * maxResults), maxResults); + String nextPageToken = null; + do { + LOGGER.info(String.format("Fetching page %d", (currentPage + 1))); + final Pair, String> result = jiraClient.searchIssues(task, nextPageToken, maxResults); + final List issues = result.getLeft(); + nextPageToken = result.getRight(); issues.forEach(issue -> JiraUtil.addRecord(issue, schema, task, pageBuilder)); currentPage++; - } + } while(nextPageToken != null); } pageBuilder.finish(); } @@ -193,7 +195,8 @@ public ConfigDiff guess(final ConfigSource config) private List getGuessedColumns(final JiraClient jiraClient, final PluginTask task) { - final List issues = jiraClient.searchIssues(task, 0, GUESS_RECORDS_COUNT); + final Pair, String> result = jiraClient.searchIssues(task, null, GUESS_RECORDS_COUNT); + final List issues = result.getLeft(); if (issues.isEmpty()) { throw new ConfigException("Could not guess schema due to empty data set"); } diff --git a/src/main/java/org/embulk/input/jira/client/JiraClient.java b/src/main/java/org/embulk/input/jira/client/JiraClient.java index 09391ee..b192f67 100644 --- a/src/main/java/org/embulk/input/jira/client/JiraClient.java +++ b/src/main/java/org/embulk/input/jira/client/JiraClient.java @@ -6,6 +6,7 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; +import org.apache.commons.lang3.tuple.Pair; import org.apache.http.HttpStatus; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; @@ -64,11 +65,18 @@ public void checkUserCredentials(final PluginTask task) } } - public List searchIssues(final PluginTask task, final int startAt, final int maxResults) + public Pair, String> searchIssues(final PluginTask task, final String nextPageToken, final int maxResults) { - final String response = searchJiraAPI(task, startAt, maxResults); + final String response = searchJiraAPI(task, nextPageToken, maxResults); final JsonObject result = new JsonParser().parse(response).getAsJsonObject(); - return StreamSupport.stream(result.get("issues").getAsJsonArray().spliterator(), false) + final JsonElement newNextPageTokenJson = result.get("nextPageToken"); + final String newNextPageToken; + if (newNextPageTokenJson == null) { + newNextPageToken = null; + } else { + newNextPageToken = newNextPageTokenJson.getAsString(); + } + final List issues = StreamSupport.stream(result.get("issues").getAsJsonArray().spliterator(), false) .map(jsonElement -> { final JsonObject json = jsonElement.getAsJsonObject(); final JsonObject fields = json.get("fields").getAsJsonObject(); @@ -81,14 +89,10 @@ public List searchIssues(final PluginTask task, final int startAt, final return new Issue(json); }) .collect(Collectors.toList()); + return Pair.of(issues, newNextPageToken); } - public int getTotalCount(final PluginTask task) - { - return new JsonParser().parse(searchJiraAPI(task, 0, MIN_RESULTS)).getAsJsonObject().get("total").getAsInt(); - } - - private String searchJiraAPI(final PluginTask task, final int startAt, final int maxResults) + private String searchJiraAPI(final PluginTask task, final String nextPageToken, final int maxResults) { try { return RetryExecutor.builder() @@ -101,7 +105,7 @@ private String searchJiraAPI(final PluginTask task, final int startAt, final int @Override public String call() throws Exception { - return authorizeAndRequest(task, JiraUtil.buildSearchUrl(task.getUri()), createSearchIssuesBody(task, startAt, maxResults)); + return authorizeAndRequest(task, JiraUtil.buildSearchUrl(task.getUri()), createSearchIssuesBody(task, nextPageToken, maxResults)); } @Override @@ -243,21 +247,22 @@ private HttpRequestBase createGetRequest(final PluginTask task, final String url return request; } - private String createSearchIssuesBody(final PluginTask task, final int startAt, final int maxResults) + private String createSearchIssuesBody(final PluginTask task, final String nextPageToken, final int maxResults) { final JsonObject body = new JsonObject(); final Optional jql = task.getJQL(); body.add("jql", new JsonPrimitive(jql.orElse(""))); - body.add("startAt", new JsonPrimitive(startAt)); + if (nextPageToken != null) { + body.add("nextPageToken", new JsonPrimitive(nextPageToken)); + } body.add("maxResults", new JsonPrimitive(maxResults)); final JsonArray fields = new JsonArray(); fields.add("*all"); body.add("fields", fields); - final JsonArray expand = new JsonArray(); - task.getExpand().forEach(e -> { - expand.add(e); - }); - body.add("expand", expand); + final String expands = task.getExpand().stream().collect(Collectors.joining(",")); + if (!expands.isEmpty()) { + body.add("expand", new JsonPrimitive(expands)); + } return body.toString(); } }