diff --git a/README.md b/README.md index ddadf8a..488e607 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Required Embulk version >= 0.9.20 - **columns** target issue attributes. You can generate this configuration by `guess` command (array, required) - **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 1) - **retry_limit**: Try to retry this times (integer, default: 5) +- **expand_json_on_guess** The boolean value is to enable/disable json expanding when `guess`. (boolean, default: true) ## Example diff --git a/build.gradle b/build.gradle index 4495645..72ebae4 100644 --- a/build.gradle +++ b/build.gradle @@ -14,7 +14,7 @@ configurations { provided } -version = "0.2.10" +version = "0.2.11" sourceCompatibility = 1.8 targetCompatibility = 1.8 diff --git a/src/main/java/org/embulk/input/jira/Issue.java b/src/main/java/org/embulk/input/jira/Issue.java index 7445ee1..659351e 100644 --- a/src/main/java/org/embulk/input/jira/Issue.java +++ b/src/main/java/org/embulk/input/jira/Issue.java @@ -58,16 +58,16 @@ else if (keys.isEmpty() || (json.isJsonArray() && json.getAsJsonArray().size() = } } - public synchronized JsonObject getFlatten() + public synchronized JsonObject getFlatten(Boolean expandJsonOnGuess) { if (flatten == null) { flatten = new JsonObject(); - manipulatingFlattenJson(json, ""); + manipulatingFlattenJson(json, "", expandJsonOnGuess); } return flatten; } - private void manipulatingFlattenJson(JsonElement in, String prefix) + private void manipulatingFlattenJson(JsonElement in, String prefix, boolean expandJsonOnGuess) { if (in.isJsonObject()) { JsonObject obj = in.getAsJsonObject(); @@ -83,7 +83,12 @@ private void manipulatingFlattenJson(JsonElement in, String prefix) for (Entry entry : obj.entrySet()) { String key = entry.getKey(); JsonElement value = entry.getValue(); - manipulatingFlattenJson(value, appendPrefix(prefix, key)); + if (expandJsonOnGuess) { + manipulatingFlattenJson(value, appendPrefix(prefix, key), expandJsonOnGuess); + } + else { + flatten.add(key, value); + } } } } @@ -106,7 +111,7 @@ else if (in.isJsonArray()) { newObj.get(key).getAsJsonArray().add(elem.getAsJsonObject().get(key)); } } - manipulatingFlattenJson(newObj, prefix); + manipulatingFlattenJson(newObj, prefix, expandJsonOnGuess); } else { flatten.add(prefix, diff --git a/src/main/java/org/embulk/input/jira/JiraInputPlugin.java b/src/main/java/org/embulk/input/jira/JiraInputPlugin.java index 148b2c9..fdd135b 100644 --- a/src/main/java/org/embulk/input/jira/JiraInputPlugin.java +++ b/src/main/java/org/embulk/input/jira/JiraInputPlugin.java @@ -86,6 +86,10 @@ public interface PluginTask @Config("auth_method") @ConfigDefault("\"basic\"") public AuthenticateMethod getAuthMethod(); + + @Config("expand_json_on_guess") + @ConfigDefault("true") + public boolean getExpandJsonOnGuess(); } @Override @@ -160,7 +164,7 @@ public ConfigDiff guess(final ConfigSource config) if (issues.isEmpty()) { throw new ConfigException("Could not guess schema due to empty data set"); } - final Buffer sample = Buffer.copyOf(createSamples(issues, getUniqueAttributes(issues)).toString().getBytes()); + final Buffer sample = Buffer.copyOf(createSamples(issues, getUniqueAttributes(issues, task.getExpandJsonOnGuess()), task.getExpandJsonOnGuess()).toString().getBytes()); final JsonNode columns = Exec.getInjector().getInstance(GuessExecutor.class) .guessParserConfig(sample, Exec.newConfigSource(), createGuessConfig()) .get(JsonNode.class, "columns"); @@ -174,22 +178,22 @@ private ConfigSource createGuessConfig() .set("guess_sample_buffer_bytes", GUESS_BUFFER_SIZE); } - private SortedSet getUniqueAttributes(final List issues) + private SortedSet getUniqueAttributes(final List issues, final boolean expandJsonOnGuess) { final SortedSet uniqueAttributes = new TreeSet<>(); for (final Issue issue : issues) { - for (final Entry entry : issue.getFlatten().entrySet()) { + for (final Entry entry : issue.getFlatten(expandJsonOnGuess).entrySet()) { uniqueAttributes.add(entry.getKey()); } } return uniqueAttributes; } - private JsonArray createSamples(final List issues, final Set uniqueAttributes) + private JsonArray createSamples(final List issues, final Set uniqueAttributes, final boolean expandJsonOnGuess) { final JsonArray samples = new JsonArray(); for (final Issue issue : issues) { - final JsonObject flatten = issue.getFlatten(); + final JsonObject flatten = issue.getFlatten(expandJsonOnGuess); final JsonObject unified = new JsonObject(); for (final String key : uniqueAttributes) { JsonElement value = flatten.get(key); diff --git a/src/test/java/org/embulk/input/jira/IssueTest.java b/src/test/java/org/embulk/input/jira/IssueTest.java index d0e0a82..b556fc3 100644 --- a/src/test/java/org/embulk/input/jira/IssueTest.java +++ b/src/test/java/org/embulk/input/jira/IssueTest.java @@ -31,7 +31,7 @@ public void test_toRecord_simple() String testName = "simple"; Issue issue = new Issue(flattenData.get(testName).getAsJsonObject()); JsonObject expected = flattenExpected.get(testName).getAsJsonObject(); - assertEquals(expected, issue.getFlatten()); + assertEquals(expected, issue.getFlatten(true)); } @Test @@ -40,7 +40,7 @@ public void test_toRecord_twoLevels() String testName = "twoLevels"; Issue issue = new Issue(flattenData.get(testName).getAsJsonObject()); JsonObject expected = flattenExpected.get(testName).getAsJsonObject(); - assertEquals(expected, issue.getFlatten()); + assertEquals(expected, issue.getFlatten(true)); } @Test @@ -49,7 +49,7 @@ public void test_toRecord_threeLevels() String testName = "threeLevels"; Issue issue = new Issue(flattenData.get(testName).getAsJsonObject()); JsonObject expected = flattenExpected.get(testName).getAsJsonObject(); - assertEquals(expected, issue.getFlatten()); + assertEquals(expected, issue.getFlatten(true)); } @Test @@ -58,7 +58,7 @@ public void test_toRecord_threeLevelsWithoutKeys() String testName = "threeLevelsWithoutKeys"; Issue issue = new Issue(flattenData.get(testName).getAsJsonObject()); JsonObject expected = flattenExpected.get(testName).getAsJsonObject(); - assertEquals(expected, issue.getFlatten()); + assertEquals(expected, issue.getFlatten(true)); } @Test @@ -67,7 +67,7 @@ public void test_toRecord_threeLevelsWithKeys() String testName = "threeLevelsWithKeys"; Issue issue = new Issue(flattenData.get(testName).getAsJsonObject()); JsonObject expected = flattenExpected.get(testName).getAsJsonObject(); - assertEquals(expected, issue.getFlatten()); + assertEquals(expected, issue.getFlatten(true)); } @Test @@ -76,7 +76,7 @@ public void test_toRecord_threeLevelsWithNullKeys() String testName = "threeLevelsWithNullKeys"; Issue issue = new Issue(flattenData.get(testName).getAsJsonObject()); JsonObject expected = flattenExpected.get(testName).getAsJsonObject(); - assertEquals(expected, issue.getFlatten()); + assertEquals(expected, issue.getFlatten(true)); } @Test @@ -85,7 +85,7 @@ public void test_toRecord_arrayWithAllJsonObjectWithSameKeysAndEmptyObject() String testName = "arrayWithAllJsonObjectWithSameKeysAndEmptyObject"; Issue issue = new Issue(flattenData.get(testName).getAsJsonObject()); JsonObject expected = flattenExpected.get(testName).getAsJsonObject(); - assertEquals(expected, issue.getFlatten()); + assertEquals(expected, issue.getFlatten(true)); } @Test @@ -94,7 +94,7 @@ public void test_toRecord_arrayWithAllJsonObjectWithSameKeysAndNotEmptyObject() String testName = "arrayWithAllJsonObjectWithSameKeysAndNotEmptyObject"; Issue issue = new Issue(flattenData.get(testName).getAsJsonObject()); JsonObject expected = flattenExpected.get(testName).getAsJsonObject(); - assertEquals(expected, issue.getFlatten()); + assertEquals(expected, issue.getFlatten(true)); } @Test @@ -103,7 +103,7 @@ public void test_toRecord_arrayWithAllJsonObjectWithoutSameKeys() String testName = "arrayWithAllJsonObjectWithoutSameKeys"; Issue issue = new Issue(flattenData.get(testName).getAsJsonObject()); JsonObject expected = flattenExpected.get(testName).getAsJsonObject(); - assertEquals(expected, issue.getFlatten()); + assertEquals(expected, issue.getFlatten(true)); } @Test