diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..6282f6e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +--- +version: 2 +updates: + # Enable workflow version updates for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..2309ca6 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,41 @@ +name: Check + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: "zulu" + + - name: Grant execute permission for gradlew + run: chmod +x gradlew + + - name: Build with testing + run: ./gradlew check --console rich --info + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: tests + path: ./build/reports/tests/test + + - name: JaCoCo test report + if: success() + run: ./gradlew jacocoTestReport + + - uses: actions/upload-artifact@v4 + if: success() + with: + name: jacoco + path: ./build/reports/coverage diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..b7eb13b --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,31 @@ +--- +name: "CodeQL" + +on: + schedule: + - cron: '31 2 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [java] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/gem-push.yml b/.github/workflows/gem-push.yml new file mode 100644 index 0000000..d51f9e8 --- /dev/null +++ b/.github/workflows/gem-push.yml @@ -0,0 +1,27 @@ +name: Ruby Gem + +on: + workflow_dispatch: + push: + tags: + - '*' + +jobs: + build: + name: Build + Publish + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v4 + - name: Set up Ruby 2.7 + uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: push gem + uses: trocco-io/push-gem-to-gpr-action@v2 + with: + language: java + gem-path: "./build/gems/*.gem" + github-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 83d575c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ -language: java - -jdk: oraclejdk8 - -dist: trusty - -env: - global: - secure: "NCkU3f60cn5Gmzq+NwDmKdQSq+ApF/rPqSHPirk1+ZPW9n+H9912meUuJr9qyn03MPdc6wAQinsl3skdPNh1Qz02TI/WZGMiCQTLh3nKIMzaaxx6I+OZ1TV7A4MoJMQ7oYQdXdbvRYXpOVRnfVDxpUC+BDk2T8xmjfm9cht+PGTzM6RxpmSL842hiudkcyxHxgEf66EGpi6h2G2PqYNfQrksqiVMWlLiTO5a2ee+iw9qTMhgKrhEKLCb+dFB56gtKIOpd3MxieLwf3PDcsjM+TSaWo6Bp+mjdA3zGJNz5wbWRVSdrIvKvxmf3eRqAGssBhBDO/LUN3LnI+/T/7J9XZZd7WXNudo/t6HQXhJkUIvOucCxSg8/uRsxrMAmF2zkJOnYGSvN3HNqdzyz86TK4/Xckl+gX51b0uYncHfZYSojn4py1RWPYpX/L2RHbzMn1MJ7B3/VZ21vY97aHhDgNyyAMxPnTAuR1IZicEgA8y8AmZkkK9tWq5wvAOKI91V+W8iBEaO2dZIib+KdJSUUj/Q3TM3i4wX9O/AU7tknsnCApvxQeveELJ4SxfFd/Z2imqmQWM0wIgp1u7Vjguf8kSaxcuQHonZNdwvvYOkbYxHvUqltzlmE1qFC/kQe6o9fS3NuiugHCYVeFs+LXVVW+nl0pWztFYMicUmQq7T781w=" - -script: - - ./gradlew gem - - ./gradlew checkstyle test jacocoTestReport diff --git a/.travis.yml.erb b/.travis.yml.erb deleted file mode 100644 index f9de8bc..0000000 --- a/.travis.yml.erb +++ /dev/null @@ -1,37 +0,0 @@ -language: ruby - -jdk: oraclejdk8 - -env: - global: - secure: "NCkU3f60cn5Gmzq+NwDmKdQSq+ApF/rPqSHPirk1+ZPW9n+H9912meUuJr9qyn03MPdc6wAQinsl3skdPNh1Qz02TI/WZGMiCQTLh3nKIMzaaxx6I+OZ1TV7A4MoJMQ7oYQdXdbvRYXpOVRnfVDxpUC+BDk2T8xmjfm9cht+PGTzM6RxpmSL842hiudkcyxHxgEf66EGpi6h2G2PqYNfQrksqiVMWlLiTO5a2ee+iw9qTMhgKrhEKLCb+dFB56gtKIOpd3MxieLwf3PDcsjM+TSaWo6Bp+mjdA3zGJNz5wbWRVSdrIvKvxmf3eRqAGssBhBDO/LUN3LnI+/T/7J9XZZd7WXNudo/t6HQXhJkUIvOucCxSg8/uRsxrMAmF2zkJOnYGSvN3HNqdzyz86TK4/Xckl+gX51b0uYncHfZYSojn4py1RWPYpX/L2RHbzMn1MJ7B3/VZ21vY97aHhDgNyyAMxPnTAuR1IZicEgA8y8AmZkkK9tWq5wvAOKI91V+W8iBEaO2dZIib+KdJSUUj/Q3TM3i4wX9O/AU7tknsnCApvxQeveELJ4SxfFd/Z2imqmQWM0wIgp1u7Vjguf8kSaxcuQHonZNdwvvYOkbYxHvUqltzlmE1qFC/kQe6o9fS3NuiugHCYVeFs+LXVVW+nl0pWztFYMicUmQq7T781w=" - -before_install: - - | - ruby -v - # Currently, Travis can't treat jruby 9.0.5.0 - rvm get head - rvm use jruby-9.0.5.0 --install - ruby -v - - gem i bundler - -rvm: jruby-9.0.5.0 - -gemfile: -<% versions.each do |file| -%> - - gemfiles/<%= file %> -<% end -%> - -matrix: - exclude: - - jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build - include: - <% matrix.each do |m| -%> -<%= m %> - <% end %> - - allow_failures: - # Ignore failure for *-latest - <% versions.find_all{|file| file.to_s.match(/-latest/)}.each do |file| -%> -- gemfile: <%= file %> - <% end %> diff --git a/CHANGELOG.md b/CHANGELOG.md index 8203311..cfc819f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,30 @@ +## 0.3.1 - 2023-05-19 +* [enhancement] Update library, minor code refactoring +* PR [#78](https://github.com/treasure-data/embulk-input-jira/pull/78) + +## 0.2.15 - 2022-05-23 +* [enhancement] Catchup with embulk v0.10.32 +* PR [#71](https://github.com/treasure-data/embulk-input-jira/pull/71) + +## 0.2.14 - 2021-10-12 +* [enhancement] Graceful exit in case no data of dynamic schema +* PR [#70](https://github.com/treasure-data/embulk-input-jira/pull/70) + +## 0.2.13 - 2021-09-30 +* [enhancement] Support dynamic schema +* PR [#69](https://github.com/treasure-data/embulk-input-jira/pull/69) + +## 0.2.12 - 2021-08-24 +* [enhancement] Mordernized with embulk v0.10.x styles +* [enhancement] Use embulk-guess-util +* [enhancement] Remove internal jruby guess plugin + +## 0.2.11 - 2020-11-12 + +* [enhancement] Apply Embulk Gradle plugin. +* [enhancement] Apply embulk-util-config for config loading. +* [enhancement] Upgrade Embulk to v0.10.19, along with necessary refactors. + ## 0.2.10 - 2020-03-18 * [enhancement] Use `java.util.Optional` instead of `com.google.common.base.Optional`, use `LoggerFactory.getLogger` instead of `Exec.getLogger` [#58](https://github.com/treasure-data/embulk-input-jira/pull/58) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 43acdd5..0000000 --- a/LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index 7045acd..d86898c 100644 --- a/README.md +++ b/README.md @@ -9,47 +9,57 @@ embulk-input-jira is the Embulk input plugin for [JIRA](https://www.atlassian.co ## Overview -Required Embulk version >= 0.9.20 +Required Embulk version >= 0.10.19 -- **Plugin type**: input -- **Resume supported**: no -- **Cleanup supported**: no -- **Guess supported**: yes +* **Plugin type**: input +* **Resume supported**: no +* **Cleanup supported**: no +* **Guess supported**: yes ## Configuration - **Since JIRA is going to deprecate the basic authentication with passwords and cookie-based authentication to their APIs, we highly recommend you to use email and API key to authenticate to JIRA APIs. [Deprecated notice](https://developer.atlassian.com/cloud/jira/platform/deprecation-notice-basic-auth-and-cookie-based-auth/)** -- **username** JIRA username or email (string, required) -- **password** JIRA password or API keys (string, required) -- **uri** JIRA API endpoint (string, required) -- **jql** [JQL](https://confluence.atlassian.com/display/JIRA/Advanced+Searching) for extract target issues (string, required) -- **expand** Use [expand](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/#api-rest-api-3-search-get) to include additional information about issues in the response (array, optional) -- **columns** target issue attributes. You can generate this configuration by `guess` command (array, required) -- **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 1) -- **retry_limit**: Try to retry this times (integer, default: 5) -- **max_results**: The maximum number of items to return per page (integer, default: 50) -- **expand_json_on_guess** The boolean value is to enable/disable json expanding when `guess`. (boolean, default: true) +- **username** JIRA username or email (string, required) +- **password** JIRA password or API keys (string, required) +- **uri** JIRA API endpoint (string, required) +- **jql** [JQL](https://confluence.atlassian.com/display/JIRA/Advanced+Searching) for extract target issues (string, required) +- **expand** Use [expand](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/#api-rest-api-3-search-get) to include additional information about issues in the response (array, optional) +- **dynamic_schema** Used it to refresh the schema each time ingestion (boolean, default: `false`) +- **columns** target issue attributes. You can generate this configuration by `guess` command (array, required) +- **retry_initial_wait_sec**: Wait seconds for exponential backoff initial value (integer, default: 1) +- **retry_limit**: Try to retry this times (integer, default: 5) +- **max_results**: The maximum number of items to return per page (integer, default: 50) +- **expand_json_on_guess** The boolean value is to enable/disable json expanding when `guess`. (boolean, default: true) ## Example ```yaml in: - type: jira - username: USERNAME - password: PASSWORD - uri: http://localhost:8090 - jql: project = PRO AND summary~Fix - columns: - - { name: id, type: long } - - { name: key, type: string } - - { name: project.name, type: string } - - { name: summary, type: string } - - { name: assignee.name, type: string } + type: jira + username: USERNAME + password: PASSWORD + uri: http://localhost:8090 + jql: project = PRO AND summary~Fix + columns: + - {name: id, type: long} + - {name: key, type: string} + - {name: project.name, type: string} + - {name: summary, type: string} + - {name: assignee.name, type: string} ``` ## Build ``` -$ ./gradlew gem # -t to watch change of files and rebuild continuously +$ ./gradlew checkstyle test jacocoTestReport; ./gradlew gem; +``` + +## Build and Test With Local File +``` +$ rm -rf build; ./gradlew gem; embulk guess -L ./build/gemContents/ {path_to_yaml_file} +``` + +## Publish +``` +$ rm -rf build; ./gradlew gem; ./gradlew publishMavenPublicationToMavenCentralRepository ``` diff --git a/build.gradle b/build.gradle index e9f031f..e33afda 100644 --- a/build.gradle +++ b/build.gradle @@ -1,42 +1,91 @@ plugins { - id "com.jfrog.bintray" version "1.1" - id "com.github.jruby-gradle.base" version "1.5.0" id "java" id "checkstyle" id "jacoco" + id "org.embulk.embulk-plugins" version "0.4.2" + id "com.palantir.git-version" version "3.0.0" +// id "maven" +// id 'maven-publish' +// id "signing" } -import com.github.jrubygradle.JRubyExec + repositories { mavenCentral() - jcenter() -} -configurations { - provided } -version = "0.2.12" +group = "trocco-io" +version = { + def vd = versionDetails() + if (vd.commitDistance == 0 && vd.lastTag ==~ /^[0-9]+\.[0-9]+\.[0-9]+([.-][.a-zA-Z0-9-]+)?/) { + vd.lastTag + } else { + "0.0.0.${vd.gitHash}" + } +}() +description = "JIRA Embulk input plugin." sourceCompatibility = 1.8 targetCompatibility = 1.8 +def embulkVersion = '0.10.32' + +tasks.withType(JavaCompile) { + options.compilerArgs << "-Xlint:deprecation" << "-Xlint:unchecked" + options.encoding = "UTF-8" +} + +java { + withJavadocJar() + withSourcesJar() +} + dependencies { - compile "org.embulk:embulk-core:0.9.20" - provided "org.embulk:embulk-core:0.9.20" + compileOnly "org.embulk:embulk-api:$embulkVersion" + compileOnly "org.embulk:embulk-spi:$embulkVersion" + + compile('org.embulk:embulk-util-config:0.3.1') { + exclude group: 'com.fasterxml.jackson.core', module: 'jackson-annotations' + exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core' + exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind' + exclude group: 'com.fasterxml.jackson.datatype', module: 'jackson-datatype-jdk8' + exclude group: 'javax.validation', module: 'validation-api' + } + compile('org.embulk:embulk-util-retryhelper-jetty92:0.8.2') + compile 'org.embulk:embulk-util-timestamp:0.2.1' + compile('org.embulk:embulk-util-guess:0.1.2') + + compile('org.embulk:embulk-util-json:0.1.1') { + exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core' + exclude group: 'org.msgpack', module: 'msgpack-core' + } + + // Explicit dependencies for embulk-util-* that matches with Embulk 0.10.32 + compile 'com.fasterxml.jackson.core:jackson-core:2.6.7' + compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.7' + compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7' + compile 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7' + compile 'javax.validation:validation-api:1.1.0.Final' + compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.6' + compile group: 'org.glassfish.jersey.core', name: 'jersey-client', version: '2.27' + // Java EE dependencies (required on Java 9+) for jersey-client + compile "javax.xml.bind:jaxb-api:2.2.11" + compile "com.sun.xml.bind:jaxb-core:2.2.11" + compile "com.sun.xml.bind:jaxb-impl:2.2.11" + compile "javax.activation:activation:1.1.1" + compile group: 'com.google.code.gson', name: 'gson', version: '2.8.5' - testCompile "junit:junit:4.+" - testCompile "org.mockito:mockito-core:2.+" - testCompile "org.embulk:embulk-test:0.9.20" - testCompile "org.embulk:embulk-core:0.9.20:tests" -} + compile 'com.google.guava:guava:18.0' + compile 'org.apache.commons:commons-lang3:3.4' -task classpath(type: Copy, dependsOn: ["jar"]) { - doFirst { file("classpath").deleteDir() } - from (configurations.runtime - configurations.provided + files(jar.archivePath)) - into "classpath" + testCompile "junit:junit:4.10" + testCompile "org.embulk:embulk-junit4:$embulkVersion" + testCompile "org.embulk:embulk-core:$embulkVersion" + testCompile "org.embulk:embulk-core:$embulkVersion:tests" + testCompile "org.embulk:embulk-deps:$embulkVersion" + testCompile "org.mockito:mockito-core:2.28.2" } -clean { delete "classpath" } checkstyle { configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml") @@ -55,58 +104,126 @@ task checkstyle(type: Checkstyle) { source = sourceSets.main.allJava + sourceSets.test.allJava } -task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) { - jrubyArgs "-S" - script "gem" - scriptArgs "build", "${project.name}.gemspec" - doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") } +embulkPlugin { + mainClass = "org.embulk.input.jira.JiraInputPlugin" + category = "input" + type = "jira" } -task gemPush(type: JRubyExec, dependsOn: ["gem"]) { - jrubyArgs "-S" - script "gem" - scriptArgs "push", "pkg/${project.name}-${project.version}.gem" +javadoc { + options { + locale = "en_US" + encoding = "UTF-8" + } } -task "package"(dependsOn: ["gemspec", "classpath"]) { - doLast { - println "> Build succeeded." - println "> You can run embulk with '-L ${file(".").absolutePath}' argument." +jar { + from rootProject.file("LICENSE") +} + +sourcesJar { + from rootProject.file("LICENSE") +} + +javadocJar { + from rootProject.file("LICENSE") +} + +// It should not publish a `.module` file in Maven Central. +// https://docs.gradle.org/current/userguide/publishing_gradle_module_metadata.html#sub:disabling-gmm-publication +tasks.withType(GenerateModuleMetadata) { + enabled = false +} +/* +publishing { + publications { + maven(MavenPublication) { + groupId = project.group + artifactId = project.name + + from components.java // Must be "components.java". The dependency modification works only for it. + // javadocJar and sourcesJar are added by java.withJavadocJar() and java.withSourcesJar() above. + // See: https://docs.gradle.org/current/javadoc/org/gradle/api/plugins/JavaPluginExtension.html + + pom { // https://central.sonatype.org/pages/requirements.html + packaging "jar" + + name = project.name + description = project.description + url = "https://github.com/treasure-data/embulk-input-jira" + + licenses { + license { + // http://central.sonatype.org/pages/requirements.html#license-information + name = "The Apache License, Version 2.0" + url = "https://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + + developers { + // Authors after reimplementing it in Java. + developer { + name = "Phu Nguyen" + email = "phu@treasure-data.com" + } + developer { + name = "Huy Le" + email = "huy.lenq@gmail.com" + } + developer { + name = "Thanh Le" + email = "legiangthanh@gmail.com" + } + } + + scm { + connection = "scm:git:git://github.com/treasure-data/embulk-input-jira.git" + developerConnection = "scm:git:git@github.com:treasure-data/embulk-input-jira.git" + url = "https://github.com/treasure-data/embulk-input-jira" + } + } + } + } + + repositories { + maven { // publishMavenPublicationToMavenCentralRepository + name = "mavenCentral" + if (project.version.endsWith("-SNAPSHOT")) { + url "https://oss.sonatype.org/content/repositories/snapshots" + } else { + url "https://oss.sonatype.org/service/local/staging/deploy/maven2" + } + + credentials { + username = project.hasProperty("ossrhUsername") ? ossrhUsername : "" + password = project.hasProperty("ossrhPassword") ? ossrhPassword : "" + } + } } } +signing { + sign publishing.publications.maven +} +*/ jacocoTestReport { group = "Reporting" reports { xml.enabled false csv.enabled false - html.destination "${buildDir}/reports/coverage" + html.destination file("${buildDir}/reports/coverage") } } -task gemspec { - ext.gemspecFile = file("${project.name}.gemspec") - inputs.file "build.gradle" - outputs.file gemspecFile - doLast { gemspecFile.write($/ -Gem::Specification.new do |spec| - spec.name = "${project.name}" - spec.version = "${project.version}" - spec.authors = ["uu59", "yoshihara"] - spec.summary = %[Jira input plugin for Embulk] - spec.description = %[Loads records from Jira.] - spec.email = ["k@uu59.org", "h.yoshihara@everyleaf.com"] - spec.licenses = ["Apache-2.0"] - spec.homepage = "https://github.com/treasure-data/embulk-input-jira" - - spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"] - spec.test_files = spec.files.grep(%r"^(test|spec)/") - spec.require_paths = ["lib"] - - spec.add_development_dependency 'bundler', ['~> 1.0'] - spec.add_development_dependency 'rake', ['~> 12.0'] -end -/$) - } +gem { + authors = ['uu59', 'yoshihara'] + email = ['k@uu59.org', 'h.yoshihara@everyleaf.com'] + summary = project.description + homepage = 'https://github.com/treasure-data/embulk-input-jira' + licenses = ['Apache-2.0'] +} +/* +gemPush { + host = "https://rubygems.org" } -clean { delete "${project.name}.gemspec" } +*/ diff --git a/gradle/dependency-locks/embulkPluginRuntime.lockfile b/gradle/dependency-locks/embulkPluginRuntime.lockfile new file mode 100644 index 0000000..b9b65dd --- /dev/null +++ b/gradle/dependency-locks/embulkPluginRuntime.lockfile @@ -0,0 +1,39 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.fasterxml.jackson.core:jackson-annotations:2.6.7 +com.fasterxml.jackson.core:jackson-core:2.6.7 +com.fasterxml.jackson.core:jackson-databind:2.6.7 +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7 +com.google.code.gson:gson:2.8.5 +com.google.guava:guava:18.0 +com.ibm.icu:icu4j:54.1.1 +com.sun.xml.bind:jaxb-core:2.2.11 +com.sun.xml.bind:jaxb-impl:2.2.11 +commons-codec:commons-codec:1.10 +commons-logging:commons-logging:1.2 +javax.activation:activation:1.1.1 +javax.annotation:javax.annotation-api:1.2 +javax.validation:validation-api:1.1.0.Final +javax.ws.rs:javax.ws.rs-api:2.1 +javax.xml.bind:jaxb-api:2.2.11 +org.apache.commons:commons-lang3:3.4 +org.apache.httpcomponents:httpclient:4.5.6 +org.apache.httpcomponents:httpcore:4.4.10 +org.eclipse.jetty:jetty-client:9.2.14.v20151106 +org.eclipse.jetty:jetty-http:9.2.14.v20151106 +org.eclipse.jetty:jetty-io:9.2.14.v20151106 +org.eclipse.jetty:jetty-util:9.2.14.v20151106 +org.embulk:embulk-util-config:0.3.1 +org.embulk:embulk-util-file:0.1.1 +org.embulk:embulk-util-guess:0.1.2 +org.embulk:embulk-util-json:0.1.1 +org.embulk:embulk-util-retryhelper-jetty92:0.8.2 +org.embulk:embulk-util-retryhelper:0.8.2 +org.embulk:embulk-util-rubytime:0.3.2 +org.embulk:embulk-util-text:0.1.0 +org.embulk:embulk-util-timestamp:0.2.1 +org.glassfish.hk2.external:javax.inject:2.5.0-b42 +org.glassfish.hk2:osgi-resource-locator:1.0.1 +org.glassfish.jersey.core:jersey-client:2.27 +org.glassfish.jersey.core:jersey-common:2.27 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7a3265e..5c2d1cf 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index f16d266..bb8b2fc 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip diff --git a/gradlew b/gradlew index cccdd3d..8e25e6c 100755 --- a/gradlew +++ b/gradlew @@ -1,5 +1,21 @@ #!/usr/bin/env sh +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + ############################################################################## ## ## Gradle start up script for UN*X @@ -28,7 +44,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" diff --git a/gradlew.bat b/gradlew.bat index e95643d..24467a1 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,3 +1,19 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + @if "%DEBUG%" == "" @echo off @rem ########################################################################## @rem @@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome diff --git a/lib/embulk/guess/jira.rb b/lib/embulk/guess/jira.rb deleted file mode 100644 index 449720a..0000000 --- a/lib/embulk/guess/jira.rb +++ /dev/null @@ -1,24 +0,0 @@ -# Not to be used as a standalone guess plugin -# This is just a thin wrapper to leverage the SchemaGuess Ruby implementation from Java side - -require 'json' - -module Embulk - module Guess - class JiraGuess < TextGuessPlugin - Plugin.register_guess("jira", self) - - def guess_text(config, sample_text) - {:columns => - SchemaGuess.from_hash_records(JSON.parse(sample_text)).map do |c| - { - name: c.name, - type: c.type, - **(c.format ? {format: c.format} : {}) - } - end - } - end - end - end -end \ No newline at end of file diff --git a/lib/embulk/input/jira.rb b/lib/embulk/input/jira.rb deleted file mode 100644 index 47555c7..0000000 --- a/lib/embulk/input/jira.rb +++ /dev/null @@ -1,3 +0,0 @@ -Embulk::JavaPlugin.register_input( - "jira", "org.embulk.input.jira.JiraInputPlugin", - File.expand_path('../../../../classpath', __FILE__)) diff --git a/src/main/java/org/embulk/input/jira/Issue.java b/src/main/java/org/embulk/input/jira/Issue.java index 659351e..38a7ae2 100644 --- a/src/main/java/org/embulk/input/jira/Issue.java +++ b/src/main/java/org/embulk/input/jira/Issue.java @@ -18,20 +18,20 @@ public class Issue { private JsonObject flatten; - private JsonObject json; + private final JsonObject json; - public Issue(JsonObject original) + public Issue(final JsonObject original) { this.json = original; } - public JsonElement getValue(String path) + public JsonElement getValue(final String path) { - List keys = new ArrayList<>(Arrays.asList(path.split("\\."))); + final List keys = new ArrayList<>(Arrays.asList(path.split("\\."))); return get(json, keys); } - private JsonElement get(JsonElement json, List keys) + private JsonElement get(final JsonElement json, final List keys) { if (json == null || json.isJsonNull()) { return JsonNull.INSTANCE; @@ -39,11 +39,11 @@ private JsonElement get(JsonElement json, List keys) else if (keys.isEmpty() || (json.isJsonArray() && json.getAsJsonArray().size() == 0)) { return json; } - String key = keys.get(0); + final String key = keys.get(0); keys.remove(0); if (json.isJsonArray()) { - JsonArray arrays = new JsonArray(); - for (JsonElement elem : json.getAsJsonArray()) { + final JsonArray arrays = new JsonArray(); + for (final JsonElement elem : json.getAsJsonArray()) { if (elem.isJsonObject()) { arrays.add(elem.getAsJsonObject().get(key)); } @@ -53,12 +53,10 @@ else if (keys.isEmpty() || (json.isJsonArray() && json.getAsJsonArray().size() = } return get(arrays, keys); } - else { - return get(json.getAsJsonObject().get(key), keys); - } + return get(json.getAsJsonObject().get(key), keys); } - public synchronized JsonObject getFlatten(Boolean expandJsonOnGuess) + public synchronized JsonObject getFlatten(final boolean expandJsonOnGuess) { if (flatten == null) { flatten = new JsonObject(); @@ -67,10 +65,10 @@ public synchronized JsonObject getFlatten(Boolean expandJsonOnGuess) return flatten; } - private void manipulatingFlattenJson(JsonElement in, String prefix, boolean expandJsonOnGuess) + private void manipulatingFlattenJson(final JsonElement in, final String prefix, final boolean expandJsonOnGuess) { if (in.isJsonObject()) { - JsonObject obj = in.getAsJsonObject(); + final JsonObject obj = in.getAsJsonObject(); // NOTE: If you want to flatten JSON completely, please remove this if and addHeuristicValue if (StringUtils.countMatches(prefix, ".") > 1) { addHeuristicValue(obj, prefix); @@ -80,9 +78,9 @@ private void manipulatingFlattenJson(JsonElement in, String prefix, boolean expa flatten.add(prefix, obj); } else { - for (Entry entry : obj.entrySet()) { - String key = entry.getKey(); - JsonElement value = entry.getValue(); + for (final Entry entry : obj.entrySet()) { + final String key = entry.getKey(); + final JsonElement value = entry.getValue(); if (expandJsonOnGuess) { manipulatingFlattenJson(value, appendPrefix(prefix, key), expandJsonOnGuess); } @@ -93,21 +91,21 @@ private void manipulatingFlattenJson(JsonElement in, String prefix, boolean expa } } else if (in.isJsonArray()) { - JsonArray arrayObj = in.getAsJsonArray(); - boolean isAllJsonObject = arrayObj.size() > 0 && StreamSupport.stream(arrayObj.spliterator(), false).allMatch(JsonElement::isJsonObject); + final JsonArray arrayObj = in.getAsJsonArray(); + final boolean isAllJsonObject = arrayObj.size() > 0 && StreamSupport.stream(arrayObj.spliterator(), false).allMatch(JsonElement::isJsonObject); if (isAllJsonObject) { - Map occurents = new HashMap<>(); - for (JsonElement element : arrayObj) { - JsonObject obj = element.getAsJsonObject(); - for (Entry entry : obj.entrySet()) { - String key = entry.getKey(); + final Map occurents = new HashMap<>(); + for (final JsonElement element : arrayObj) { + final JsonObject obj = element.getAsJsonObject(); + for (final Entry entry : obj.entrySet()) { + final String key = entry.getKey(); occurents.merge(key, 1, Integer::sum); } } - JsonObject newObj = new JsonObject(); - for (String key : occurents.keySet()) { + final JsonObject newObj = new JsonObject(); + for (final String key : occurents.keySet()) { newObj.add(key, new JsonArray()); - for (JsonElement elem : arrayObj) { + for (final JsonElement elem : arrayObj) { newObj.get(key).getAsJsonArray().add(elem.getAsJsonObject().get(key)); } } @@ -126,13 +124,13 @@ else if (in.isJsonPrimitive()) { } } - private void addHeuristicValue(JsonObject json, String prefix) + private void addHeuristicValue(final JsonObject json, final String prefix) { - List keys = Arrays.asList("name", "key", "id"); - List heuristic = new ArrayList<>(); - for (Entry entry : json.entrySet()) { - String key = entry.getKey(); - JsonElement value = entry.getValue(); + final List keys = Arrays.asList("name", "key", "id"); + final List heuristic = new ArrayList<>(); + for (final Entry entry : json.entrySet()) { + final String key = entry.getKey(); + final JsonElement value = entry.getValue(); if (keys.contains(key) && !value.isJsonNull()) { heuristic.add(key); } @@ -141,14 +139,14 @@ private void addHeuristicValue(JsonObject json, String prefix) flatten.add(prefix, new JsonPrimitive(json.toString())); } else { - for (String key : heuristic) { - JsonElement value = json.get(key); + for (final String key : heuristic) { + final JsonElement value = json.get(key); flatten.add(appendPrefix(prefix, key), value); } } } - private String appendPrefix(String prefix, String key) + private String appendPrefix(final String prefix, final String key) { return prefix.isEmpty() ? key : prefix + "." + key; } diff --git a/src/main/java/org/embulk/input/jira/JiraInputPlugin.java b/src/main/java/org/embulk/input/jira/JiraInputPlugin.java index ffb6665..1140836 100644 --- a/src/main/java/org/embulk/input/jira/JiraInputPlugin.java +++ b/src/main/java/org/embulk/input/jira/JiraInputPlugin.java @@ -1,35 +1,35 @@ package org.embulk.input.jira; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonNull; import com.google.gson.JsonObject; - -import org.embulk.config.Config; -import org.embulk.config.ConfigDefault; import org.embulk.config.ConfigDiff; import org.embulk.config.ConfigException; import org.embulk.config.ConfigSource; -import org.embulk.config.Task; import org.embulk.config.TaskReport; import org.embulk.config.TaskSource; -import org.embulk.exec.GuessExecutor; import org.embulk.input.jira.client.JiraClient; import org.embulk.input.jira.util.JiraUtil; -import org.embulk.spi.Buffer; import org.embulk.spi.Exec; import org.embulk.spi.InputPlugin; import org.embulk.spi.PageBuilder; import org.embulk.spi.PageOutput; import org.embulk.spi.Schema; -import org.embulk.spi.SchemaConfig; +import org.embulk.util.config.Config; +import org.embulk.util.config.ConfigDefault; +import org.embulk.util.config.ConfigMapper; +import org.embulk.util.config.ConfigMapperFactory; +import org.embulk.util.config.Task; +import org.embulk.util.config.TaskMapper; +import org.embulk.util.config.units.ColumnConfig; +import org.embulk.util.config.units.SchemaConfig; +import org.embulk.util.guess.SchemaGuess; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map.Entry; import java.util.Optional; @@ -37,7 +37,6 @@ import java.util.SortedSet; import java.util.TreeSet; -import static org.embulk.input.jira.Constant.GUESS_BUFFER_SIZE; import static org.embulk.input.jira.Constant.GUESS_RECORDS_COUNT; import static org.embulk.input.jira.Constant.PREVIEW_RECORDS_COUNT; @@ -45,6 +44,14 @@ public class JiraInputPlugin implements InputPlugin { private static final Logger LOGGER = LoggerFactory.getLogger(JiraInputPlugin.class); + @VisibleForTesting + public static final ConfigMapperFactory CONFIG_MAPPER_FACTORY = ConfigMapperFactory + .builder() + .addDefaultModules() + .build(); + @VisibleForTesting + public static final ConfigMapper CONFIG_MAPPER = CONFIG_MAPPER_FACTORY.createConfigMapper(); + private static final TaskMapper TASK_MAPPER = CONFIG_MAPPER_FACTORY.createTaskMapper(); public interface PluginTask extends Task @@ -86,6 +93,10 @@ public interface PluginTask @ConfigDefault("[]") public List getExpand(); + @Config("dynamic_schema") + @ConfigDefault("false") + public boolean getDynamicSchema(); + @Config("columns") public SchemaConfig getColumns(); @@ -103,12 +114,28 @@ public interface PluginTask public ConfigDiff transaction(final ConfigSource config, final InputPlugin.Control control) { - final PluginTask task = config.loadConfig(PluginTask.class); - - final Schema schema = task.getColumns().toSchema(); + final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class); + SchemaConfig schemaConfig = task.getColumns(); + if (task.getDynamicSchema()) { + final JiraClient jiraClient = getJiraClient(); + final List columns = new ArrayList<>(); + try { + final List guessedColumns = getGuessedColumns(jiraClient, task); + for (final ConfigDiff guessedColumn : guessedColumns) { + columns.add(new ColumnConfig(CONFIG_MAPPER_FACTORY.newConfigSource().merge(guessedColumn))); + } + } + catch (final ConfigException e) { + if (!e.getMessage().equals("Could not guess schema due to empty data set")) { + throw e; + } + } + schemaConfig = new SchemaConfig(columns); + } + final Schema schema = schemaConfig.toSchema(); final int taskCount = 1; - return resume(task.dump(), schema, taskCount, control); + return resume(task.toTaskSource(), schema, taskCount, control); } @Override @@ -117,14 +144,7 @@ public ConfigDiff resume(final TaskSource taskSource, final InputPlugin.Control control) { control.run(taskSource, schema, taskCount); - return Exec.newConfigDiff(); - } - - @Override - public void cleanup(final TaskSource taskSource, - final Schema schema, final int taskCount, - final List successTaskReports) - { + return CONFIG_MAPPER_FACTORY.newConfigDiff(); } @Override @@ -132,7 +152,7 @@ public TaskReport run(final TaskSource taskSource, final Schema schema, final int taskIndex, final PageOutput output) { - final PluginTask task = taskSource.loadTask(PluginTask.class); + final PluginTask task = TASK_MAPPER.map(taskSource, PluginTask.class); JiraUtil.validateTaskConfig(task); final JiraClient jiraClient = getJiraClient(); jiraClient.checkUserCredentials(task); @@ -156,34 +176,31 @@ public TaskReport run(final TaskSource taskSource, } pageBuilder.finish(); } - return Exec.newTaskReport(); + return CONFIG_MAPPER_FACTORY.newTaskReport(); } @Override public ConfigDiff guess(final ConfigSource config) { // Reset columns in case already have or missing on configuration - config.set("columns", new ObjectMapper().createArrayNode()); - final PluginTask task = config.loadConfig(PluginTask.class); + config.set("columns", new ArrayList<>()); + final PluginTask task = CONFIG_MAPPER.map(config, PluginTask.class); JiraUtil.validateTaskConfig(task); final JiraClient jiraClient = getJiraClient(); jiraClient.checkUserCredentials(task); + return CONFIG_MAPPER_FACTORY.newConfigDiff().set("columns", getGuessedColumns(jiraClient, task)); + } + + private List getGuessedColumns(final JiraClient jiraClient, final PluginTask task) + { final List issues = jiraClient.searchIssues(task, 0, GUESS_RECORDS_COUNT); if (issues.isEmpty()) { throw new ConfigException("Could not guess schema due to empty data set"); } - final Buffer sample = Buffer.copyOf(createSamples(issues, getUniqueAttributes(issues, task.getExpandJsonOnGuess()), task.getExpandJsonOnGuess()).toString().getBytes()); - final JsonNode columns = Exec.getInjector().getInstance(GuessExecutor.class) - .guessParserConfig(sample, Exec.newConfigSource(), createGuessConfig()) - .get(JsonNode.class, "columns"); - return Exec.newConfigDiff().set("columns", columns); - } - - private ConfigSource createGuessConfig() - { - return Exec.newConfigSource() - .set("guess_plugins", ImmutableList.of("jira")) - .set("guess_sample_buffer_bytes", GUESS_BUFFER_SIZE); + final boolean expandJsonOnGuess = task.getExpandJsonOnGuess(); + final List columns = SchemaGuess.of(CONFIG_MAPPER_FACTORY).fromLinkedHashMapRecords(createGuessSample(issues, getUniqueAttributes(issues, expandJsonOnGuess), expandJsonOnGuess)); + columns.forEach(conf -> conf.remove("index")); + return columns; } private SortedSet getUniqueAttributes(final List issues, final boolean expandJsonOnGuess) @@ -197,9 +214,9 @@ private SortedSet getUniqueAttributes(final List issues, final bo return uniqueAttributes; } - private JsonArray createSamples(final List issues, final Set uniqueAttributes, final boolean expandJsonOnGuess) + private List> createGuessSample(final List issues, final Set uniqueAttributes, final boolean expandJsonOnGuess) { - final JsonArray samples = new JsonArray(); + final List> samples = new ArrayList<>(); for (final Issue issue : issues) { final JsonObject flatten = issue.getFlatten(expandJsonOnGuess); final JsonObject unified = new JsonObject(); @@ -210,21 +227,16 @@ private JsonArray createSamples(final List issues, final Set uniq } unified.add(key, value); } - samples.add(unified); + samples.add(JiraUtil.toLinkedHashMap(unified)); } return samples; } @VisibleForTesting - public GuessExecutor getGuessExecutor() - { - return Exec.getInjector().getInstance(GuessExecutor.class); - } - - @VisibleForTesting + @SuppressWarnings("deprecation") // TODO: For compatibility with Embulk v0.9 public PageBuilder getPageBuilder(final Schema schema, final PageOutput output) { - return new PageBuilder(Exec.getBufferAllocator(), schema, output); + return new PageBuilder(Exec.getBufferAllocator(), schema, output); // TODO: Use Exec#getPageBuilder } @VisibleForTesting @@ -238,4 +250,7 @@ public JiraClient getJiraClient() { return new JiraClient(); } + @Override + public void cleanup(final TaskSource taskSource, final Schema schema, final int taskCount, final List successTaskReports) + {} } diff --git a/src/main/java/org/embulk/input/jira/client/JiraClient.java b/src/main/java/org/embulk/input/jira/client/JiraClient.java index 66aea8f..09391ee 100644 --- a/src/main/java/org/embulk/input/jira/client/JiraClient.java +++ b/src/main/java/org/embulk/input/jira/client/JiraClient.java @@ -6,7 +6,6 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; - import org.apache.http.HttpStatus; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; @@ -23,8 +22,9 @@ import org.embulk.input.jira.JiraInputPlugin.PluginTask; import org.embulk.input.jira.util.JiraException; import org.embulk.input.jira.util.JiraUtil; -import org.embulk.spi.util.RetryExecutor.RetryGiveupException; -import org.embulk.spi.util.RetryExecutor.Retryable; +import org.embulk.util.retryhelper.RetryExecutor; +import org.embulk.util.retryhelper.RetryGiveupException; +import org.embulk.util.retryhelper.Retryable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +43,6 @@ import static org.apache.http.HttpHeaders.CONTENT_TYPE; import static org.embulk.input.jira.Constant.HTTP_TIMEOUT; import static org.embulk.input.jira.Constant.MIN_RESULTS; -import static org.embulk.spi.util.RetryExecutor.retryExecutor; public class JiraClient { @@ -61,9 +60,7 @@ public void checkUserCredentials(final PluginTask task) if (e.getStatusCode() == HttpStatus.SC_UNAUTHORIZED) { throw new ConfigException("Could not authorize with your credential."); } - else { - throw new ConfigException("Could not authorize with your credential due to problems when contacting JIRA API."); - } + throw new ConfigException("Could not authorize with your credential due to problems when contacting JIRA API."); } } @@ -94,11 +91,13 @@ public int getTotalCount(final PluginTask task) private String searchJiraAPI(final PluginTask task, final int startAt, final int maxResults) { try { - return retryExecutor().withRetryLimit(task.getRetryLimit()) - .withInitialRetryWait(task.getInitialRetryIntervalMillis()) - .withMaxRetryWait(task.getMaximumRetryIntervalMillis()) - .runInterruptible(new Retryable() - { + return RetryExecutor.builder() + .withRetryLimit(task.getRetryLimit()) + .withInitialRetryWaitMillis(task.getInitialRetryIntervalMillis()) + .withMaxRetryWaitMillis(task.getMaximumRetryIntervalMillis()) + .build() + .runInterruptible(new Retryable() + { @Override public String call() throws Exception { diff --git a/src/main/java/org/embulk/input/jira/util/JiraUtil.java b/src/main/java/org/embulk/input/jira/util/JiraUtil.java index d560599..f3ab47a 100644 --- a/src/main/java/org/embulk/input/jira/util/JiraUtil.java +++ b/src/main/java/org/embulk/input/jira/util/JiraUtil.java @@ -1,6 +1,7 @@ package org.embulk.input.jira.util; import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; @@ -13,17 +14,18 @@ import org.embulk.input.jira.Issue; import org.embulk.input.jira.JiraInputPlugin.PluginTask; import org.embulk.spi.Column; -import org.embulk.spi.ColumnConfig; import org.embulk.spi.ColumnVisitor; import org.embulk.spi.PageBuilder; import org.embulk.spi.Schema; -import org.embulk.spi.json.JsonParser; -import org.embulk.spi.time.Timestamp; -import org.embulk.spi.time.TimestampParser; +import org.embulk.util.config.units.ColumnConfig; +import org.embulk.util.json.JsonParser; +import org.embulk.util.timestamp.TimestampFormatter; import javax.ws.rs.core.UriBuilder; import java.io.IOException; +import java.time.Instant; +import java.util.LinkedHashMap; import java.util.List; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -97,7 +99,7 @@ public static void validateTaskConfig(final PluginTask task) * For getting the timestamp value of the node * Sometime if the parser could not parse the value then return null * */ - private static Timestamp getTimestampValue(final PluginTask task, final Column column, final String value) + private static Instant getTimestampValue(final PluginTask task, final Column column, final String value) { final List columnConfigs = task.getColumns().getColumns(); String pattern = DEFAULT_TIMESTAMP_PATTERN; @@ -110,14 +112,16 @@ private static Timestamp getTimestampValue(final PluginTask task, final Column c break; } } - final TimestampParser parser = TimestampParser.of(pattern, "UTC"); - Timestamp result = null; + final TimestampFormatter formatter = TimestampFormatter + .builder(pattern, true) + .setDefaultZoneFromString("UTC") + .build(); try { - result = parser.parse(value); + return formatter.parse(value); } catch (final Exception e) { + return null; } - return result; } /* @@ -126,13 +130,12 @@ private static Timestamp getTimestampValue(final PluginTask task, final Column c * */ private static Long getLongValue(final JsonElement value) { - Long result = null; try { - result = value.getAsLong(); + return value.getAsLong(); } catch (final Exception e) { + return null; } - return result; } /* @@ -141,13 +144,12 @@ private static Long getLongValue(final JsonElement value) * */ private static Double getDoubleValue(final JsonElement value) { - Double result = null; try { - result = value.getAsDouble(); + return value.getAsDouble(); } catch (final Exception e) { + return null; } - return result; } /* @@ -156,13 +158,12 @@ private static Double getDoubleValue(final JsonElement value) * */ private static Boolean getBooleanValue(final JsonElement value) { - Boolean result = null; try { - result = value.getAsBoolean(); + return value.getAsBoolean(); } catch (final Exception e) { + return null; } - return result; } public static void addRecord(final Issue issue, final Schema schema, final PluginTask task, final PageBuilder pageBuilder) @@ -196,9 +197,7 @@ else if (data.isJsonArray()) { if (obj.isJsonPrimitive()) { return obj.getAsString(); } - else { - return obj.toString(); - } + return obj.toString(); }) .collect(Collectors.joining(","))); } @@ -208,6 +207,7 @@ else if (data.isJsonArray()) { } @Override + @SuppressWarnings("deprecation") // TODO: For compatibility with Embulk v0.9 public void timestampColumn(final Column column) { final JsonElement data = issue.getValue(column.getName()); @@ -215,12 +215,13 @@ public void timestampColumn(final Column column) pageBuilder.setNull(column); } else { - final Timestamp value = getTimestampValue(task, column, data.getAsString()); + final Instant value = getTimestampValue(task, column, data.getAsString()); if (value == null) { pageBuilder.setNull(column); } else { - pageBuilder.setTimestamp(column, value); + // TODO: Use Instant instead of Timestamp + pageBuilder.setTimestamp(column, org.embulk.spi.time.Timestamp.ofInstant(value)); } } } @@ -263,4 +264,19 @@ public void doubleColumn(final Column column) }); pageBuilder.addRecord(); } + + public static LinkedHashMap toLinkedHashMap(final JsonObject flt) + { + final LinkedHashMap result = new LinkedHashMap<>(); + for (final String key : flt.keySet()) { + final JsonElement elem = flt.get(key); + if (elem.isJsonPrimitive()) { + result.put(key, flt.get(key).getAsString()); + } + else { + result.put(key, elem); + } + } + return result; + } } diff --git a/src/test/java/org/embulk/input/jira/JiraInputPluginTest.java b/src/test/java/org/embulk/input/jira/JiraInputPluginTest.java index be54845..6cfbe5c 100644 --- a/src/test/java/org/embulk/input/jira/JiraInputPluginTest.java +++ b/src/test/java/org/embulk/input/jira/JiraInputPluginTest.java @@ -8,6 +8,7 @@ import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; +import org.embulk.EmbulkTestRuntime; import org.embulk.config.ConfigDiff; import org.embulk.config.ConfigSource; import org.embulk.config.TaskReport; @@ -27,6 +28,7 @@ import java.util.ArrayList; import java.util.List; +import static org.embulk.input.jira.JiraInputPlugin.CONFIG_MAPPER; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.times; @@ -36,17 +38,17 @@ public class JiraInputPluginTest { @Rule - public JiraPluginTestRuntime runtime = new JiraPluginTestRuntime(); + public EmbulkTestRuntime runtime = new EmbulkTestRuntime(); private JiraInputPlugin plugin; private JiraClient jiraClient; private JsonObject data; private ConfigSource config; - private CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); - private CloseableHttpResponse response = Mockito.mock(CloseableHttpResponse.class); - private StatusLine statusLine = Mockito.mock(StatusLine.class); + private final CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); + private final CloseableHttpResponse response = Mockito.mock(CloseableHttpResponse.class); + private final StatusLine statusLine = Mockito.mock(StatusLine.class); - private MockPageOutput output = new MockPageOutput(); + private final MockPageOutput output = new MockPageOutput(); private PageBuilder pageBuilder; @Before @@ -57,9 +59,8 @@ public void setUp() throws IOException jiraClient = Mockito.spy(new JiraClient()); data = TestHelpers.getJsonFromFile("jira_input_plugin.json"); config = TestHelpers.config(); - config.loadConfig(PluginTask.class); + CONFIG_MAPPER.map(config, PluginTask.class); pageBuilder = Mockito.mock(PageBuilder.class); - //pageBuilder = new PageBuilder(Exec.getBufferAllocator(), config.loadConfig(PluginTask.class).getColumns().toSchema(), output); } when(plugin.getJiraClient()).thenReturn(jiraClient); when(jiraClient.createHttpClient()).thenReturn(client); @@ -71,35 +72,69 @@ public void setUp() throws IOException @Test public void test_run_withEmptyResult() throws IOException { - JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); - JsonObject searchResponse = data.get("emptyResult").getAsJsonObject(); + final JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); + final JsonObject searchResponse = data.get("emptyResult").getAsJsonObject(); when(statusLine.getStatusCode()) - .thenReturn(authorizeResponse.get("statusCode").getAsInt()) - .thenReturn(searchResponse.get("statusCode").getAsInt()); + .thenReturn(authorizeResponse.get("statusCode").getAsInt()) + .thenReturn(searchResponse.get("statusCode").getAsInt()); when(response.getEntity()) - .thenReturn(new StringEntity(authorizeResponse.get("body").toString())) - .thenReturn(new StringEntity(searchResponse.get("body").toString())); + .thenReturn(new StringEntity(authorizeResponse.get("body").toString())) + .thenReturn(new StringEntity(searchResponse.get("body").toString())); plugin.transaction(config, new Control()); - // Check credential 1 + getTotal 1 + loadData 0 + // Check credential 1 + getTotal 1 + loadData 0 verify(jiraClient, times(2)).createHttpClient(); verify(pageBuilder, times(0)).addRecord(); verify(pageBuilder, times(1)).finish(); } + public void test_runDynamicSchema_withEmptyResult() throws IOException + { + final JsonObject searchResponse = data.get("emptyResult").getAsJsonObject(); + + when(statusLine.getStatusCode()) + .thenReturn(searchResponse.get("statusCode").getAsInt()); + when(response.getEntity()) + .thenReturn(new StringEntity(searchResponse.get("body").toString())); + plugin.transaction(TestHelpers.dynamicSchemaConfig(), new Control()); + verify(pageBuilder, times(0)).addRecord(); + } + + @Test + public void test_runDynamicSchema_withResult() throws IOException + { + final JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); + final JsonObject searchResponse = data.get("oneRecordResult").getAsJsonObject(); + + when(statusLine.getStatusCode()) + .thenReturn(searchResponse.get("statusCode").getAsInt()) + .thenReturn(authorizeResponse.get("statusCode").getAsInt()) + .thenReturn(searchResponse.get("statusCode").getAsInt()); + when(response.getEntity()) + .thenReturn(new StringEntity(searchResponse.get("body").toString())) + .thenReturn(new StringEntity(authorizeResponse.get("body").toString())) + .thenReturn(new StringEntity(searchResponse.get("body").toString())); + + plugin.transaction(TestHelpers.dynamicSchemaConfig(), new Control()); + // Check credential 1 + getTotal 1 + loadData 2 + verify(jiraClient, times(4)).createHttpClient(); + verify(pageBuilder, times(1)).addRecord(); + verify(pageBuilder, times(1)).finish(); + } + @Test public void test_run_with1RecordsResult() throws IOException { - JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); - JsonObject searchResponse = data.get("oneRecordResult").getAsJsonObject(); + final JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); + final JsonObject searchResponse = data.get("oneRecordResult").getAsJsonObject(); when(statusLine.getStatusCode()) - .thenReturn(authorizeResponse.get("statusCode").getAsInt()) - .thenReturn(searchResponse.get("statusCode").getAsInt()); + .thenReturn(authorizeResponse.get("statusCode").getAsInt()) + .thenReturn(searchResponse.get("statusCode").getAsInt()); when(response.getEntity()) - .thenReturn(new StringEntity(authorizeResponse.get("body").toString())) - .thenReturn(new StringEntity(searchResponse.get("body").toString())); + .thenReturn(new StringEntity(authorizeResponse.get("body").toString())) + .thenReturn(new StringEntity(searchResponse.get("body").toString())); plugin.transaction(config, new Control()); // Check credential 1 + getTotal 1 + loadData 1 @@ -111,15 +146,15 @@ public void test_run_with1RecordsResult() throws IOException @Test public void test_run_with2PagesResult() throws IOException { - JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); - JsonObject searchResponse = data.get("2PagesResult").getAsJsonObject(); + final JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); + final JsonObject searchResponse = data.get("2PagesResult").getAsJsonObject(); when(statusLine.getStatusCode()) - .thenReturn(authorizeResponse.get("statusCode").getAsInt()) - .thenReturn(searchResponse.get("statusCode").getAsInt()); + .thenReturn(authorizeResponse.get("statusCode").getAsInt()) + .thenReturn(searchResponse.get("statusCode").getAsInt()); when(response.getEntity()) - .thenReturn(new StringEntity(authorizeResponse.get("body").toString())) - .thenReturn(new StringEntity(searchResponse.get("body").toString())); + .thenReturn(new StringEntity(authorizeResponse.get("body").toString())) + .thenReturn(new StringEntity(searchResponse.get("body").toString())); plugin.transaction(config, new Control()); // Check credential 1 + getTotal 1 + loadData 2 @@ -132,14 +167,14 @@ public void test_run_with2PagesResult() throws IOException public void test_preview_withEmptyResult() throws IOException { when(plugin.isPreview()).thenReturn(true); - JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); - JsonObject searchResponse = data.get("emptyResult").getAsJsonObject(); + final JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); + final JsonObject searchResponse = data.get("emptyResult").getAsJsonObject(); when(statusLine.getStatusCode()) - .thenReturn(authorizeResponse.get("statusCode").getAsInt()) - .thenReturn(searchResponse.get("statusCode").getAsInt()); + .thenReturn(authorizeResponse.get("statusCode").getAsInt()) + .thenReturn(searchResponse.get("statusCode").getAsInt()); when(response.getEntity()) - .thenReturn(new StringEntity(searchResponse.get("body").toString())); + .thenReturn(new StringEntity(searchResponse.get("body").toString())); plugin.transaction(config, new Control()); // Check credential 1 + loadData 1 @@ -152,15 +187,15 @@ public void test_preview_withEmptyResult() throws IOException public void test_preview_with1RecordsResult() throws IOException { when(plugin.isPreview()).thenReturn(true); - JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); - JsonObject searchResponse = data.get("oneRecordResult").getAsJsonObject(); + final JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); + final JsonObject searchResponse = data.get("oneRecordResult").getAsJsonObject(); when(statusLine.getStatusCode()) - .thenReturn(authorizeResponse.get("statusCode").getAsInt()) - .thenReturn(searchResponse.get("statusCode").getAsInt()); + .thenReturn(authorizeResponse.get("statusCode").getAsInt()) + .thenReturn(searchResponse.get("statusCode").getAsInt()); when(response.getEntity()) - .thenReturn(new StringEntity(authorizeResponse.get("body").toString())) - .thenReturn(new StringEntity(searchResponse.get("body").toString())); + .thenReturn(new StringEntity(authorizeResponse.get("body").toString())) + .thenReturn(new StringEntity(searchResponse.get("body").toString())); plugin.transaction(config, new Control()); // Check credential 1 + loadData 1 @@ -172,20 +207,21 @@ public void test_preview_with1RecordsResult() throws IOException @Test public void test_guess() throws IOException { - ConfigSource configSource = TestHelpers.config(); - JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); - JsonObject searchResponse = data.get("guessDataResult").getAsJsonObject(); + final ConfigSource configSource = TestHelpers.config(); + + final JsonObject authorizeResponse = data.get("authenticateSuccess").getAsJsonObject(); + final JsonObject searchResponse = data.get("guessDataResult").getAsJsonObject(); when(statusLine.getStatusCode()) - .thenReturn(authorizeResponse.get("statusCode").getAsInt()) - .thenReturn(searchResponse.get("statusCode").getAsInt()); + .thenReturn(authorizeResponse.get("statusCode").getAsInt()) + .thenReturn(searchResponse.get("statusCode").getAsInt()); when(response.getEntity()) - .thenReturn(new StringEntity(searchResponse.get("body").toString())) - .thenReturn(new StringEntity(searchResponse.get("body").toString())); + .thenReturn(new StringEntity(searchResponse.get("body").toString())) + .thenReturn(new StringEntity(searchResponse.get("body").toString())); - ConfigDiff result = plugin.guess(configSource); - JsonElement expected = data.get("guessResult").getAsJsonObject(); - JsonElement actual = new JsonParser().parse(result.toString()); + final ConfigDiff result = plugin.guess(configSource); + final JsonElement expected = data.get("guessResult").getAsJsonObject(); + final JsonElement actual = new JsonParser().parse(result.toString()); assertEquals(expected, actual); } @@ -194,7 +230,7 @@ private class Control implements InputPlugin.Control @Override public List run(final TaskSource taskSource, final Schema schema, final int taskCount) { - List reports = new ArrayList<>(); + final List reports = new ArrayList<>(); for (int i = 0; i < taskCount; i++) { reports.add(plugin.run(taskSource, schema, i, output)); } diff --git a/src/test/java/org/embulk/input/jira/JiraPluginTestRuntime.java b/src/test/java/org/embulk/input/jira/JiraPluginTestRuntime.java deleted file mode 100644 index 38b8a8c..0000000 --- a/src/test/java/org/embulk/input/jira/JiraPluginTestRuntime.java +++ /dev/null @@ -1,132 +0,0 @@ -package org.embulk.input.jira; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.inject.Binder; -import com.google.inject.Injector; -import com.google.inject.Module; -import org.embulk.GuiceBinder; -import org.embulk.RandomManager; -import org.embulk.TestPluginSourceModule; -import org.embulk.TestUtilityModule; -import org.embulk.config.ConfigSource; -import org.embulk.config.DataSourceImpl; -import org.embulk.config.ModelManager; -import org.embulk.exec.ExecModule; -import org.embulk.exec.ExtensionServiceLoaderModule; -import org.embulk.exec.SystemConfigModule; -import org.embulk.jruby.JRubyScriptingModule; -import org.embulk.plugin.BuiltinPluginSourceModule; -import org.embulk.plugin.PluginClassLoaderFactory; -import org.embulk.spi.BufferAllocator; -import org.embulk.spi.Exec; -import org.embulk.spi.ExecAction; -import org.embulk.spi.ExecSession; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; - -import java.util.Random; - -/** - * This is a clone from {@link org.embulk.EmbulkTestRuntime}, since there is no easy way to extend it. - * The only modification is on the provided systemConfig, enable tests to run `embulk/guess/jira.rb` - */ -public class JiraPluginTestRuntime extends GuiceBinder -{ - private static ConfigSource getSystemConfig() - { - final ObjectNode configNode = JsonNodeFactory.instance.objectNode(); - configNode.set("jruby_load_path", JsonNodeFactory.instance.arrayNode().add("lib")); - - return new DataSourceImpl(new ModelManager(null, new ObjectMapper()), configNode); - } - - public static class TestRuntimeModule implements Module - { - @Override - public void configure(final Binder binder) - { - final ConfigSource systemConfig = getSystemConfig(); - new SystemConfigModule(systemConfig).configure(binder); - new ExecModule(systemConfig).configure(binder); - new ExtensionServiceLoaderModule(systemConfig).configure(binder); - new BuiltinPluginSourceModule().configure(binder); - new JRubyScriptingModule(systemConfig).configure(binder); - new TestUtilityModule().configure(binder); - new TestPluginSourceModule().configure(binder); - } - } - - private final ExecSession exec; - - public JiraPluginTestRuntime() - { - super(new TestRuntimeModule()); - final Injector injector = getInjector(); - final ConfigSource execConfig = new DataSourceImpl(injector.getInstance(ModelManager.class)); - this.exec = ExecSession.builder(injector).fromExecConfig(execConfig).build(); - } - - public ExecSession getExec() - { - return exec; - } - - public BufferAllocator getBufferAllocator() - { - return getInstance(BufferAllocator.class); - } - - public ModelManager getModelManager() - { - return getInstance(ModelManager.class); - } - - public Random getRandom() - { - return getInstance(RandomManager.class).getRandom(); - } - - public PluginClassLoaderFactory getPluginClassLoaderFactory() - { - return getInstance(PluginClassLoaderFactory.class); - } - - @Override - public Statement apply(final Statement base, final Description description) - { - final Statement superStatement = JiraPluginTestRuntime.super.apply(base, description); - return new Statement() { - @Override - public void evaluate() throws Throwable - { - try { - Exec.doWith(exec, (ExecAction) () -> { - try { - superStatement.evaluate(); - } - catch (final Throwable ex) { - throw new RuntimeExecutionException(ex); - } - return null; - }); - } - catch (final RuntimeException ex) { - throw ex.getCause(); - } - finally { - exec.cleanup(); - } - } - }; - } - - private static class RuntimeExecutionException extends RuntimeException - { - public RuntimeExecutionException(final Throwable cause) - { - super(cause); - } - } -} diff --git a/src/test/java/org/embulk/input/jira/TestHelpers.java b/src/test/java/org/embulk/input/jira/TestHelpers.java index f6c0c69..807bdc7 100644 --- a/src/test/java/org/embulk/input/jira/TestHelpers.java +++ b/src/test/java/org/embulk/input/jira/TestHelpers.java @@ -1,27 +1,28 @@ package org.embulk.input.jira; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.guava.GuavaModule; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.google.common.io.Resources; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.stream.JsonReader; - -import org.embulk.config.ConfigLoader; import org.embulk.config.ConfigSource; -import org.embulk.config.ModelManager; +import org.embulk.spi.type.Types; +import org.embulk.util.config.units.ColumnConfig; +import org.embulk.util.config.units.SchemaConfig; -import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayList; + +import static org.embulk.input.jira.JiraInputPlugin.CONFIG_MAPPER_FACTORY; public final class TestHelpers { - private TestHelpers() {} + private TestHelpers() + { + } public static JsonObject getJsonFromFile(final String fileName) throws IOException - { + { final String path = Resources.getResource(fileName).getPath(); try (JsonReader reader = new JsonReader(new FileReader(path))) { final JsonParser parser = new JsonParser(); @@ -29,13 +30,35 @@ public static JsonObject getJsonFromFile(final String fileName) throws IOExcepti } } - public static ConfigSource config() throws IOException + @SuppressWarnings("serial") + public static ConfigSource config() { - final String path = Resources.getResource("config.yml").getPath(); - final ObjectMapper mapper = new ObjectMapper() - .registerModule(new GuavaModule()) - .registerModule(new Jdk8Module()); - final ConfigLoader configLoader = new ConfigLoader(new ModelManager(null, mapper)); - return configLoader.fromYamlFile(new File(path)); + return CONFIG_MAPPER_FACTORY.newConfigSource() + .set("type", "jira") + .set("username", "example@example.com") + .set("password", "XXXXXXXXXXXXXXXXX") + .set("uri", "https://example.com/") + .set("jql", "project = example") + .set("retry_limit", 3) + .set("columns", new SchemaConfig(new ArrayList() + { + { + add(new ColumnConfig("boolean", Types.BOOLEAN, EMPTY_CONFIG_SOURCE)); + add(new ColumnConfig("long", Types.LONG, EMPTY_CONFIG_SOURCE)); + add(new ColumnConfig("double", Types.DOUBLE, EMPTY_CONFIG_SOURCE)); + add(new ColumnConfig("string", Types.STRING, EMPTY_CONFIG_SOURCE)); + add(new ColumnConfig("date", + Types.TIMESTAMP, CONFIG_MAPPER_FACTORY + .newConfigSource().set("format", "%Y-%m-%dT%H:%M:%S.%L%z"))); + add(new ColumnConfig("json", Types.JSON, EMPTY_CONFIG_SOURCE)); + } + })); } + + public static ConfigSource dynamicSchemaConfig() + { + return config().set("dynamic_schema", true); + } + + private static final ConfigSource EMPTY_CONFIG_SOURCE = CONFIG_MAPPER_FACTORY.newConfigSource(); } diff --git a/src/test/java/org/embulk/input/jira/client/JiraClientTest.java b/src/test/java/org/embulk/input/jira/client/JiraClientTest.java index 54f9ecf..9562ade 100644 --- a/src/test/java/org/embulk/input/jira/client/JiraClientTest.java +++ b/src/test/java/org/embulk/input/jira/client/JiraClientTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; +import static org.embulk.input.jira.JiraInputPlugin.CONFIG_MAPPER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.times; @@ -45,7 +46,7 @@ public void setUp() throws IOException if (jiraClient == null) { jiraClient = Mockito.spy(new JiraClient()); response = Mockito.mock(CloseableHttpResponse.class); - task = TestHelpers.config().loadConfig(PluginTask.class); + task = CONFIG_MAPPER.map(TestHelpers.config(), PluginTask.class); data = TestHelpers.getJsonFromFile("jira_client.json"); } when(jiraClient.createHttpClient()).thenReturn(client); @@ -233,13 +234,13 @@ public void test_searchIssues_emptyJql() throws IOException when(statusLine.getStatusCode()).thenReturn(statusCode); when(response.getEntity()).thenReturn(new StringEntity(body)); ConfigSource config = TestHelpers.config().remove("jql"); - task = config.loadConfig(PluginTask.class); + task = CONFIG_MAPPER.map(config, PluginTask.class); List issues = jiraClient.searchIssues(task, 0, 50); assertEquals(issues.size(), 2); config = TestHelpers.config().set("jql", ""); - task = config.loadConfig(PluginTask.class); + task = CONFIG_MAPPER.map(config, PluginTask.class); issues = jiraClient.searchIssues(task, 0, 50); assertEquals(issues.size(), 2); diff --git a/src/test/java/org/embulk/input/jira/util/JiraUtilTest.java b/src/test/java/org/embulk/input/jira/util/JiraUtilTest.java index ca3012f..85d5c52 100644 --- a/src/test/java/org/embulk/input/jira/util/JiraUtilTest.java +++ b/src/test/java/org/embulk/input/jira/util/JiraUtilTest.java @@ -1,7 +1,6 @@ package org.embulk.input.jira.util; import com.google.gson.JsonObject; - import org.embulk.config.ConfigException; import org.embulk.config.ConfigSource; import org.embulk.input.jira.Issue; @@ -10,16 +9,17 @@ import org.embulk.spi.Column; import org.embulk.spi.PageBuilder; import org.embulk.spi.Schema; -import org.embulk.spi.json.JsonParser; -import org.embulk.spi.time.Timestamp; -import org.embulk.spi.time.TimestampParser; +import org.embulk.util.json.JsonParser; +import org.embulk.util.timestamp.TimestampFormatter; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; import org.msgpack.value.Value; import java.io.IOException; +import java.time.Instant; +import static org.embulk.input.jira.JiraInputPlugin.CONFIG_MAPPER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.times; @@ -41,7 +41,7 @@ public class JiraUtilTest public static void setUp() throws IOException { data = TestHelpers.getJsonFromFile("jira_util.json"); - pluginTask = TestHelpers.config().loadConfig(PluginTask.class); + pluginTask = CONFIG_MAPPER.map(TestHelpers.config(), PluginTask.class); schema = pluginTask.getColumns().toSchema(); booleanColumn = schema.getColumn(0); longColumn = schema.getColumn(1); @@ -50,6 +50,7 @@ public static void setUp() throws IOException dateColumn = schema.getColumn(4); jsonColumn = schema.getColumn(5); } + @Test public void test_calculateTotalPage() { @@ -102,7 +103,7 @@ public void test_buildPermissionUrl() @Test public void test_buildSearchUrl() throws IOException { - PluginTask task = TestHelpers.config().loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(TestHelpers.config(), PluginTask.class); String expected = "https://example.com/rest/api/latest/search"; String actual = JiraUtil.buildSearchUrl(task.getUri()); assertEquals(expected, actual); @@ -112,7 +113,7 @@ public void test_buildSearchUrl() throws IOException public void test_validateTaskConfig_allValid() throws IOException { ConfigSource configSource = TestHelpers.config(); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); } @@ -122,7 +123,7 @@ public void test_validateTaskConfig_emptyUsername() throws IOException ConfigException exception = assertThrows("Username or email could not be empty", ConfigException.class, () -> { ConfigSource configSource = TestHelpers.config(); configSource.set("username", ""); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); }); assertEquals("Username or email could not be empty", exception.getMessage()); @@ -134,7 +135,7 @@ public void test_validateTaskConfig_emptyPassword() throws IOException ConfigException exception = assertThrows(ConfigException.class, () -> { ConfigSource configSource = TestHelpers.config(); configSource.set("password", ""); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); }); assertEquals("Password could not be empty", exception.getMessage()); @@ -146,7 +147,7 @@ public void test_validateTaskConfig_emptyUri() throws IOException ConfigException exception = assertThrows(ConfigException.class, () -> { ConfigSource configSource = TestHelpers.config(); configSource.set("uri", ""); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); }); assertEquals("JIRA API endpoint could not be empty", exception.getMessage()); @@ -158,7 +159,7 @@ public void test_validateTaskConfig_nonExistedUri() throws IOException ConfigException exception = assertThrows(ConfigException.class, () -> { ConfigSource configSource = TestHelpers.config(); configSource.set("uri", "https://not-existed-domain"); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); }); assertEquals("JIRA API endpoint is incorrect or not available", exception.getMessage()); @@ -170,7 +171,7 @@ public void test_validateTaskConfig_invalidUriProtocol() throws IOException ConfigException exception = assertThrows(ConfigException.class, () -> { ConfigSource configSource = TestHelpers.config(); configSource.set("uri", "ftp://example.com"); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); }); assertEquals("JIRA API endpoint is incorrect or not available", exception.getMessage()); @@ -182,7 +183,7 @@ public void test_validateTaskConfig_containSpaceUri() throws IOException ConfigException exception = assertThrows(ConfigException.class, () -> { ConfigSource configSource = TestHelpers.config(); configSource.set("uri", "https://example .com"); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); }); assertEquals("JIRA API endpoint is incorrect or not available", exception.getMessage()); @@ -193,7 +194,7 @@ public void test_validateTaskConfig_emptyJql() throws IOException { ConfigSource configSource = TestHelpers.config(); configSource.set("jql", ""); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); } @@ -202,7 +203,7 @@ public void test_validateTaskConfig_missingJql() throws IOException { ConfigSource configSource = TestHelpers.config(); configSource.remove("jql"); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); } @@ -212,7 +213,7 @@ public void test_validateTaskConfig_RetryIntervalIs0() throws IOException ConfigException exception = assertThrows(ConfigException.class, () -> { ConfigSource configSource = TestHelpers.config(); configSource.set("initial_retry_interval_millis", 0); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); }); assertEquals("Initial retry delay should be equal or greater than 1", exception.getMessage()); @@ -224,7 +225,7 @@ public void test_validateTaskConfig_RetryIntervalIsNegative() throws IOException ConfigException exception = assertThrows(ConfigException.class, () -> { ConfigSource configSource = TestHelpers.config(); configSource.set("initial_retry_interval_millis", -1); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); }); assertEquals("Initial retry delay should be equal or greater than 1", exception.getMessage()); @@ -236,7 +237,7 @@ public void test_validateTaskConfig_RetryLimitGreaterThan10() throws IOException ConfigException exception = assertThrows(ConfigException.class, () -> { ConfigSource configSource = TestHelpers.config(); configSource.set("retry_limit", 11); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); }); assertEquals("Retry limit should between 0 and 10", exception.getMessage()); @@ -248,13 +249,14 @@ public void test_validateTaskConfig_RetryLimitLessThan0() throws IOException ConfigException exception = assertThrows(ConfigException.class, () -> { ConfigSource configSource = TestHelpers.config(); configSource.set("retry_limit", -1); - PluginTask task = configSource.loadConfig(PluginTask.class); + PluginTask task = CONFIG_MAPPER.map(configSource, PluginTask.class); JiraUtil.validateTaskConfig(task); }); assertEquals("Retry limit should between 0 and 10", exception.getMessage()); } @Test + @SuppressWarnings("deprecation") // TODO: For compatibility with Embulk v0.9 public void test_addRecord_allRight() { String testName = "allRight"; @@ -265,7 +267,10 @@ public void test_addRecord_allRight() Long longValue = Long.valueOf(1); Double doubleValue = Double.valueOf(1); String stringValue = "string"; - Timestamp dateValue = TimestampParser.of("%Y-%m-%dT%H:%M:%S.%L%z", "UTC").parse("2019-01-01T00:00:00.000Z"); + Instant dateValue = TimestampFormatter + .builder("%Y-%m-%dT%H:%M:%S.%L%z", true) + .setDefaultZoneFromString("UTC") + .build().parse("2019-01-01T00:00:00.000Z"); Value jsonValue = new JsonParser().parse("{}"); JiraUtil.addRecord(issue, schema, pluginTask, mock); @@ -274,7 +279,8 @@ public void test_addRecord_allRight() verify(mock, times(1)).setLong(longColumn, longValue); verify(mock, times(1)).setDouble(doubleColumn, doubleValue); verify(mock, times(1)).setString(stringColumn, stringValue); - verify(mock, times(1)).setTimestamp(dateColumn, dateValue); + // TODO: Use Instant instead of Timestamp + verify(mock, times(1)).setTimestamp(dateColumn, org.embulk.spi.time.Timestamp.ofInstant(dateValue)); verify(mock, times(1)).setJson(jsonColumn, jsonValue); } diff --git a/src/test/resources/config.yml b/src/test/resources/config.yml deleted file mode 100644 index a37d6a2..0000000 --- a/src/test/resources/config.yml +++ /dev/null @@ -1,13 +0,0 @@ -type: jira -username: example@example.com -password: XXXXXXXXXXXXXXXXX -uri: "https://example.com/" -jql: project = example -retry_limit: 3 -columns: - - {name: boolean, type: boolean} - - {name: long, type: long} - - {name: double, type: double} - - {name: string, type: string} - - {name: date, type: timestamp, format: '%Y-%m-%dT%H:%M:%S.%L%z'} - - {name: json, type: json} \ No newline at end of file