Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

NUTCH-3008 indexer-elastic: downgrade to ES 7.10.2 to address licensing issues #806

Merged
merged 1 commit into from
Mar 14, 2024
Merged

NUTCH-3008 indexer-elastic: downgrade to ES 7.10.2 to address licensing issues #806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/plugin/indexer-elastic/howto_upgrade_es.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
(eventually with different versions)
- duplicated libs can be added to the exclusions of transitive dependencies in
build/plugins/indexer-elastic/ivy.xml
- but it should be made sure that the library versions in ivy/ivy.xml correspend to
- but it should be made sure that the library versions in ivy/ivy.xml correspond to
those required by Tika

5. Remove the locally "installed" dependencies in src/plugin/indexer-elastic/lib/:
Expand All @@ -47,4 +47,4 @@
6. Build Nutch and run all unit tests:

$ cd ../../../
$ ant clean runtime test
$ ant clean runtime test
2 changes: 1 addition & 1 deletion src/plugin/indexer-elastic/ivy.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
</publications>

<dependencies>
<dependency org="org.elasticsearch.client" name="elasticsearch-rest-high-level-client" rev="7.13.2">
<dependency org="org.elasticsearch.client" name="elasticsearch-rest-high-level-client" rev="7.10.2">
<!-- exclusions of dependencies provided in Nutch core (ivy/ivy.xml) -->
<exclude org="commons-codec" name="commons-codec" />
<exclude org="commons-logging" name="commons-logging" />
Expand Down
37 changes: 18 additions & 19 deletions src/plugin/indexer-elastic/plugin.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,17 @@
</library>
<!-- Elastic Rest Client Dependencies -->
<!-- end of Elastic Rest Client dependencies -->
<library name="HdrHistogram-2.1.9.jar"/>
<library name="aggs-matrix-stats-client-7.13.2.jar"/>
<library name="aggs-matrix-stats-client-7.10.2.jar"/>
<library name="compiler-0.9.6.jar"/>
<library name="elasticsearch-7.13.2.jar"/>
<library name="elasticsearch-cli-7.13.2.jar"/>
<library name="elasticsearch-core-7.13.2.jar"/>
<library name="elasticsearch-geo-7.13.2.jar"/>
<library name="elasticsearch-plugin-classloader-7.13.2.jar"/>
<library name="elasticsearch-rest-client-7.13.2.jar"/>
<library name="elasticsearch-rest-high-level-client-7.13.2.jar"/>
<library name="elasticsearch-secure-sm-7.13.2.jar"/>
<library name="elasticsearch-x-content-7.13.2.jar"/>
<library name="elasticsearch-7.10.2.jar"/>
<library name="elasticsearch-cli-7.10.2.jar"/>
<library name="elasticsearch-core-7.10.2.jar"/>
<library name="elasticsearch-geo-7.10.2.jar"/>
<library name="elasticsearch-rest-client-7.10.2.jar"/>
<library name="elasticsearch-rest-high-level-client-7.10.2.jar"/>
<library name="elasticsearch-secure-sm-7.10.2.jar"/>
<library name="elasticsearch-x-content-7.10.2.jar"/>
<library name="HdrHistogram-2.1.9.jar"/>
<library name="hppc-0.8.1.jar"/>
<library name="httpasyncclient-4.1.4.jar"/>
<library name="httpclient-4.5.10.jar"/>
Expand All @@ -43,10 +42,10 @@
<library name="jackson-dataformat-cbor-2.10.4.jar"/>
<library name="jackson-dataformat-smile-2.10.4.jar"/>
<library name="jackson-dataformat-yaml-2.10.4.jar"/>
<library name="jna-5.7.0-1.jar"/>
<library name="joda-time-2.10.10.jar"/>
<library name="jna-5.5.0.jar"/>
<library name="joda-time-2.10.4.jar"/>
<library name="jopt-simple-5.0.2.jar"/>
<library name="lang-mustache-client-7.13.2.jar"/>
<library name="lang-mustache-client-7.10.2.jar"/>
<library name="lucene-analyzers-common-8.11.2.jar"/>
<library name="lucene-backward-codecs-8.11.2.jar"/>
<library name="lucene-core-8.11.2.jar"/>
Expand All @@ -58,12 +57,12 @@
<library name="lucene-queries-8.11.2.jar"/>
<library name="lucene-queryparser-8.11.2.jar"/>
<library name="lucene-sandbox-8.11.2.jar"/>
<library name="lucene-spatial-extras-8.11.2.jar"/>
<library name="lucene-spatial3d-8.11.2.jar"/>
<library name="lucene-spatial-extras-8.11.2.jar"/>
<library name="lucene-suggest-8.11.2.jar"/>
<library name="mapper-extras-client-7.13.2.jar"/>
<library name="parent-join-client-7.13.2.jar"/>
<library name="rank-eval-client-7.13.2.jar"/>
<library name="mapper-extras-client-7.10.2.jar"/>
<library name="parent-join-client-7.10.2.jar"/>
<library name="rank-eval-client-7.10.2.jar"/>
<library name="s2-geometry-library-java-1.0.0.jar"/>
<library name="snakeyaml-1.26.jar"/>
<library name="spatial4j-0.7.jar"/>
Expand All @@ -74,4 +73,4 @@
<extension id="org.apache.nutch.indexer.elastic" name="Elasticsearch Index Writer" point="org.apache.nutch.indexer.IndexWriter">
<implementation id="ElasticIndexWriter" class="org.apache.nutch.indexwriter.elastic.ElasticIndexWriter" />
</extension>
</plugin>
</plugin>
2 changes: 1 addition & 1 deletion ...gin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ public void open(IndexWriterParams parameters) throws IOException {
.builder(
(request, bulkListener) -> client.bulkAsync(request,
RequestOptions.DEFAULT, bulkListener),
bulkProcessorListener(), "nutch-indexer-elastic")
bulkProcessorListener())
.setBulkActions(maxBulkDocs)
.setBulkSize(new ByteSizeValue(maxBulkLength, ByteSizeUnit.BYTES))
.setConcurrentRequests(1)
Expand Down