Skip to content

Commit f611f1c

Browse files
martijnvgmartijnvg
committed
ingest: Move processors from core to ingest-common module.
Folded grok processor into ingest-common module. The rest tests have been moved to ingest-common module as well, because these tests don't run in the rest-api-spec module but in the distribution:integ-test-zip module and adding a test plugin there felt just wrong to me. I think this is ok. I left a tiny ingest rest test behind in that tests with an empty pipeline. Removed messy tests, these tests were already covered in the rest tests Added ingest test plugin in test infra so that each module testing integration with ingest doesn't need write its own plugin Moved reindex ingest tests to qa module Closes #18490
1 parent c0da8e4 commit f611f1c

File tree

120 files changed

+407
-468
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

120 files changed

+407
-468
lines changed

‎buildSrc/src/main/resources/checkstyle_suppressions.xml‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -501,7 +501,6 @@
501501
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]CompoundProcessor.java" checks="LineLength" />
502502
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]IngestDocument.java" checks="LineLength" />
503503
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]Pipeline.java" checks="LineLength" />
504-
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]ConvertProcessor.java" checks="LineLength" />
505504
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]GcNames.java" checks="LineLength" />
506505
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]HotThreads.java" checks="LineLength" />
507506
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmStats.java" checks="LineLength" />
@@ -1035,14 +1034,11 @@
10351034
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreIntegrationIT.java" checks="LineLength" />
10361035
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreTests.java" checks="LineLength" />
10371036
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]template[/\\]SimpleIndexTemplateIT.java" checks="LineLength" />
1038-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]AbstractStringProcessorTestCase.java" checks="LineLength" />
1039-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]AppendProcessorTests.java" checks="LineLength" />
1040-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]DateFormatTests.java" checks="LineLength" />
1041-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]DateProcessorTests.java" checks="LineLength" />
1042-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]GsubProcessorTests.java" checks="LineLength" />
1043-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]RenameProcessorTests.java" checks="LineLength" />
1044-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]SetProcessorTests.java" checks="LineLength" />
1045-
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]SplitProcessorTests.java" checks="LineLength" />
1037+
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineExecutionServiceTests.java" checks="LineLength" />
1038+
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineStoreTests.java" checks="LineLength" />
1039+
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]CompoundProcessorTests.java" checks="LineLength" />
1040+
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]PipelineFactoryTests.java" checks="LineLength" />
1041+
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]ValueSourceTests.java" checks="LineLength" />
10461042
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mget[/\\]SimpleMgetIT.java" checks="LineLength" />
10471043
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmGcMonitorServiceSettingsTests.java" checks="LineLength" />
10481044
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]os[/\\]OsProbeTests.java" checks="LineLength" />
@@ -1154,9 +1150,6 @@
11541150
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]update[/\\]UpdateIT.java" checks="LineLength" />
11551151
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]validate[/\\]SimpleValidateQueryIT.java" checks="LineLength" />
11561152
<suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]versioning[/\\]SimpleVersioningIT.java" checks="LineLength" />
1157-
<suppress files="modules[/\\]ingest-grok[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]grok[/\\]Grok.java" checks="LineLength" />
1158-
<suppress files="modules[/\\]ingest-grok[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]grok[/\\]GrokProcessorTests.java" checks="LineLength" />
1159-
<suppress files="modules[/\\]ingest-grok[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]grok[/\\]GrokTests.java" checks="LineLength" />
11601153
<suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionPlugin.java" checks="LineLength" />
11611154
<suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionScriptEngineService.java" checks="LineLength" />
11621155
<suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionSearchScript.java" checks="LineLength" />

‎core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
import java.util.ArrayList;
3030
import java.util.List;
3131

32-
import static org.elasticsearch.ingest.processor.TrackingResultProcessor.decorate;
32+
import static org.elasticsearch.action.ingest.TrackingResultProcessor.decorate;
3333

3434
class SimulateExecutionService {
3535

core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java renamed to core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@
1717
* under the License.
1818
*/
1919

20-
package org.elasticsearch.ingest.processor;
20+
package org.elasticsearch.action.ingest;
2121

22-
import org.elasticsearch.action.ingest.SimulateProcessorResult;
2322
import org.elasticsearch.ingest.core.CompoundProcessor;
2423
import org.elasticsearch.ingest.core.IngestDocument;
2524
import org.elasticsearch.ingest.core.Processor;

‎core/src/main/java/org/elasticsearch/node/NodeModule.java‎

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -24,22 +24,6 @@
2424
import org.elasticsearch.ingest.ProcessorsRegistry;
2525
import org.elasticsearch.ingest.core.Processor;
2626
import org.elasticsearch.ingest.core.TemplateService;
27-
import org.elasticsearch.ingest.processor.AppendProcessor;
28-
import org.elasticsearch.ingest.processor.ConvertProcessor;
29-
import org.elasticsearch.ingest.processor.DateProcessor;
30-
import org.elasticsearch.ingest.processor.DateIndexNameProcessor;
31-
import org.elasticsearch.ingest.processor.FailProcessor;
32-
import org.elasticsearch.ingest.processor.ForEachProcessor;
33-
import org.elasticsearch.ingest.processor.GsubProcessor;
34-
import org.elasticsearch.ingest.processor.JoinProcessor;
35-
import org.elasticsearch.ingest.processor.LowercaseProcessor;
36-
import org.elasticsearch.ingest.processor.RemoveProcessor;
37-
import org.elasticsearch.ingest.processor.RenameProcessor;
38-
import org.elasticsearch.ingest.processor.SetProcessor;
39-
import org.elasticsearch.ingest.processor.SortProcessor;
40-
import org.elasticsearch.ingest.processor.SplitProcessor;
41-
import org.elasticsearch.ingest.processor.TrimProcessor;
42-
import org.elasticsearch.ingest.processor.UppercaseProcessor;
4327
import org.elasticsearch.monitor.MonitorService;
4428
import org.elasticsearch.node.service.NodeService;
4529

@@ -61,23 +45,6 @@ public NodeModule(Node node, MonitorService monitorService) {
6145
this.node = node;
6246
this.monitorService = monitorService;
6347
this.processorsRegistryBuilder = new ProcessorsRegistry.Builder();
64-
65-
registerProcessor(DateProcessor.TYPE, (templateService, registry) -> new DateProcessor.Factory());
66-
registerProcessor(SetProcessor.TYPE, (templateService, registry) -> new SetProcessor.Factory(templateService));
67-
registerProcessor(AppendProcessor.TYPE, (templateService, registry) -> new AppendProcessor.Factory(templateService));
68-
registerProcessor(RenameProcessor.TYPE, (templateService, registry) -> new RenameProcessor.Factory());
69-
registerProcessor(RemoveProcessor.TYPE, (templateService, registry) -> new RemoveProcessor.Factory(templateService));
70-
registerProcessor(SplitProcessor.TYPE, (templateService, registry) -> new SplitProcessor.Factory());
71-
registerProcessor(JoinProcessor.TYPE, (templateService, registry) -> new JoinProcessor.Factory());
72-
registerProcessor(UppercaseProcessor.TYPE, (templateService, registry) -> new UppercaseProcessor.Factory());
73-
registerProcessor(LowercaseProcessor.TYPE, (templateService, registry) -> new LowercaseProcessor.Factory());
74-
registerProcessor(TrimProcessor.TYPE, (templateService, registry) -> new TrimProcessor.Factory());
75-
registerProcessor(ConvertProcessor.TYPE, (templateService, registry) -> new ConvertProcessor.Factory());
76-
registerProcessor(GsubProcessor.TYPE, (templateService, registry) -> new GsubProcessor.Factory());
77-
registerProcessor(FailProcessor.TYPE, (templateService, registry) -> new FailProcessor.Factory(templateService));
78-
registerProcessor(ForEachProcessor.TYPE, (templateService, registry) -> new ForEachProcessor.Factory(registry));
79-
registerProcessor(DateIndexNameProcessor.TYPE, (templateService, registry) -> new DateIndexNameProcessor.Factory());
80-
registerProcessor(SortProcessor.TYPE, (templateService, registry) -> new SortProcessor.Factory());
8148
}
8249

8350
@Override

core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java renamed to core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,11 @@
1717
* under the License.
1818
*/
1919

20-
package org.elasticsearch.ingest.processor;
20+
package org.elasticsearch.action.ingest;
2121

2222
import org.elasticsearch.ElasticsearchException;
2323
import org.elasticsearch.action.ingest.SimulateProcessorResult;
24+
import org.elasticsearch.action.ingest.TrackingResultProcessor;
2425
import org.elasticsearch.ingest.TestProcessor;
2526
import org.elasticsearch.ingest.core.CompoundProcessor;
2627
import org.elasticsearch.ingest.core.IngestDocument;
@@ -36,7 +37,7 @@
3637
import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_MESSAGE_FIELD;
3738
import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD;
3839
import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD;
39-
import static org.elasticsearch.ingest.processor.TrackingResultProcessor.decorate;
40+
import static org.elasticsearch.action.ingest.TrackingResultProcessor.decorate;
4041
import static org.hamcrest.CoreMatchers.equalTo;
4142
import static org.hamcrest.Matchers.nullValue;
4243

‎core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java‎

Lines changed: 1 addition & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@
3838
import org.elasticsearch.common.bytes.BytesReference;
3939
import org.elasticsearch.common.settings.Settings;
4040
import org.elasticsearch.ingest.core.IngestDocument;
41-
import org.elasticsearch.node.NodeModule;
4241
import org.elasticsearch.plugins.Plugin;
4342
import org.elasticsearch.test.ESIntegTestCase;
4443

@@ -69,7 +68,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
6968

7069
@Override
7170
protected Collection<Class<? extends Plugin>> nodePlugins() {
72-
return pluginList(IngestPlugin.class);
71+
return pluginList(IngestTestPlugin.class);
7372
}
7473

7574
public void testSimulate() throws Exception {
@@ -234,28 +233,4 @@ public void testPutWithPipelineFactoryError() throws Exception {
234233
protected Collection<Class<? extends Plugin>> getMockPlugins() {
235234
return Collections.singletonList(TestSeedPlugin.class);
236235
}
237-
238-
public static class IngestPlugin extends Plugin {
239-
240-
@Override
241-
public String name() {
242-
return "ingest";
243-
}
244-
245-
@Override
246-
public String description() {
247-
return "ingest mock";
248-
}
249-
250-
public void onModule(NodeModule nodeModule) {
251-
nodeModule.registerProcessor("test", (templateService, registry) -> config ->
252-
new TestProcessor("id", "test", ingestDocument -> {
253-
ingestDocument.setFieldValue("processed", true);
254-
if (ingestDocument.getFieldValue("fail", Boolean.class)) {
255-
throw new IllegalArgumentException("test processor failed");
256-
}
257-
})
258-
);
259-
}
260-
}
261236
}

‎core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ public IngestProcessorNotInstalledOnAllNodesIT() throws IOException {
5757

5858
@Override
5959
protected Collection<Class<? extends Plugin>> nodePlugins() {
60-
return installPlugin ? pluginList(IngestClientIT.IngestPlugin.class) : Collections.emptyList();
60+
return installPlugin ? pluginList(IngestTestPlugin.class) : Collections.emptyList();
6161
}
6262

6363
@Override

‎core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,11 @@
3131
import org.elasticsearch.common.bytes.BytesArray;
3232
import org.elasticsearch.common.settings.Settings;
3333
import org.elasticsearch.common.transport.LocalTransportAddress;
34+
import org.elasticsearch.ingest.core.IngestDocument;
3435
import org.elasticsearch.ingest.core.IngestInfo;
3536
import org.elasticsearch.ingest.core.Pipeline;
37+
import org.elasticsearch.ingest.core.Processor;
3638
import org.elasticsearch.ingest.core.ProcessorInfo;
37-
import org.elasticsearch.ingest.processor.RemoveProcessor;
38-
import org.elasticsearch.ingest.processor.SetProcessor;
3939
import org.elasticsearch.test.ESTestCase;
4040
import org.junit.Before;
4141

@@ -60,9 +60,45 @@ public class PipelineStoreTests extends ESTestCase {
6060
public void init() throws Exception {
6161
store = new PipelineStore(Settings.EMPTY);
6262
ProcessorsRegistry.Builder registryBuilder = new ProcessorsRegistry.Builder();
63-
registryBuilder.registerProcessor("set", (templateService, registry) -> new SetProcessor.Factory(TestTemplateService.instance()));
64-
registryBuilder.registerProcessor("remove", (templateService, registry) ->
65-
new RemoveProcessor.Factory(TestTemplateService.instance()));
63+
registryBuilder.registerProcessor("set", (templateService, registry) -> config -> {
64+
String field = (String) config.remove("field");
65+
String value = (String) config.remove("value");
66+
return new Processor() {
67+
@Override
68+
public void execute(IngestDocument ingestDocument) throws Exception {
69+
ingestDocument.setFieldValue(field, value);
70+
}
71+
72+
@Override
73+
public String getType() {
74+
return "set";
75+
}
76+
77+
@Override
78+
public String getTag() {
79+
return null;
80+
}
81+
};
82+
});
83+
registryBuilder.registerProcessor("remove", (templateService, registry) -> config -> {
84+
String field = (String) config.remove("field");
85+
return new Processor() {
86+
@Override
87+
public void execute(IngestDocument ingestDocument) throws Exception {
88+
ingestDocument.removeField(field);
89+
}
90+
91+
@Override
92+
public String getType() {
93+
return "remove";
94+
}
95+
96+
@Override
97+
public String getTag() {
98+
return null;
99+
}
100+
};
101+
});
66102
store.buildProcessorFactoryRegistry(registryBuilder, null);
67103
}
68104

‎core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,13 @@
2323
import org.elasticsearch.ingest.ProcessorsRegistry;
2424
import org.elasticsearch.ingest.TestProcessor;
2525
import org.elasticsearch.ingest.TestTemplateService;
26-
import org.elasticsearch.ingest.processor.FailProcessor;
27-
import org.elasticsearch.ingest.processor.SetProcessor;
2826
import org.elasticsearch.test.ESTestCase;
2927

3028
import java.util.Arrays;
3129
import java.util.Collections;
3230
import java.util.HashMap;
3331
import java.util.List;
3432
import java.util.Map;
35-
import java.util.prefs.PreferencesFactory;
3633

3734
import static org.hamcrest.CoreMatchers.equalTo;
3835
import static org.hamcrest.Matchers.is;

modules/ingest-grok/build.gradle renamed to modules/ingest-common/build.gradle

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
*/
1919

2020
esplugin {
21-
description 'Ingest processor that uses grok patterns to split text'
22-
classname 'org.elasticsearch.ingest.grok.IngestGrokPlugin'
21+
description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources'
22+
classname 'org.elasticsearch.ingest.IngestCommonPlugin'
2323
}
2424

2525
dependencies {
@@ -28,8 +28,8 @@ dependencies {
2828
compile 'org.jruby.jcodings:jcodings:1.0.12'
2929
}
3030

31-
compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked,-serial"
32-
compileTestJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
31+
compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
32+
compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
3333

3434
thirdPartyAudit.excludes = [
3535
// joni has AsmCompilerSupport, but that isn't being used:

0 commit comments

Comments
 (0)