From c8d2ac31687ca0f414927e267d57e7caf6dcc607 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 3 Apr 2019 10:53:38 +0200 Subject: [PATCH] add HLRC protocol tests for transform state and stats --- .../client/core/IndexerJobStats.java | 4 +- .../DataFrameIndexerTransformStats.java | 24 ++++---- .../DataFrameIndexerTransformStats.java | 36 ++++++------ .../DataFrameTransformStateAndStats.java | 8 ++- ...taFrameIndexerTransformStatsHlrcTests.java | 54 ++++++++++++++++++ ...aFrameTransformStateAndStatsHlrcTests.java | 57 +++++++++++++++++++ .../DataFrameTransformStateHlrcTests.java | 15 +++-- ...portGetDataFrameTransformsStatsAction.java | 2 +- 8 files changed, 163 insertions(+), 37 deletions(-) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameIndexerTransformStatsHlrcTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsHlrcTests.java rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/{ => hlrc}/DataFrameTransformStateHlrcTests.java (75%) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java index 71aea2e81e968..a358f53d9880e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java @@ -47,11 +47,11 @@ public abstract class IndexerJobStats { private final long indexFailures; private final long searchFailures; - public IndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations, + public IndexerJobStats(long numPages, long numInputDocuments, long numOutputDocuments, long numInvocations, long indexTime, long searchTime, long indexTotal, long searchTotal, long indexFailures, long searchFailures) { this.numPages = numPages; this.numInputDocuments = numInputDocuments; - this.numOuputDocuments = numOuputDocuments; + this.numOuputDocuments = numOutputDocuments; this.numInvocations = numInvocations; this.indexTime = indexTime; this.indexTotal = indexTotal; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameIndexerTransformStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameIndexerTransformStats.java index aa40b7cb86c9b..356c3d98916b3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameIndexerTransformStats.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameIndexerTransformStats.java @@ -29,25 +29,25 @@ public class DataFrameIndexerTransformStats extends IndexerJobStats { - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser LENIENT_PARSER = new ConstructingObjectParser<>( NAME, true, args -> new DataFrameIndexerTransformStats((long) args[0], (long) args[1], (long) args[2], (long) args[3], (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9])); static { - PARSER.declareLong(constructorArg(), NUM_PAGES); - PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS); - PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS); - PARSER.declareLong(constructorArg(), NUM_INVOCATIONS); - PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS); - PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS); - PARSER.declareLong(constructorArg(), INDEX_TOTAL); - PARSER.declareLong(constructorArg(), SEARCH_TOTAL); - PARSER.declareLong(constructorArg(), INDEX_FAILURES); - PARSER.declareLong(constructorArg(), SEARCH_FAILURES); + LENIENT_PARSER.declareLong(constructorArg(), NUM_PAGES); + LENIENT_PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS); + LENIENT_PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS); + LENIENT_PARSER.declareLong(constructorArg(), NUM_INVOCATIONS); + LENIENT_PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS); + LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS); + LENIENT_PARSER.declareLong(constructorArg(), INDEX_TOTAL); + LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TOTAL); + LENIENT_PARSER.declareLong(constructorArg(), INDEX_FAILURES); + LENIENT_PARSER.declareLong(constructorArg(), SEARCH_FAILURES); } public static DataFrameIndexerTransformStats fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); + return LENIENT_PARSER.parse(parser, null); } public DataFrameIndexerTransformStats(long numPages, long numInputDocuments, long numOuputDocuments, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameIndexerTransformStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameIndexerTransformStats.java index 05cb6dea33ed9..c2981c40dfdc1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameIndexerTransformStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameIndexerTransformStats.java @@ -23,7 +23,7 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; public class DataFrameIndexerTransformStats extends IndexerJobStats { - private static final String DEFAULT_TRANSFORM_ID = "_all"; + public static final String DEFAULT_TRANSFORM_ID = "_all"; public static final String NAME = "data_frame_indexer_transform_stats"; public static ParseField NUM_PAGES = new ParseField("pages_processed"); @@ -37,23 +37,25 @@ public class DataFrameIndexerTransformStats extends IndexerJobStats { public static ParseField SEARCH_FAILURES = new ParseField("search_failures"); public static ParseField INDEX_FAILURES = new ParseField("index_failures"); - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - NAME, args -> new DataFrameIndexerTransformStats((String) args[0], (long) args[1], (long) args[2], (long) args[3], - (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9], (long) args[10])); + private static final ConstructingObjectParser LENIENT_PARSER = new ConstructingObjectParser<>( + NAME, true, + args -> new DataFrameIndexerTransformStats(args[0] != null ? (String) args[0] : DEFAULT_TRANSFORM_ID, + (long) args[1], (long) args[2], (long) args[3], (long) args[4], (long) args[5], (long) args[6], (long) args[7], + (long) args[8], (long) args[9], (long) args[10])); static { - PARSER.declareString(optionalConstructorArg(), DataFrameField.ID); - PARSER.declareLong(constructorArg(), NUM_PAGES); - PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS); - PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS); - PARSER.declareLong(constructorArg(), NUM_INVOCATIONS); - PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS); - PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS); - PARSER.declareLong(constructorArg(), INDEX_TOTAL); - PARSER.declareLong(constructorArg(), SEARCH_TOTAL); - PARSER.declareLong(constructorArg(), INDEX_FAILURES); - PARSER.declareLong(constructorArg(), SEARCH_FAILURES); - PARSER.declareString(optionalConstructorArg(), DataFrameField.INDEX_DOC_TYPE); + LENIENT_PARSER.declareString(optionalConstructorArg(), DataFrameField.ID); + LENIENT_PARSER.declareLong(constructorArg(), NUM_PAGES); + LENIENT_PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS); + LENIENT_PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS); + LENIENT_PARSER.declareLong(constructorArg(), NUM_INVOCATIONS); + LENIENT_PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS); + LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS); + LENIENT_PARSER.declareLong(constructorArg(), INDEX_TOTAL); + LENIENT_PARSER.declareLong(constructorArg(), SEARCH_TOTAL); + LENIENT_PARSER.declareLong(constructorArg(), INDEX_FAILURES); + LENIENT_PARSER.declareLong(constructorArg(), SEARCH_FAILURES); + LENIENT_PARSER.declareString(optionalConstructorArg(), DataFrameField.INDEX_DOC_TYPE); } private final String transformId; @@ -197,7 +199,7 @@ public int hashCode() { public static DataFrameIndexerTransformStats fromXContent(XContentParser parser) { try { - return PARSER.parse(parser, null); + return LENIENT_PARSER.parse(parser, null); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java index dc58be9e6638a..8cb1ebd314451 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateAndStats.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.dataframe.transforms; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -29,7 +30,7 @@ public class DataFrameTransformStateAndStats implements Writeable, ToXContentObj private final DataFrameIndexerTransformStats transformStats; public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - NAME, + NAME, true, a -> new DataFrameTransformStateAndStats((String) a[0], (DataFrameTransformState) a[1], (DataFrameIndexerTransformStats) a[2])); static { @@ -110,4 +111,9 @@ public DataFrameIndexerTransformStats getTransformStats() { public DataFrameTransformState getTransformState() { return transformState; } + + @Override + public String toString() { + return Strings.toString(this); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameIndexerTransformStatsHlrcTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameIndexerTransformStatsHlrcTests.java new file mode 100644 index 0000000000000..9dd4bdbdbbda9 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameIndexerTransformStatsHlrcTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.dataframe.transforms.hlrc; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.protocol.AbstractHlrcXContentTestCase; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStatsTests; + +import java.io.IOException; + +public class DataFrameIndexerTransformStatsHlrcTests extends AbstractHlrcXContentTestCase< + DataFrameIndexerTransformStats, + org.elasticsearch.client.dataframe.transforms.DataFrameIndexerTransformStats> { + + public static DataFrameIndexerTransformStats fromHlrc( + org.elasticsearch.client.dataframe.transforms.DataFrameIndexerTransformStats instance) { + return DataFrameIndexerTransformStats.withDefaultTransformId(instance.getNumPages(), instance.getNumDocuments(), + instance.getOutputDocuments(), instance.getNumInvocations(), instance.getIndexTime(), instance.getSearchTime(), + instance.getIndexTotal(), instance.getSearchTotal(), instance.getIndexFailures(), instance.getSearchFailures()); + } + + @Override + public org.elasticsearch.client.dataframe.transforms.DataFrameIndexerTransformStats doHlrcParseInstance(XContentParser parser) + throws IOException { + return org.elasticsearch.client.dataframe.transforms.DataFrameIndexerTransformStats.fromXContent(parser); + } + + @Override + public DataFrameIndexerTransformStats convertHlrcToInternal( + org.elasticsearch.client.dataframe.transforms.DataFrameIndexerTransformStats instance) { + return fromHlrc(instance); + } + + @Override + protected DataFrameIndexerTransformStats createTestInstance() { + return DataFrameIndexerTransformStatsTests.randomStats(DataFrameIndexerTransformStats.DEFAULT_TRANSFORM_ID); + } + + @Override + protected DataFrameIndexerTransformStats doParseInstance(XContentParser parser) throws IOException { + return DataFrameIndexerTransformStats.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsHlrcTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsHlrcTests.java new file mode 100644 index 0000000000000..fe63c56941f8d --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameTransformStateAndStatsHlrcTests.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.dataframe.transforms.hlrc; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.protocol.AbstractHlrcXContentTestCase; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameIndexerTransformStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStats; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateAndStatsTests; + +import java.io.IOException; +import java.util.function.Predicate; + +public class DataFrameTransformStateAndStatsHlrcTests extends AbstractHlrcXContentTestCase { + + @Override + public org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats doHlrcParseInstance(XContentParser parser) + throws IOException { + return org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats.fromXContent(parser); + } + + @Override + public DataFrameTransformStateAndStats convertHlrcToInternal( + org.elasticsearch.client.dataframe.transforms.DataFrameTransformStateAndStats instance) { + return new DataFrameTransformStateAndStats(instance.getId(), + DataFrameTransformStateHlrcTests.fromHlrc(instance.getTransformState()), + DataFrameIndexerTransformStatsHlrcTests.fromHlrc(instance.getTransformStats())); + } + + @Override + protected DataFrameTransformStateAndStats createTestInstance() { + // the transform id is not part of HLRC as it's only to a field for internal storage, therefore use a default id + return DataFrameTransformStateAndStatsTests + .randomDataFrameTransformStateAndStats(DataFrameIndexerTransformStats.DEFAULT_TRANSFORM_ID); + } + + @Override + protected DataFrameTransformStateAndStats doParseInstance(XContentParser parser) throws IOException { + return DataFrameTransformStateAndStats.PARSER.apply(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return field -> field.equals("state.current_position"); + } +} + diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateHlrcTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameTransformStateHlrcTests.java similarity index 75% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateHlrcTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameTransformStateHlrcTests.java index 4f1c6b1f7615e..c6e4acd5c82b7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStateHlrcTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/hlrc/DataFrameTransformStateHlrcTests.java @@ -4,10 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.core.dataframe.transforms; +package org.elasticsearch.xpack.core.dataframe.transforms.hlrc; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.protocol.AbstractHlrcXContentTestCase; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformStateTests; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformTaskState; import org.elasticsearch.xpack.core.indexing.IndexerState; import java.io.IOException; @@ -16,6 +19,12 @@ public class DataFrameTransformStateHlrcTests extends AbstractHlrcXContentTestCase { + public static DataFrameTransformState fromHlrc(org.elasticsearch.client.dataframe.transforms.DataFrameTransformState instance) { + return new DataFrameTransformState(DataFrameTransformTaskState.fromString(instance.getTaskState().value()), + IndexerState.fromString(instance.getIndexerState().value()), instance.getPosition(), instance.getGeneration(), + instance.getReason()); + } + @Override public org.elasticsearch.client.dataframe.transforms.DataFrameTransformState doHlrcParseInstance(XContentParser parser) throws IOException { @@ -24,9 +33,7 @@ public org.elasticsearch.client.dataframe.transforms.DataFrameTransformState doH @Override public DataFrameTransformState convertHlrcToInternal(org.elasticsearch.client.dataframe.transforms.DataFrameTransformState instance) { - return new DataFrameTransformState(DataFrameTransformTaskState.fromString(instance.getTaskState().value()), - IndexerState.fromString(instance.getIndexerState().value()), - instance.getPosition(), instance.getGeneration(), instance.getReason()); + return fromHlrc(instance); } @Override diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java index c2f64dd66a583..eeabbe0b5ebe1 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/action/TransportGetDataFrameTransformsStatsAction.java @@ -214,7 +214,7 @@ private static DataFrameIndexerTransformStats parseFromSource(BytesReference sou try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentType.JSON) .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - return DataFrameIndexerTransformStats.PARSER.apply(parser, null); + return DataFrameIndexerTransformStats.fromXContent(parser); } } }