From e5f3b96e6ef1b9d77097f9b8a44afdfa5b2e4666 Mon Sep 17 00:00:00 2001 From: kasakrisz <33458261+kasakrisz@users.noreply.github.com> Date: Tue, 3 Jul 2018 09:56:36 +0200 Subject: [PATCH] AMBARI-24230. [Logsearch] Update hdfs configuration from Configuration Editor fails with 400 Error (#1660) --- .../inputconfig/impl/InputConfigGson.java | 2 +- .../impl/MapFieldCopyDescriptorImpl.java | 2 +- .../impl/MapFieldNameDescriptorImpl.java | 8 +- .../impl/MapFieldValueDescriptorImpl.java | 8 +- .../impl/PostMapValuesAdapter.java | 8 +- .../logsearch/domain/StoryDataRegistry.java | 24 +++ .../ambari/logsearch/domain/WebClient.java | 79 ++++++++ .../logsearch/steps/LogSearchApiSteps.java | 46 ++--- .../steps/LogSearchConfigApiSteps.java | 67 +++++++ .../story/LogSearchBackendStories.java | 15 +- .../backend/log_search_cofig_api_tests.story | 9 + .../src/main/resources/alias_config.json | 6 +- .../src/main/resources/filters.config.json | 54 +++--- .../samples/config/config_audit.json | 52 +++--- .../model/common/LSServerMapFieldCopy.java | 2 +- .../model/common/LSServerMapFieldName.java | 2 +- .../model/common/LSServerMapFieldValue.java | 2 +- ...LSServerPostMapValuesListDeserializer.java | 20 +- .../src/mockdata/mock-data-get.ts | 28 +-- .../shipper-conf/input.config-ambari.json | 176 +++++++++--------- .../shipper-conf/input.config-hdfs.json | 52 +++--- .../shipper-conf/input.config-hst.json | 2 +- 22 files changed, 413 insertions(+), 251 deletions(-) create mode 100644 ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/WebClient.java create mode 100644 ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchConfigApiSteps.java create mode 100644 ambari-logsearch-it/src/test/resources/stories/backend/log_search_cofig_api_tests.story diff --git a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java index 3b78affcbe8..a7d133354ad 100644 --- a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java +++ b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java @@ -27,7 +27,7 @@ import com.google.gson.GsonBuilder; /** - * Helper class to convert betweeb json string and InputConfig class. + * Helper class to convert between json string and InputConfig class. */ public class InputConfigGson { public static Gson gson; diff --git a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java index e7b8fdf6818..ca7a6aa90d9 100644 --- a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java +++ b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java @@ -33,7 +33,7 @@ public class MapFieldCopyDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldCopyDescriptor { @Override public String getJsonName() { - return "map_fieldcopy"; + return "map_field_copy"; } @ShipperConfigElementDescription( diff --git a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java index e4663641f31..7fdaa21b348 100644 --- a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java +++ b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java @@ -28,22 +28,22 @@ @ShipperConfigTypeDescription( name = "Map Field Name", - description = "The name of the mapping element should be map_fieldname. The value json element should contain the following parameter:" + description = "The name of the mapping element should be map_field_name. The value json element should contain the following parameter:" ) public class MapFieldNameDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldNameDescriptor { @Override public String getJsonName() { - return "map_fieldname"; + return "map_field_name"; } @ShipperConfigElementDescription( - path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldname/new_field_name", + path = "/filter/[]/post_map_values/{field_name}/[]/map_field_name/new_field_name", type = "string", description = "The name of the renamed field", examples = {"new_name"} ) @Expose - @SerializedName("new_fieldname") + @SerializedName("new_field_name") private String newFieldName; @Override diff --git a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java index a80a994434d..ba4686fdcec 100644 --- a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java +++ b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java @@ -28,16 +28,16 @@ @ShipperConfigTypeDescription( name = "Map Field Value", - description = "The name of the mapping element should be map_fieldvalue. The value json element should contain the following parameter:" + description = "The name of the mapping element should be map_field_value. The value json element should contain the following parameter:" ) public class MapFieldValueDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldValueDescriptor { @Override public String getJsonName() { - return "map_fieldvalue"; + return "map_field_value"; } @ShipperConfigElementDescription( - path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldvalue/pre_value", + path = "/filter/[]/post_map_values/{field_name}/[]/map_field_value/pre_value", type = "string", description = "The value that the field must match (ignoring case) to be mapped", examples = {"old_value"} @@ -47,7 +47,7 @@ public String getJsonName() { private String preValue; @ShipperConfigElementDescription( - path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldvalue/post_value", + path = "/filter/[]/post_map_values/{field_name}/[]/map_field_value/post_value", type = "string", description = "The value to which the field is modified to", examples = {"new_value"} diff --git a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java index 67810665441..f1ed2bdf419 100644 --- a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java +++ b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java @@ -57,13 +57,13 @@ private PostMapValuesImpl createPostMapValues(JsonElement e, JsonDeserialization case "map_date": mappers.add(context.deserialize(m.getValue(), MapDateDescriptorImpl.class)); break; - case "map_fieldcopy": + case "map_field_copy": mappers.add(context.deserialize(m.getValue(), MapFieldCopyDescriptorImpl.class)); break; - case "map_fieldname": + case "map_field_name": mappers.add(context.deserialize(m.getValue(), MapFieldNameDescriptorImpl.class)); break; - case "map_fieldvalue": + case "map_field_value": mappers.add(context.deserialize(m.getValue(), MapFieldValueDescriptorImpl.class)); break; case "map_anonymize": @@ -97,7 +97,7 @@ public JsonElement serialize(List src, Type typeOfSrc, JsonSe private JsonElement createMapperObject(PostMapValuesImpl postMapValues, JsonSerializationContext context) { JsonObject jsonObject = new JsonObject(); for (MapFieldDescriptor m : postMapValues.getMappers()) { - jsonObject.add(((MapFieldDescriptorImpl)m).getJsonName(), context.serialize(m)); + jsonObject.add(m.getJsonName(), context.serialize(m)); } return jsonObject; } diff --git a/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java index 58398819844..b173ff15655 100644 --- a/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java +++ b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/StoryDataRegistry.java @@ -18,12 +18,29 @@ */ package org.apache.ambari.logsearch.domain; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputAdapter; import org.apache.solr.client.solrj.SolrClient; import org.jbehave.web.selenium.WebDriverProvider; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + public class StoryDataRegistry { public static final StoryDataRegistry INSTANCE = new StoryDataRegistry(); + public static final String CLUSTER = "cl1"; + public static final String LOGSEARCH_GLOBAL_CONFIG = "[\n" + + " {\n" + + " \"add_fields\": {\n" + + " \"cluster\": \""+ CLUSTER +"\"\n" + + " },\n" + + " \"source\": \"file\",\n" + + " \"tail\": \"true\",\n" + + " \"gen_event_md5\": \"true\"\n" + + " }\n" + + "]"; + + private SolrClient solrClient; private boolean logsearchContainerStarted = false; private String dockerHost; @@ -38,6 +55,9 @@ public class StoryDataRegistry { private WebDriverProvider webDriverProvider; private StoryDataRegistry() { + JsonParser jsonParser = new JsonParser(); + JsonElement globalConfigJsonElement = jsonParser.parse(LOGSEARCH_GLOBAL_CONFIG); + InputAdapter.setGlobalConfigs(globalConfigJsonElement.getAsJsonArray()); } public String getDockerHost() { @@ -115,4 +135,8 @@ public String getShellScriptFolder() { public void setShellScriptFolder(String shellScriptFolder) { this.shellScriptFolder = shellScriptFolder; } + + public WebClient logsearchClient() { + return new WebClient(dockerHost, logsearchPort); + } } diff --git a/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/WebClient.java b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/WebClient.java new file mode 100644 index 00000000000..1c14cd08ac1 --- /dev/null +++ b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/domain/WebClient.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.domain; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; + +import org.glassfish.jersey.client.JerseyClient; +import org.glassfish.jersey.client.JerseyClientBuilder; +import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class WebClient { + private static Logger LOG = LoggerFactory.getLogger(WebClient.class); + + private final String host; + private final int port; + + public WebClient(String host, int port) { + this.host = host; + this.port = port; + } + + public String get(String path) { + JerseyClient jerseyClient = JerseyClientBuilder.createClient(); + HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basicBuilder() + .credentials("admin", "admin") + .build(); + jerseyClient.register(authFeature); + + String url = String.format("http://%s:%d%s", host, port, path); + + LOG.info("Url: {}", url); + + WebTarget target = jerseyClient.target(url); + Invocation.Builder invocationBuilder = target.request(MediaType.APPLICATION_JSON_TYPE); + return invocationBuilder.get().readEntity(String.class); + } + + public String put(String path, String requestBody) { + JerseyClient jerseyClient = JerseyClientBuilder.createClient(); + HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basicBuilder() + .credentials("admin", "admin") + .build(); + jerseyClient.register(authFeature); + + String url = String.format("http://%s:%d%s", host, port, path); + + LOG.info("Url: {}", url); + + WebTarget target = jerseyClient.target(url); + Invocation.Builder invocationBuilder = target.request(MediaType.APPLICATION_JSON_TYPE); + String response = invocationBuilder.put(Entity.entity(requestBody, MediaType.APPLICATION_JSON_TYPE)).readEntity(String.class); + + LOG.info("Response: {}", response); + + return response; + } + +} diff --git a/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchApiSteps.java b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchApiSteps.java index 90fff949699..9e94a4920cc 100644 --- a/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchApiSteps.java +++ b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchApiSteps.java @@ -18,14 +18,15 @@ */ package org.apache.ambari.logsearch.steps; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.zjsonpatch.JsonDiff; -import com.google.common.io.Resources; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + import org.apache.ambari.logsearch.domain.StoryDataRegistry; -import org.glassfish.jersey.client.JerseyClient; -import org.glassfish.jersey.client.JerseyClientBuilder; -import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; import org.jbehave.core.annotations.Named; import org.jbehave.core.annotations.Then; import org.jbehave.core.annotations.When; @@ -33,16 +34,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.ws.rs.client.Invocation; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.MediaType; -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.zjsonpatch.JsonDiff; +import com.google.common.io.Resources; public class LogSearchApiSteps { @@ -52,22 +47,7 @@ public class LogSearchApiSteps { @When("LogSearch api query sent: ") public void sendApiQuery(@Named("apiQuery") String apiQuery) { - JerseyClient jerseyClient = JerseyClientBuilder.createClient(); - HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basicBuilder() - .credentials("admin", "admin") - .build(); - jerseyClient.register(authFeature); - - String logsearchUrl = String.format("http://%s:%d%s", - StoryDataRegistry.INSTANCE.getDockerHost(), - StoryDataRegistry.INSTANCE.getLogsearchPort(), - apiQuery); - - LOG.info("Url: {}", logsearchUrl); - - WebTarget target = jerseyClient.target(logsearchUrl); - Invocation.Builder invocationBuilder = target.request(MediaType.APPLICATION_JSON_TYPE); - response = invocationBuilder.get().readEntity(String.class); + response = StoryDataRegistry.INSTANCE.logsearchClient().get(apiQuery); } diff --git a/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchConfigApiSteps.java b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchConfigApiSteps.java new file mode 100644 index 00000000000..1a3ef024148 --- /dev/null +++ b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchConfigApiSteps.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.steps; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.not; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigGson; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigImpl; +import org.apache.ambari.logsearch.domain.StoryDataRegistry; +import org.hamcrest.Matchers; +import org.jbehave.core.annotations.Then; +import org.jbehave.core.annotations.When; + +public class LogSearchConfigApiSteps { + private String response; + private InputConfig inputConfig; + + @When("LogSearch api request sent: $url") + public String sendApiRequest(String url) { + response = StoryDataRegistry.INSTANCE.logsearchClient().get(url); + return response; + } + + @Then("Result is an input.config of $inputConfigType with log file path $logFilePath") + public void checkInputConfig(String inputConfigType, String logFilePath) { + checkInputConfig(response, inputConfigType, logFilePath); + } + + public void checkInputConfig(String result, String type, String path) { + inputConfig = InputConfigGson.gson.fromJson(response, InputConfigImpl.class); + assertThat(inputConfig.getInput(), is(not(Matchers.nullValue()))); + assertThat(inputConfig.getInput(), hasSize(1)); + assertThat(inputConfig.getInput().get(0).getType(), is(type)); + assertThat(inputConfig.getInput().get(0).getPath(), is(path)); + } + + @When("Update input config of $inputConfigType path to $logFilePath at $url") + public void changeAndPut(String inputConfigType, String logFilePath, String url) { + String putRequest = response.replace(inputConfig.getInput().get(0).getPath(), logFilePath); + String putResponse = StoryDataRegistry.INSTANCE.logsearchClient().put( + url, putRequest); + assertThat(putResponse, is("")); + + String getResponse = sendApiRequest(url); + checkInputConfig(getResponse, inputConfigType, logFilePath); + } +} diff --git a/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchBackendStories.java b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchBackendStories.java index fa7a527c235..127b67a96d7 100644 --- a/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchBackendStories.java +++ b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchBackendStories.java @@ -18,12 +18,12 @@ */ package org.apache.ambari.logsearch.story; -import com.google.common.base.Predicate; -import com.google.common.collect.Collections2; -import com.google.common.collect.Lists; +import java.util.List; + import org.apache.ambari.logsearch.steps.LogSearchApiSteps; -import org.apache.ambari.logsearch.steps.SolrSteps; +import org.apache.ambari.logsearch.steps.LogSearchConfigApiSteps; import org.apache.ambari.logsearch.steps.LogSearchDockerSteps; +import org.apache.ambari.logsearch.steps.SolrSteps; import org.jbehave.core.configuration.Configuration; import org.jbehave.core.configuration.MostUsefulConfiguration; import org.jbehave.core.junit.JUnitStories; @@ -33,7 +33,9 @@ import org.jbehave.core.steps.InstanceStepsFactory; import org.junit.Test; -import java.util.List; +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; +import com.google.common.collect.Lists; public class LogSearchBackendStories extends JUnitStories { @@ -53,7 +55,8 @@ public InjectableStepsFactory stepsFactory() { return new InstanceStepsFactory(configuration(), new LogSearchDockerSteps(), new SolrSteps(), - new LogSearchApiSteps()); + new LogSearchApiSteps(), + new LogSearchConfigApiSteps()); } @Test diff --git a/ambari-logsearch-it/src/test/resources/stories/backend/log_search_cofig_api_tests.story b/ambari-logsearch-it/src/test/resources/stories/backend/log_search_cofig_api_tests.story new file mode 100644 index 00000000000..26be38cabd0 --- /dev/null +++ b/ambari-logsearch-it/src/test/resources/stories/backend/log_search_cofig_api_tests.story @@ -0,0 +1,9 @@ +Scenario: scenario description + +Given logsearch docker container +When LogSearch api request sent: /api/v1/shipper/input/cl1/services/ambari +Then Result is an input.config of ambari_audit with log file path /root/test-logs/ambari-server/ambari-audit.log + +Given logsearch docker container +When Update input config of ambari_audit path to /root/test-logs/ambari-server/ambari-audit.log.1 at /api/v1/shipper/input/cl1/services/ambari +Then Result is an input.config of ambari_audit with log file path /root/test-logs/ambari-server/ambari-audit.log.1 \ No newline at end of file diff --git a/ambari-logsearch-logfeeder/src/main/resources/alias_config.json b/ambari-logsearch-logfeeder/src/main/resources/alias_config.json index 4656b5b85d0..3da5b0ad669 100644 --- a/ambari-logsearch-logfeeder/src/main/resources/alias_config.json +++ b/ambari-logsearch-logfeeder/src/main/resources/alias_config.json @@ -25,13 +25,13 @@ "map_date": { "klass": "org.apache.ambari.logfeeder.mapper.MapperDate" }, - "map_fieldcopy": { + "map_field_copy": { "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldCopy" }, - "map_fieldname": { + "map_field_name": { "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldName" }, - "map_fieldvalue": { + "map_field_value": { "klass": "org.apache.ambari.logfeeder.mapper.MapperFieldValue" }, "map_anonymize": { diff --git a/ambari-logsearch-logfeeder/src/main/resources/filters.config.json b/ambari-logsearch-logfeeder/src/main/resources/filters.config.json index 9493c6c2633..d9006da0332 100644 --- a/ambari-logsearch-logfeeder/src/main/resources/filters.config.json +++ b/ambari-logsearch-logfeeder/src/main/resources/filters.config.json @@ -155,7 +155,7 @@ }, "level":{ - "map_fieldvalue":{ + "map_field_value":{ "pre_value":"WARNING", "post_value":"WARN" } @@ -509,55 +509,55 @@ "field_split":"\t", "post_map_values":{ "src":{ - "map_fieldname":{ - "new_fieldname":"resource" + "map_field_name":{ + "new_field_name":"resource" } }, "ip":{ - "map_fieldname":{ - "new_fieldname":"cliIP" + "map_field_name":{ + "new_field_name":"cliIP" } }, "allowed":[ { - "map_fieldvalue":{ + "map_field_value":{ "pre_value":"true", "post_value":"1" } }, { - "map_fieldvalue":{ + "map_field_value":{ "pre_value":"false", "post_value":"0" } }, { - "map_fieldname":{ - "new_fieldname":"result" + "map_field_name":{ + "new_field_name":"result" } } ], "cmd":{ - "map_fieldname":{ - "new_fieldname":"action" + "map_field_name":{ + "new_field_name":"action" } }, "proto":{ - "map_fieldname":{ - "new_fieldname":"cliType" + "map_field_name":{ + "new_field_name":"cliType" } }, "callerContext":{ - "map_fieldname":{ - "new_fieldname":"req_caller_id" + "map_field_name":{ + "new_field_name":"req_caller_id" } } @@ -582,38 +582,38 @@ "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", "post_map_values":{ "user":{ - "map_fieldname":{ - "new_fieldname":"reqUser" + "map_field_name":{ + "new_field_name":"reqUser" } }, "x_user":{ - "map_fieldname":{ - "new_fieldname":"reqUser" + "map_field_name":{ + "new_field_name":"reqUser" } }, "p_user":{ - "map_fieldname":{ - "new_fieldname":"reqUser" + "map_field_name":{ + "new_field_name":"reqUser" } }, "k_user":{ - "map_fieldname":{ - "new_fieldname":"proxyUsers" + "map_field_name":{ + "new_field_name":"proxyUsers" } }, "p_authType":{ - "map_fieldname":{ - "new_fieldname":"authType" + "map_field_name":{ + "new_field_name":"authType" } }, "k_authType":{ - "map_fieldname":{ - "new_fieldname":"proxyAuthType" + "map_field_name":{ + "new_field_name":"proxyAuthType" } } diff --git a/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json b/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json index 05f3bff1a41..5d827125731 100644 --- a/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json +++ b/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json @@ -39,52 +39,52 @@ "field_split": "\t", "post_map_values": { "src": { - "map_fieldname": { - "new_fieldname": "resource" + "map_field_name": { + "new_field_name": "resource" } }, "ip": { - "map_fieldname": { - "new_fieldname": "cliIP" + "map_field_name": { + "new_field_name": "cliIP" } }, "allowed": [{ - "map_fieldvalue": { + "map_field_value": { "pre_value": "true", "post_value": "1" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "false", "post_value": "0" } }, { - "map_fieldname": { - "new_fieldname": "result" + "map_field_name": { + "new_field_name": "result" } } ], "cmd": { - "map_fieldname": { - "new_fieldname": "action" + "map_field_name": { + "new_field_name": "action" } }, "proto": { - "map_fieldname": { - "new_fieldname": "cliType" + "map_field_name": { + "new_field_name": "cliType" } }, "callerContext": { - "map_fieldname": { - "new_fieldname": "req_caller_id" + "map_field_name": { + "new_field_name": "req_caller_id" } } @@ -108,38 +108,38 @@ "message_pattern": "%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", "post_map_values": { "user": { - "map_fieldname": { - "new_fieldname": "reqUser" + "map_field_name": { + "new_field_name": "reqUser" } }, "x_user": { - "map_fieldname": { - "new_fieldname": "reqUser" + "map_field_name": { + "new_field_name": "reqUser" } }, "p_user": { - "map_fieldname": { - "new_fieldname": "reqUser" + "map_field_name": { + "new_field_name": "reqUser" } }, "k_user": { - "map_fieldname": { - "new_fieldname": "proxyUsers" + "map_field_name": { + "new_field_name": "proxyUsers" } }, "p_authType": { - "map_fieldname": { - "new_fieldname": "authType" + "map_field_name": { + "new_field_name": "authType" } }, "k_authType": { - "map_fieldname": { - "new_fieldname": "proxyAuthType" + "map_field_name": { + "new_field_name": "proxyAuthType" } } diff --git a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldCopy.java b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldCopy.java index ab12698249a..4da55016724 100644 --- a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldCopy.java +++ b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldCopy.java @@ -31,7 +31,7 @@ public class LSServerMapFieldCopy extends LSServerMapField { @Override public String getName() { - return "map_fieldcopy"; + return "map_field_copy"; } @NotNull diff --git a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldName.java b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldName.java index ab7311c6136..8468b380720 100644 --- a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldName.java +++ b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldName.java @@ -31,7 +31,7 @@ public class LSServerMapFieldName extends LSServerMapField { @Override public String getName() { - return "map_fieldname"; + return "map_field_name"; } @NotNull diff --git a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldValue.java b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldValue.java index a7d3949dc94..389fcb3dfac 100644 --- a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldValue.java +++ b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapFieldValue.java @@ -31,7 +31,7 @@ public class LSServerMapFieldValue extends LSServerMapField { @Override public String getName() { - return "map_fieldvalue"; + return "map_field_value"; } @NotNull diff --git a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerPostMapValuesListDeserializer.java b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerPostMapValuesListDeserializer.java index 258f64ad022..de21e20aab0 100644 --- a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerPostMapValuesListDeserializer.java +++ b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerPostMapValuesListDeserializer.java @@ -36,7 +36,7 @@ public class LSServerPostMapValuesListDeserializer extends JsonDeserializer { @Override public LSServerPostMapValuesList deserialize(JsonParser jp, DeserializationContext ctxt) - throws IOException, JsonProcessingException { + throws IOException { ObjectCodec oc = jp.getCodec(); JsonNode node = oc.readTree(jp); @@ -49,24 +49,24 @@ public LSServerPostMapValuesList deserialize(JsonParser jp, DeserializationConte JsonNode mapperProperties = mapperData.getValue(); switch (mapperType) { case "map_date" : - LSServerMapDate mapDate = oc.treeToValue((TreeNode)mapperProperties, LSServerMapDate.class); + LSServerMapDate mapDate = oc.treeToValue(mapperProperties, LSServerMapDate.class); mappers.add(mapDate); break; - case "map_fieldname" : - LSServerMapFieldName mapFieldName = oc.treeToValue((TreeNode)mapperProperties, LSServerMapFieldName.class); + case "map_field_name" : + LSServerMapFieldName mapFieldName = oc.treeToValue(mapperProperties, LSServerMapFieldName.class); mappers.add(mapFieldName); break; - case "map_fieldvalue" : - LSServerMapFieldValue mapFieldValue = oc.treeToValue((TreeNode)mapperProperties, LSServerMapFieldValue.class); + case "map_field_value" : + LSServerMapFieldValue mapFieldValue = oc.treeToValue(mapperProperties, LSServerMapFieldValue.class); mappers.add(mapFieldValue); break; - case "map_fieldcopy" : - LSServerMapFieldCopy mapFieldCopy = oc.treeToValue((TreeNode)mapperProperties, LSServerMapFieldCopy.class); + case "map_field_copy" : + LSServerMapFieldCopy mapFieldCopy = oc.treeToValue(mapperProperties, LSServerMapFieldCopy.class); mappers.add(mapFieldCopy); break; case "map_anonymize" : - LSServerMapFieldAnonymize mapAnonyimize = oc.treeToValue((TreeNode)mapperProperties, LSServerMapFieldAnonymize.class); - mappers.add(mapAnonyimize); + LSServerMapFieldAnonymize mapAnonymize = oc.treeToValue(mapperProperties, LSServerMapFieldAnonymize.class); + mappers.add(mapAnonymize); break; } } diff --git a/ambari-logsearch-web/src/mockdata/mock-data-get.ts b/ambari-logsearch-web/src/mockdata/mock-data-get.ts index 1c2b76c6ba2..8511cc46c4e 100644 --- a/ambari-logsearch-web/src/mockdata/mock-data-get.ts +++ b/ambari-logsearch-web/src/mockdata/mock-data-get.ts @@ -1954,54 +1954,54 @@ export const mockDataGet = { 'post_map_values': { 'callerContext': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'req_caller_id' } } ], 'src': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'resource' } } ], 'allowed': [ { - 'map_fieldvalue': { + 'map_field_value': { 'pre_value': 'true', 'post_value': '1' } }, { - 'map_fieldvalue': { + 'map_field_value': { 'pre_value': 'false', 'post_value': '0' } }, { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'result' } } ], 'ip': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'cliIP' } } ], 'proto': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'cliType' } } ], 'cmd': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'action' } } @@ -2025,42 +2025,42 @@ export const mockDataGet = { 'post_map_values': { 'k_authType': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'proxyAuthType' } } ], 'p_authType': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'authType' } } ], 'x_user': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'reqUser' } } ], 'k_user': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'proxyUsers' } } ], 'p_user': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'reqUser' } } ], 'user': [ { - 'map_fieldname': { + 'map_field_name': { 'new_field_name': 'reqUser' } } diff --git a/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json b/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json index 081e16e2e7f..66a58f15170 100644 --- a/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json +++ b/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json @@ -49,324 +49,324 @@ "value_borders": "()", "post_map_values": { "User": { - "map_fieldvalue": { + "map_field_value": { "pre_value": "null", "post_value": "unknown" }, - "map_fieldname": { - "new_fieldname": "reqUser" + "map_field_name": { + "new_field_name": "reqUser" } }, "Hostname": { - "map_fieldname": { - "new_fieldname": "host" + "map_field_name": { + "new_field_name": "host" } }, "Host name": { - "map_fieldname": { - "new_fieldname": "host" + "map_field_name": { + "new_field_name": "host" } }, "RemoteIp": { - "map_fieldname": { - "new_fieldname": "cliIP" + "map_field_name": { + "new_field_name": "cliIP" } }, "RequestType": { - "map_fieldname": { - "new_fieldname": "cliType" + "map_field_name": { + "new_field_name": "cliType" } }, "Operation": { - "map_fieldname": { - "new_fieldname": "action" + "map_field_name": { + "new_field_name": "action" } }, "url": { - "map_fieldname": { - "new_fieldname": "resource" + "map_field_name": { + "new_field_name": "resource" } }, "ResourcePath": { - "map_fieldname": { - "new_fieldname": "resource" + "map_field_name": { + "new_field_name": "resource" } }, "Cluster name": { - "map_fieldname": { - "new_fieldname": "cluster" + "map_field_name": { + "new_field_name": "cluster" } }, "Reason": { - "map_fieldname": { - "new_fieldname": "reason" + "map_field_name": { + "new_field_name": "reason" } }, "Base URL": { - "map_fieldname": { - "new_fieldname": "ws_base_url" + "map_field_name": { + "new_field_name": "ws_base_url" } }, "Command": { - "map_fieldvalue": { + "map_field_value": { "pre_value": "null", "post_value": "unknown" }, - "map_fieldname": { - "new_fieldname": "ws_command" + "map_field_name": { + "new_field_name": "ws_command" } }, "Component": { - "map_fieldname": { - "new_fieldname": "ws_component" + "map_field_name": { + "new_field_name": "ws_component" } }, "Details": { - "map_fieldname": { - "new_fieldname": "ws_details" + "map_field_name": { + "new_field_name": "ws_details" } }, "Display name": { - "map_fieldvalue": { + "map_field_value": { "pre_value": "null", "post_value": "unknown" }, - "map_fieldname": { - "new_fieldname": "ws_display_name" + "map_field_name": { + "new_field_name": "ws_display_name" } }, "OS": { - "map_fieldname": { - "new_fieldname": "ws_os" + "map_field_name": { + "new_field_name": "ws_os" } }, "Repo id": { - "map_fieldname": { - "new_fieldname": "ws_repo_id" + "map_field_name": { + "new_field_name": "ws_repo_id" } }, "Repo version": { - "map_fieldvalue": { + "map_field_value": { "pre_value": "null", "post_value": "unknown" }, - "map_fieldname": { - "new_fieldname": "ws_repo_version" + "map_field_name": { + "new_field_name": "ws_repo_version" } }, "Repositories": { - "map_fieldname": { - "new_fieldname": "ws_repositories" + "map_field_name": { + "new_field_name": "ws_repositories" } }, "RequestId": { - "map_fieldname": { - "new_fieldname": "ws_request_id" + "map_field_name": { + "new_field_name": "ws_request_id" } }, "Roles": { - "map_fieldname": { - "new_fieldname": "ws_roles" + "map_field_name": { + "new_field_name": "ws_roles" } }, "Stack": { - "map_fieldname": { - "new_fieldname": "ws_stack" + "map_field_name": { + "new_field_name": "ws_stack" } }, "Stack version": { - "map_fieldname": { - "new_fieldname": "ws_stack_version" + "map_field_name": { + "new_field_name": "ws_stack_version" } }, "TaskId": { - "map_fieldname": { - "new_fieldname": "ws_task_id" + "map_field_name": { + "new_field_name": "ws_task_id" } }, "VersionNote": { - "map_fieldvalue": { + "map_field_value": { "pre_value": "null", "post_value": "unknown" }, - "map_fieldname": { - "new_fieldname": "ws_version_note" + "map_field_name": { + "new_field_name": "ws_version_note" } }, "VersionNumber": { - "map_fieldvalue": { + "map_field_value": { "pre_value": "null", "post_value": "unknown" }, - "map_fieldname": { - "new_fieldname": "ws_version_number" + "map_field_name": { + "new_field_name": "ws_version_number" } }, "Status": [ { - "map_fieldcopy": { + "map_field_copy": { "copy_name": "ws_status" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "Success", "post_value": "1" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "Successfully queued", "post_value": "1" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "QUEUED", "post_value": "1" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "PENDING", "post_value": "1" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "COMPLETED", "post_value": "1" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "IN_PROGRESS", "post_value": "1" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "Failed", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "Failed to queue", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "HOLDING", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "HOLDING_FAILED", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "HOLDING_TIMEDOUT", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "FAILED", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "TIMEDOUT", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "ABORTED", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "SKIPPED_FAILED", "post_value": "0" } }, { - "map_fieldname": { - "new_fieldname": "result" + "map_field_name": { + "new_field_name": "result" } } ], "ResultStatus": [ { - "map_fieldcopy": { + "map_field_copy": { "copy_name": "ws_result_status" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "200 OK", "post_value": "1" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "201 Created", "post_value": "1" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "202 Accepted", "post_value": "1" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "400 Bad Request", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "401 Unauthorized", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "403 Forbidden", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "404 Not Found", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "409 Resource Conflict", "post_value": "0" } }, { - "map_fieldvalue": { + "map_field_value": { "pre_value": "500 Internal Server Error", "post_value": "0" } }, { - "map_fieldname": { - "new_fieldname": "result" + "map_field_name": { + "new_field_name": "result" } } ] diff --git a/docker/test-config/logfeeder/shipper-conf/input.config-hdfs.json b/docker/test-config/logfeeder/shipper-conf/input.config-hdfs.json index d975b0d4047..871c778bd6f 100644 --- a/docker/test-config/logfeeder/shipper-conf/input.config-hdfs.json +++ b/docker/test-config/logfeeder/shipper-conf/input.config-hdfs.json @@ -56,55 +56,55 @@ "field_split":"\t", "post_map_values":{ "src":{ - "map_fieldname":{ - "new_fieldname":"resource" + "map_field_name":{ + "new_field_name":"resource" } }, "ip":{ - "map_fieldname":{ - "new_fieldname":"cliIP" + "map_field_name":{ + "new_field_name":"cliIP" } }, "allowed":[ { - "map_fieldvalue":{ + "map_field_value":{ "pre_value":"true", "post_value":"1" } }, { - "map_fieldvalue":{ + "map_field_value":{ "pre_value":"false", "post_value":"0" } }, { - "map_fieldname":{ - "new_fieldname":"result" + "map_field_name":{ + "new_field_name":"result" } } ], "cmd":{ - "map_fieldname":{ - "new_fieldname":"action" + "map_field_name":{ + "new_field_name":"action" } }, "proto":{ - "map_fieldname":{ - "new_fieldname":"cliType" + "map_field_name":{ + "new_field_name":"cliType" } }, "callerContext":{ - "map_fieldname":{ - "new_fieldname":"req_caller_id" + "map_field_name":{ + "new_field_name":"req_caller_id" } } @@ -129,38 +129,38 @@ "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", "post_map_values":{ "user":{ - "map_fieldname":{ - "new_fieldname":"reqUser" + "map_field_name":{ + "new_field_name":"reqUser" } }, "x_user":{ - "map_fieldname":{ - "new_fieldname":"reqUser" + "map_field_name":{ + "new_field_name":"reqUser" } }, "p_user":{ - "map_fieldname":{ - "new_fieldname":"reqUser" + "map_field_name":{ + "new_field_name":"reqUser" } }, "k_user":{ - "map_fieldname":{ - "new_fieldname":"proxyUsers" + "map_field_name":{ + "new_field_name":"proxyUsers" } }, "p_authType":{ - "map_fieldname":{ - "new_fieldname":"authType" + "map_field_name":{ + "new_field_name":"authType" } }, "k_authType":{ - "map_fieldname":{ - "new_fieldname":"proxyAuthType" + "map_field_name":{ + "new_field_name":"proxyAuthType" } } diff --git a/docker/test-config/logfeeder/shipper-conf/input.config-hst.json b/docker/test-config/logfeeder/shipper-conf/input.config-hst.json index 9ec78d6372a..b9c4252100a 100644 --- a/docker/test-config/logfeeder/shipper-conf/input.config-hst.json +++ b/docker/test-config/logfeeder/shipper-conf/input.config-hst.json @@ -27,7 +27,7 @@ } }, "level": { - "map_fieldvalue": { + "map_field_value": { "pre_value": "WARNING", "post_value": "WARN" }