Skip to content

Commit

Permalink
AMBARI-24230. [Logsearch] Update hdfs configuration from Configuratio…
Browse files Browse the repository at this point in the history
…n Editor fails with 400 Error (#1660)
  • Loading branch information
kasakrisz authored and adoroszlai committed Jul 3, 2018
1 parent 53b92a8 commit e5f3b96
Show file tree
Hide file tree
Showing 22 changed files with 413 additions and 251 deletions.
Expand Up @@ -27,7 +27,7 @@
import com.google.gson.GsonBuilder;

/**
* Helper class to convert betweeb json string and InputConfig class.
* Helper class to convert between json string and InputConfig class.
*/
public class InputConfigGson {
public static Gson gson;
Expand Down
Expand Up @@ -33,7 +33,7 @@
public class MapFieldCopyDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldCopyDescriptor {
@Override
public String getJsonName() {
return "map_fieldcopy";
return "map_field_copy";
}

@ShipperConfigElementDescription(
Expand Down
Expand Up @@ -28,22 +28,22 @@

@ShipperConfigTypeDescription(
name = "Map Field Name",
description = "The name of the mapping element should be map_fieldname. The value json element should contain the following parameter:"
description = "The name of the mapping element should be map_field_name. The value json element should contain the following parameter:"
)
public class MapFieldNameDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldNameDescriptor {
@Override
public String getJsonName() {
return "map_fieldname";
return "map_field_name";
}

@ShipperConfigElementDescription(
path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldname/new_field_name",
path = "/filter/[]/post_map_values/{field_name}/[]/map_field_name/new_field_name",
type = "string",
description = "The name of the renamed field",
examples = {"new_name"}
)
@Expose
@SerializedName("new_fieldname")
@SerializedName("new_field_name")
private String newFieldName;

@Override
Expand Down
Expand Up @@ -28,16 +28,16 @@

@ShipperConfigTypeDescription(
name = "Map Field Value",
description = "The name of the mapping element should be map_fieldvalue. The value json element should contain the following parameter:"
description = "The name of the mapping element should be map_field_value. The value json element should contain the following parameter:"
)
public class MapFieldValueDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldValueDescriptor {
@Override
public String getJsonName() {
return "map_fieldvalue";
return "map_field_value";
}

@ShipperConfigElementDescription(
path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldvalue/pre_value",
path = "/filter/[]/post_map_values/{field_name}/[]/map_field_value/pre_value",
type = "string",
description = "The value that the field must match (ignoring case) to be mapped",
examples = {"old_value"}
Expand All @@ -47,7 +47,7 @@ public String getJsonName() {
private String preValue;

@ShipperConfigElementDescription(
path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldvalue/post_value",
path = "/filter/[]/post_map_values/{field_name}/[]/map_field_value/post_value",
type = "string",
description = "The value to which the field is modified to",
examples = {"new_value"}
Expand Down
Expand Up @@ -57,13 +57,13 @@ private PostMapValuesImpl createPostMapValues(JsonElement e, JsonDeserialization
case "map_date":
mappers.add(context.deserialize(m.getValue(), MapDateDescriptorImpl.class));
break;
case "map_fieldcopy":
case "map_field_copy":
mappers.add(context.deserialize(m.getValue(), MapFieldCopyDescriptorImpl.class));
break;
case "map_fieldname":
case "map_field_name":
mappers.add(context.deserialize(m.getValue(), MapFieldNameDescriptorImpl.class));
break;
case "map_fieldvalue":
case "map_field_value":
mappers.add(context.deserialize(m.getValue(), MapFieldValueDescriptorImpl.class));
break;
case "map_anonymize":
Expand Down Expand Up @@ -97,7 +97,7 @@ public JsonElement serialize(List<PostMapValuesImpl> src, Type typeOfSrc, JsonSe
private JsonElement createMapperObject(PostMapValuesImpl postMapValues, JsonSerializationContext context) {
JsonObject jsonObject = new JsonObject();
for (MapFieldDescriptor m : postMapValues.getMappers()) {
jsonObject.add(((MapFieldDescriptorImpl)m).getJsonName(), context.serialize(m));
jsonObject.add(m.getJsonName(), context.serialize(m));
}
return jsonObject;
}
Expand Down
Expand Up @@ -18,12 +18,29 @@
*/
package org.apache.ambari.logsearch.domain;

import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputAdapter;
import org.apache.solr.client.solrj.SolrClient;
import org.jbehave.web.selenium.WebDriverProvider;

import com.google.gson.JsonElement;
import com.google.gson.JsonParser;

public class StoryDataRegistry {
public static final StoryDataRegistry INSTANCE = new StoryDataRegistry();

public static final String CLUSTER = "cl1";
public static final String LOGSEARCH_GLOBAL_CONFIG = "[\n" +
" {\n" +
" \"add_fields\": {\n" +
" \"cluster\": \""+ CLUSTER +"\"\n" +
" },\n" +
" \"source\": \"file\",\n" +
" \"tail\": \"true\",\n" +
" \"gen_event_md5\": \"true\"\n" +
" }\n" +
"]";


private SolrClient solrClient;
private boolean logsearchContainerStarted = false;
private String dockerHost;
Expand All @@ -38,6 +55,9 @@ public class StoryDataRegistry {
private WebDriverProvider webDriverProvider;

private StoryDataRegistry() {
JsonParser jsonParser = new JsonParser();
JsonElement globalConfigJsonElement = jsonParser.parse(LOGSEARCH_GLOBAL_CONFIG);
InputAdapter.setGlobalConfigs(globalConfigJsonElement.getAsJsonArray());
}

public String getDockerHost() {
Expand Down Expand Up @@ -115,4 +135,8 @@ public String getShellScriptFolder() {
public void setShellScriptFolder(String shellScriptFolder) {
this.shellScriptFolder = shellScriptFolder;
}

public WebClient logsearchClient() {
return new WebClient(dockerHost, logsearchPort);
}
}
@@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.ambari.logsearch.domain;

import javax.ws.rs.client.Entity;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;

import org.glassfish.jersey.client.JerseyClient;
import org.glassfish.jersey.client.JerseyClientBuilder;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class WebClient {
private static Logger LOG = LoggerFactory.getLogger(WebClient.class);

private final String host;
private final int port;

public WebClient(String host, int port) {
this.host = host;
this.port = port;
}

public String get(String path) {
JerseyClient jerseyClient = JerseyClientBuilder.createClient();
HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basicBuilder()
.credentials("admin", "admin")
.build();
jerseyClient.register(authFeature);

String url = String.format("http://%s:%d%s", host, port, path);

LOG.info("Url: {}", url);

WebTarget target = jerseyClient.target(url);
Invocation.Builder invocationBuilder = target.request(MediaType.APPLICATION_JSON_TYPE);
return invocationBuilder.get().readEntity(String.class);
}

public String put(String path, String requestBody) {
JerseyClient jerseyClient = JerseyClientBuilder.createClient();
HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basicBuilder()
.credentials("admin", "admin")
.build();
jerseyClient.register(authFeature);

String url = String.format("http://%s:%d%s", host, port, path);

LOG.info("Url: {}", url);

WebTarget target = jerseyClient.target(url);
Invocation.Builder invocationBuilder = target.request(MediaType.APPLICATION_JSON_TYPE);
String response = invocationBuilder.put(Entity.entity(requestBody, MediaType.APPLICATION_JSON_TYPE)).readEntity(String.class);

LOG.info("Response: {}", response);

return response;
}

}
Expand Up @@ -18,31 +18,26 @@
*/
package org.apache.ambari.logsearch.steps;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.flipkart.zjsonpatch.JsonDiff;
import com.google.common.io.Resources;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;

import org.apache.ambari.logsearch.domain.StoryDataRegistry;
import org.glassfish.jersey.client.JerseyClient;
import org.glassfish.jersey.client.JerseyClientBuilder;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.jbehave.core.annotations.Named;
import org.jbehave.core.annotations.Then;
import org.jbehave.core.annotations.When;
import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.flipkart.zjsonpatch.JsonDiff;
import com.google.common.io.Resources;

public class LogSearchApiSteps {

Expand All @@ -52,22 +47,7 @@ public class LogSearchApiSteps {

@When("LogSearch api query sent: <apiQuery>")
public void sendApiQuery(@Named("apiQuery") String apiQuery) {
JerseyClient jerseyClient = JerseyClientBuilder.createClient();
HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basicBuilder()
.credentials("admin", "admin")
.build();
jerseyClient.register(authFeature);

String logsearchUrl = String.format("http://%s:%d%s",
StoryDataRegistry.INSTANCE.getDockerHost(),
StoryDataRegistry.INSTANCE.getLogsearchPort(),
apiQuery);

LOG.info("Url: {}", logsearchUrl);

WebTarget target = jerseyClient.target(logsearchUrl);
Invocation.Builder invocationBuilder = target.request(MediaType.APPLICATION_JSON_TYPE);
response = invocationBuilder.get().readEntity(String.class);
response = StoryDataRegistry.INSTANCE.logsearchClient().get(apiQuery);
}


Expand Down
@@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.ambari.logsearch.steps;

import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.not;

import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigGson;
import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigImpl;
import org.apache.ambari.logsearch.domain.StoryDataRegistry;
import org.hamcrest.Matchers;
import org.jbehave.core.annotations.Then;
import org.jbehave.core.annotations.When;

public class LogSearchConfigApiSteps {
private String response;
private InputConfig inputConfig;

@When("LogSearch api request sent: $url")
public String sendApiRequest(String url) {
response = StoryDataRegistry.INSTANCE.logsearchClient().get(url);
return response;
}

@Then("Result is an input.config of $inputConfigType with log file path $logFilePath")
public void checkInputConfig(String inputConfigType, String logFilePath) {
checkInputConfig(response, inputConfigType, logFilePath);
}

public void checkInputConfig(String result, String type, String path) {
inputConfig = InputConfigGson.gson.fromJson(response, InputConfigImpl.class);
assertThat(inputConfig.getInput(), is(not(Matchers.nullValue())));
assertThat(inputConfig.getInput(), hasSize(1));
assertThat(inputConfig.getInput().get(0).getType(), is(type));
assertThat(inputConfig.getInput().get(0).getPath(), is(path));
}

@When("Update input config of $inputConfigType path to $logFilePath at $url")
public void changeAndPut(String inputConfigType, String logFilePath, String url) {
String putRequest = response.replace(inputConfig.getInput().get(0).getPath(), logFilePath);
String putResponse = StoryDataRegistry.INSTANCE.logsearchClient().put(
url, putRequest);
assertThat(putResponse, is(""));

String getResponse = sendApiRequest(url);
checkInputConfig(getResponse, inputConfigType, logFilePath);
}
}

0 comments on commit e5f3b96

Please sign in to comment.