Skip to content
Permalink
Browse files
AMBARI-23142 - Add AMS Metrics publisher to Infra Solr (#556)
  • Loading branch information
kasakrisz authored and oleewere committed Apr 5, 2018
1 parent 86ecedf commit ce44a0a8f7ed388670e3433e73953e8a15d896ed
Show file tree
Hide file tree
Showing 28 changed files with 1,652 additions and 142 deletions.
@@ -196,6 +196,10 @@

<copy file="${infra.solr.plugin.dir}/target/ambari-infra-solr-plugin-${project.version}.jar"
toDir="${project.build.directory}/solr/server/solr-webapp/webapp/WEB-INF/lib/"/>
<copy file="${infra.solr.plugin.dir}/target/libs/ambari-metrics-common-${project.version}.jar"
toDir="${project.build.directory}/solr/server/solr-webapp/webapp/WEB-INF/lib/"/>
<copy file="${infra.solr.plugin.dir}/target/libs/commons-logging-1.1.1.jar"
toDir="${project.build.directory}/solr/server/solr-webapp/webapp/WEB-INF/lib/"/>
<chmod file="${project.build.directory}/solr/bin/**" perm="755"/>
<chmod file="${project.build.directory}/solr/server/scripts/**" perm="755"/>
</target>
@@ -387,6 +391,8 @@

<copy file="${infra.solr.plugin.dir}/target/ambari-infra-solr-plugin-${project.version}.jar"
toDir="${project.build.directory}/solr/server/solr-webapp/webapp/WEB-INF/lib/"/>
<copy file="${infra.solr.plugin.dir}/target/libs/ambari-metrics-common-${project.version}.jar"
toDir="${project.build.directory}/solr/server/solr-webapp/webapp/WEB-INF/lib/"/>
<chmod file="${project.build.directory}/solr/bin/**" perm="755"/>
<chmod file="${project.build.directory}/solr/server/scripts/**" perm="755"/>
</target>
@@ -89,12 +89,68 @@
<version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.sparkjava</groupId>
<artifactId>spark-core</artifactId>
<version>2.7.1</version>
<scope>test</scope>
</dependency>
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-security</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -142,6 +198,7 @@
<configuration>
<includes>
<include>**/*Stories.java</include>
<include>**/*IT.java</include>
</includes>
<systemPropertyVariables>
<log4j.configuration>file:${project.build.testOutputDirectory}/log4j.properties</log4j.configuration>
@@ -0,0 +1,107 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.ambari.infra;

import static org.apache.ambari.infra.TestUtil.doWithin;
import static org.apache.ambari.infra.TestUtil.getDockerHost;
import static org.apache.ambari.infra.TestUtil.runCommand;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Paths;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class Solr {
private static final Logger LOG = LoggerFactory.getLogger(Solr.class);
public static final String AUDIT_LOGS_COLLECTION = "audit_logs";
public static final String HADOOP_LOGS_COLLECTION = "hadoop_logs";
private static final int SOLR_PORT = 8983;

private final SolrClient solrClient;
private final String configSetPath;

public Solr() {
this("");
}

public Solr(String configSetPath) {
this.configSetPath = configSetPath;
this.solrClient = new LBHttpSolrClient.Builder().withBaseSolrUrls(String.format("http://%s:%d/solr/%s_shard1_replica1",
getDockerHost(),
SOLR_PORT,
AUDIT_LOGS_COLLECTION)).build();
}

public void waitUntilSolrIsUp() throws Exception {
try (CloseableHttpClient httpClient = HttpClientBuilder.create().setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)).build()) {
doWithin(60, "Check Solr running", () -> pingSolr(httpClient));
}
}

private boolean pingSolr(CloseableHttpClient httpClient) {
try (CloseableHttpResponse response = httpClient.execute(new HttpGet(String.format("http://%s:%d/solr/admin/collections?action=LIST", getDockerHost(), SOLR_PORT)))) {
return response.getStatusLine().getStatusCode() == 200;
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
}

public void add(SolrInputDocument solrInputDocument) {
try {
solrClient.add(solrInputDocument);
} catch (SolrServerException | IOException e) {
throw new RuntimeException(e);
}
}

public void createSolrCollection(String collectionName) {
LOG.info("Creating collection");
runCommand(new String[]{"docker", "exec", "docker_solr_1", "solr", "create_collection", "-force", "-c", collectionName, "-d", Paths.get(configSetPath, "configsets", collectionName, "conf").toString(), "-n", collectionName + "_conf"});
}

public QueryResponse query(SolrQuery query) {
try {
return solrClient.query(query);
} catch (SolrServerException | IOException e) {
throw new RuntimeException(e);
}
}

public void commit() {
try {
solrClient.commit();
} catch (SolrServerException | IOException e) {
throw new RuntimeException(e);
}
}
}
@@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.ambari.infra;

import static java.lang.System.currentTimeMillis;

import java.nio.charset.StandardCharsets;
import java.util.function.BooleanSupplier;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestUtil {
private static final Logger LOG = LoggerFactory.getLogger(TestUtil.class);

public static void doWithin(int sec, String actionName, BooleanSupplier predicate) {
doWithin(sec, actionName, () -> {
if (!predicate.getAsBoolean())
throw new RuntimeException("Predicate was false!");
});
}

public static void doWithin(int sec, String actionName, Runnable runnable) {
long start = currentTimeMillis();
Exception exception;
while (true) {
try {
runnable.run();
return;
}
catch (Exception e) {
exception = e;
}

if (currentTimeMillis() - start > sec * 1000) {
throw new AssertionError(String.format("Unable to perform action '%s' within %d seconds", actionName, sec), exception);
}
else {
LOG.info("Performing action '{}' failed. retrying...", actionName);
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
}
}

public static String getDockerHost() {
return System.getProperty("docker.host") != null ? System.getProperty("docker.host") : "localhost";
}

public static void runCommand(String[] command) {
try {
LOG.info("Exec command: {}", StringUtils.join(command, " "));
Process process = Runtime.getRuntime().exec(command);
String stdout = IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8);
LOG.info("Exec command result {}", stdout);
} catch (Exception e) {
throw new RuntimeException("Error during execute shell command: ", e);
}
}
}

0 comments on commit ce44a0a

Please sign in to comment.