Skip to content
Permalink
Browse files
AMBARI-22514, AMBARI-22653. Ambari Infra Manager: solr data exporting…
… jobs and integration test environment. (Krisztian Kasa via swagle)
  • Loading branch information
swagle committed Dec 21, 2017
1 parent ce76c05 commit 79f7f1344d197257b75a285d9fd0acfdac6c8792
Show file tree
Hide file tree
Showing 39 changed files with 1,532 additions and 209 deletions.
@@ -0,0 +1,155 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

<parent>
<artifactId>ambari-infra</artifactId>
<groupId>org.apache.ambari</groupId>
<version>2.0.0.0-SNAPSHOT</version>
</parent>

<name>Ambari Infra Manager Integration Tests</name>
<url>http://maven.apache.org</url>
<modelVersion>4.0.0</modelVersion>

<artifactId>ambari-infra-manager-it</artifactId>

<properties>
<jbehave.version>4.0.5</jbehave.version>
<failsafe-plugin.version>2.20</failsafe-plugin.version>
<docker.host>localhost</docker.host>
<stories.location>NONE</stories.location>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.5</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.20</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.20</version>
</dependency>

<dependency>
<groupId>org.jbehave</groupId>
<artifactId>jbehave-core</artifactId>
<version>${jbehave.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<version>3.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<testOutputDirectory>target/classes</testOutputDirectory>
<testResources>
<testResource>
<directory>src/test/java/</directory>
<includes>
<include>**/*.story</include>
</includes>
</testResource>
<testResource>
<directory>src/test/resources</directory>
</testResource>
</testResources>
</build>

<profiles>
<profile>
<id>it</id>
<activation>
<property>
<name>it</name>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>${failsafe-plugin.version}</version>
<executions>
<execution>
<id>run-integration-tests</id>
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
</goals>
<configuration>
<includes>
<include>**/*Stories.java</include>
</includes>
<systemPropertyVariables>
<log4j.configuration>file:${project.build.testOutputDirectory}/log4j.properties</log4j.configuration>
<docker.host>${docker.host}</docker.host>
<backend.stories.location>${stories.location}</backend.stories.location>
</systemPropertyVariables>
</configuration>
</execution>
<execution>
<id>verify-integration-tests</id>
<phase>verify</phase>
<goals>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>

</project>
@@ -0,0 +1,93 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.ambari.infra;

import org.apache.commons.io.IOUtils;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;

import static org.apache.commons.lang.StringUtils.isBlank;

// TODO: use swagger
public class InfraClient implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(InfraClient.class);

private final CloseableHttpClient httpClient;
private final URI baseUrl;

public InfraClient(String baseUrl) {
try {
this.baseUrl = new URI(baseUrl);
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
httpClient = HttpClientBuilder.create().setRetryHandler(new DefaultHttpRequestRetryHandler(0, false)).build();
}

@Override
public void close() throws Exception {
httpClient.close();
}

// TODO: return job data
public void getJobs() {
execute(new HttpGet(baseUrl));
}

private String execute(HttpRequestBase post) {
try (CloseableHttpResponse response = httpClient.execute(post)) {
String responseBodyText = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset());
LOG.info("Response code {} body {} ", response.getStatusLine().getStatusCode(), responseBodyText);
return responseBodyText;
} catch (ClientProtocolException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}

// TODO: return job data
public void startJob(String jobName, String parameters) {
URIBuilder uriBuilder = new URIBuilder(baseUrl);
uriBuilder.setScheme("http");
uriBuilder.setPath(uriBuilder.getPath() + "/" + jobName);
if (!isBlank(parameters))
uriBuilder.addParameter("params", parameters);
try {
execute(new HttpPost(uriBuilder.build()));
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
}
@@ -0,0 +1,108 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.ambari.infra;

import com.google.common.collect.Lists;
import org.apache.ambari.infra.steps.ExportJobsSteps;
import org.apache.commons.lang.StringUtils;
import org.jbehave.core.configuration.Configuration;
import org.jbehave.core.configuration.MostUsefulConfiguration;
import org.jbehave.core.io.LoadFromClasspath;
import org.jbehave.core.io.LoadFromRelativeFile;
import org.jbehave.core.io.StoryFinder;
import org.jbehave.core.io.StoryLoader;
import org.jbehave.core.junit.JUnitStories;
import org.jbehave.core.reporters.Format;
import org.jbehave.core.reporters.StoryReporterBuilder;
import org.jbehave.core.steps.InjectableStepsFactory;
import org.jbehave.core.steps.InstanceStepsFactory;
import org.jbehave.core.steps.ParameterConverters;

import java.io.File;
import java.net.URL;
import java.util.List;

import static java.util.Collections.singletonList;
import static org.jbehave.core.io.CodeLocations.codeLocationFromClass;

public class InfraManagerStories extends JUnitStories {
private static final String BACKEND_STORIES_LOCATION_PROPERTY = "backend.stories.location";
private static final String STORY_SUFFIX = ".story";

@Override
public Configuration configuration() {
return new MostUsefulConfiguration()
.useStoryLoader(getStoryLoader(BACKEND_STORIES_LOCATION_PROPERTY, this.getClass()))
.useParameterConverters(new ParameterConverters().addConverters(new OffsetDateTimeConverter()))
.useStoryReporterBuilder(
new StoryReporterBuilder().withFailureTrace(true).withDefaultFormats().withFormats(Format.CONSOLE, Format.TXT));
}

private static StoryLoader getStoryLoader(String property, Class clazz) {
boolean useExternalStoryLocation = useExternalStoryLocation(property);
if (useExternalStoryLocation) {
try {
return new LoadFromRelativeFile(new URL("file://" + System.getProperty(property)));
} catch (Exception e) {
throw new RuntimeException("Cannot load story files from url: file://" + System.getProperty(property));
}
} else {
return new LoadFromClasspath(clazz);
}
}

@Override
public InjectableStepsFactory stepsFactory() {
return new InstanceStepsFactory(configuration(), new ExportJobsSteps());
}

@Override
protected List<String> storyPaths() {
return findStories(BACKEND_STORIES_LOCATION_PROPERTY, STORY_SUFFIX, this.getClass());
}

private static List<String> findStories(String property, String suffix, Class clazz) {
if (useExternalStoryLocation(property)) {
return findStoriesInFolder(System.getProperty(property), suffix);
} else {
return new StoryFinder()
.findPaths(codeLocationFromClass(clazz).getFile(), singletonList(String.format("**/*%s", suffix)), null);
}
}

private static List<String> findStoriesInFolder(String folderAbsolutePath, String suffix) {
List<String> results = Lists.newArrayList();
File folder = new File(folderAbsolutePath);
File[] listOfFiles = folder.listFiles();
if (listOfFiles != null) {
for (File file : listOfFiles) {
if (file.getName().endsWith(suffix)) {
results.add(file.getName());
}
}
}
return results;
}

private static boolean useExternalStoryLocation(String property) {
String storyLocationProp = System.getProperty(property);
return StringUtils.isNotEmpty(storyLocationProp) && !"NONE".equals(storyLocationProp);
}

}
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.ambari.infra;

import org.jbehave.core.steps.ParameterConverters;

import java.lang.reflect.Type;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;

public class OffsetDateTimeConverter implements ParameterConverters.ParameterConverter {
private static final DateTimeFormatter SOLR_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX");

@Override
public boolean accept(Type type) {
return type instanceof Class<?> && OffsetDateTime.class.isAssignableFrom((Class<?>) type);
}

@Override
public Object convertValue(String value, Type type) {
return OffsetDateTime.parse(value, SOLR_DATETIME_FORMATTER);
}
}

0 comments on commit 79f7f13

Please sign in to comment.