Skip to content
Permalink
Browse files
Added docker maven plugin, changed defaults to make it build and run …
…easier
  • Loading branch information
agozhiy committed Feb 11, 2022
1 parent 9fffe6a commit b4e0895bbd9c8204d3c1e19bf2531f0b4d490346
Showing 4 changed files with 97 additions and 14 deletions.
@@ -5,6 +5,7 @@ source conf/drillTestConfig.properties
cd ${bin}/../

default_debug_string="-Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,address=50000,server=y,suspend=y"
JDBC_DRIVER_CP="${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar"

ARGS=()
while [[ $# > 0 ]]
@@ -20,6 +21,11 @@ do
DRILL_TEST_FRAMEWORK_JAVA_OPTS="$DRILL_TEST_FRAMEWORK_JAVA_OPTS $default_debug_string"
shift
;;
--jdbcCP)
shift
JDBC_DRIVER_CP=$1
shift
;;
*)
ARGS+=("$1")
shift
@@ -31,10 +37,12 @@ done
# and set ${JDBC_DRIVER_CP} inside conf/drillTestConfig.properties
# java -cp conf/:${JDBC_DRIVER_CP}:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar:${HADOOP_INSTALL_LOC}/lib/* org.apache.drill.test.framework.TestDriver $*

/sbin/ifconfig

# use the following line when testing apache drill JDBC driver
if [[ $HADOOP_VERSION == *"mapr"* ]]
then
java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar -Dfs.mapr.bailout.on.library.mismatch=false -Djava.io.tmpdir=/tmp/drill/tests -Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -Dzookeeper.sasl.client=false org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${JDBC_DRIVER_CP}:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar -Dfs.mapr.bailout.on.library.mismatch=false -Djava.io.tmpdir=/tmp/drill/tests -Djava.security.auth.login.config=/opt/mapr/conf/mapr.login.conf -Dzookeeper.sasl.client=false org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
else
java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${DRILL_VERSION}.jar:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
java $DRILL_TEST_FRAMEWORK_JAVA_OPTS -Xss40m -cp .:conf/:${JDBC_DRIVER_CP}:framework/target/framework-1.0.0-SNAPSHOT-shaded.jar org.apache.drill.test.framework.TestDriver "${ARGS[@]}"
fi
@@ -9,10 +9,23 @@
<testng.version>6.4</testng.version>
<log4j.version>1.2.17</log4j.version>
<jackson.version>2.10.0</jackson.version>
<drill.version>${env.DRILL_VERSION}</drill.version>
<hadoop.version>${env.HADOOP_VERSION}</hadoop.version>
<runtimeDepenencies>${env.DRILL_HOME}/conf</runtimeDepenencies>
<drill.version>1.20.0-SNAPSHOT</drill.version>
<hadoop.version>3.2.2</hadoop.version>
<drill.test.framework.root>${project.basedir}/..</drill.test.framework.root>
<drill.home>${project.basedir}/../../drill</drill.home>
<drill.distribution>${drill.home}/distribution/target/apache-drill-${drill.version}/apache-drill-${drill.version}/</drill.distribution>
<drill.jdbc.driver.path>jars/jdbc-driver/drill-jdbc-all-${drill.version}.jar</drill.jdbc.driver.path>
<drill.java.exec.path>jars/drill-java-exec-${drill.version}.jar</drill.java.exec.path>
<m2.home>${env.HOME}/.m2</m2.home>
<runtimeDepenencies>${drill.distribution}/conf</runtimeDepenencies>
<maven.surefire.plugin.version>2.19.1</maven.surefire.plugin.version>
<!-- Docker Plugin Properties -->
<maven.docker.plugin.version>0.38.1</maven.docker.plugin.version>
<docker.base.image>maven:3.8.4-jdk-11</docker.base.image>
<docker.automation.root>/root/testAutomation</docker.automation.root>
<docker.drill.test.framework.home>${docker.automation.root}/drill-test-framework</docker.drill.test.framework.home>
<docker.drill.home>${docker.automation.root}/apache-drill-${drill.version}</docker.drill.home>
<test.args>-s Functional -g empty</test.args>
</properties>
<dependencies>
<dependency>
@@ -92,7 +105,14 @@
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-jdbc-all</artifactId>
<version>${drill.version}</version>
<systemPath>${env.DRILL_HOME}/jars/jdbc-driver/drill-jdbc-all-${drill.version}.jar</systemPath>
<systemPath>${drill.distribution}/${drill.jdbc.driver.path}</systemPath>
<scope>system</scope>
</dependency>
<dependency>
<groupId>org.apache.drill.exec</groupId>
<artifactId>drill-java-exec</artifactId>
<version>${drill.version}</version>
<systemPath>${drill.distribution}/${drill.java.exec.path}</systemPath>
<scope>system</scope>
</dependency>
<dependency>
@@ -141,7 +161,7 @@
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<releases>
<enabled>true</enabled>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
@@ -150,6 +170,61 @@
</repositories>
<build>
<plugins>
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
<version>${maven.docker.plugin.version}</version>
<configuration>
<showLogs>true</showLogs>
<verbose>true</verbose>
<images>
<image>
<name>drill-test-framework</name>
<build>
<nocache>false</nocache>
<from>${docker.base.image}</from>
</build>
<run>
<containerNamePattern>%n_%t_%i</containerNamePattern>
<wait>
<exit>0</exit>
<time>1210000000</time>
</wait>
<volumes>
<bind>
<volume>${drill.test.framework.root}:${docker.drill.test.framework.home}</volume>
<volume>${drill.distribution}:${docker.drill.home}</volume>
<volume>${m2.home}:/root/.m2</volume>
</bind>
</volumes>
<workingDir>${docker.drill.test.framework.home}</workingDir>
<network>
<mode>host</mode>
</network>
<cmd>
<shell>/bin/bash bin/run_tests --jdbcCP ${docker.drill.home}/${drill.jdbc.driver.path} ${test.args}</shell>
</cmd>
</run>
</image>
</images>
</configuration>
<!--<executions>
<execution>
<phase>install</phase>
<goals>
<goal>build</goal>
</goals>
</execution>
<execution>
<id>start</id>
<phase>test</phase>
<goals>
<goal>run</goal>
<goal>stop</goal>
</goals>
</execution>
</executions>-->
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
@@ -3,7 +3,7 @@
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import oadd.org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.server.rest.profile.CoreOperatorType;

import java.util.List;
import java.util.stream.Collectors;
@@ -208,13 +208,13 @@ public String toString() {
* @param operator
* @return
*/
public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType operator) {
public long getOptimalMemoryPerOperator(final CoreOperatorType operator) {
return this.fragmentProfiles
.stream()
.flatMap(f -> f.minorFragmentProfiles
.stream()
.flatMap(m -> m.operatorProfiles.stream())
).filter(o -> o.operatorId == operator.getNumber())
).filter(o -> o.operatorId == operator.getId())
.mapToLong(o -> o.optimalMemAllocation)
.sum();
}
@@ -223,14 +223,14 @@ public long getOptimalMemoryPerOperator(final UserBitShared.CoreOperatorType ope
* Get different operators in the profile.
* @return a list of operators in the query profile.
*/
public List<UserBitShared.CoreOperatorType> getOperatorsFromProfile() {
public List<CoreOperatorType> getOperatorsFromProfile() {
return this.fragmentProfiles
.stream().flatMap(f -> f.minorFragmentProfiles
.stream()
.flatMap(m -> m.operatorProfiles.stream())
).mapToInt(o -> o.operatorId)
.distinct()
.mapToObj(UserBitShared.CoreOperatorType::forNumber)
.mapToObj(CoreOperatorType::valueOf)
.collect(Collectors.toList());
}

@@ -2,7 +2,7 @@

import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import oadd.org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.server.rest.profile.CoreOperatorType;
import org.apache.drill.test.framework.common.DrillJavaTestBase;
import org.apache.log4j.Logger;
import org.testng.Assert;
@@ -182,7 +182,7 @@ public void testTotalMemoryForQueryProfile() {
LOG.info("Memory estimated by RM planner: " + rmMemEstimate);
Assert.assertTrue(rmMemEstimate > 0,
"RM estimated memory should be greater than 0");
List<UserBitShared.CoreOperatorType> operators = profile.getOperatorsFromProfile();
List<CoreOperatorType> operators = profile.getOperatorsFromProfile();
Assert.assertTrue(operators.size() > 0,
"Number of operators in the profile should be greater than 0");
operators.forEach(LOG::info);

0 comments on commit b4e0895

Please sign in to comment.