Skip to content
Permalink
Browse files
AMBARI-22899. Upgrade Infra Solr version to 7.2.1 (#309)
* AMBARI-22899. Upgrade Infra Solr version to 7.2.1

* AMBARI-22899. Set missing Lucene version for service logs.

* AMBARI-22899. Updated Solr tarball location.

* AMBARI-22899. Delete migrate dir on start of package.
  • Loading branch information
oleewere committed Feb 10, 2018
1 parent bab13ad commit 88ce5472c15f01c07d520b7b2e24a802aca9e796
Show file tree
Hide file tree
Showing 17 changed files with 667 additions and 707 deletions.
@@ -30,7 +30,7 @@

<properties>
<mapping.base.path>/usr/lib</mapping.base.path>
<!-- original apache repo location: http://archive.apache.org/dist/lucene/solr/${solr.version}/solr-${solr.version}.tgz -->
<!-- original url: <solr.tar>http://archive.apache.org/dist/lucene/solr/${solr.version}/solr-${solr.version}.tgz</solr.tar> -->
<solr.tar>http://public-repo-1.hortonworks.com/ARTIFACTS/dist/lucene/solr/${solr.version}/solr-${solr.version}.tgz</solr.tar>
<solr.mapping.path>${mapping.base.path}/ambari-infra-solr</solr.mapping.path>
<solr.package.name>ambari-infra-solr</solr.package.name>

Large diffs are not rendered by default.

@@ -27,12 +27,21 @@

<target name="package">
<delete dir="target/package"/>
<delete dir="target/migrate"/>
<copy todir="target/package/libs" includeEmptyDirs="no">
<fileset dir="target/libs"/>
</copy>
<copy todir="target/package/libs" includeEmptyDirs="no">
<fileset file="target/*.jar"/>
</copy>
<mkdir dir="target/migrate"/>
<get src="${lucene6-core.url}" dest="target/migrate/${lucene6-core-jar.name}" usetimestamp="true"/>
<get src="${lucene6-backward-codecs.url}" dest="target/migrate/${lucene6-backward-codecs-jar.name}" usetimestamp="true"/>
<copy todir="target/package/migrate" includeEmptyDirs="no">
<fileset file="target/migrate/*.jar"/>
<fileset file="target/package/libs/lucene-core-${solr.version}.jar"/>
<fileset file="target/package/libs/lucene-backward-codecs-${solr.version}.jar"/>
</copy>
<copy todir="target/package" includeEmptyDirs="no">
<fileset file="src/main/resources/solrCloudCli.sh"/>
<fileset file="src/main/resources/solrIndexHelper.sh"/>
@@ -29,6 +29,14 @@

<artifactId>ambari-infra-solr-client</artifactId>

<properties>
<lucene6.version>6.6.0</lucene6.version>
<lucene6-core-jar.name>lucene-core-${lucene6.version}.jar</lucene6-core-jar.name>
<lucene6-core.url>http://central.maven.org/maven2/org/apache/lucene/lucene-core/${lucene6.version}/${lucene6-core-jar.name}</lucene6-core.url>
<lucene6-backward-codecs-jar.name>lucene-backward-codecs-${lucene6.version}.jar</lucene6-backward-codecs-jar.name>
<lucene6-backward-codecs.url>http://central.maven.org/maven2/org/apache/lucene/lucene-backward-codecs/${lucene6.version}/${lucene6-backward-codecs-jar.name}</lucene6-backward-codecs.url>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.solr</groupId>
@@ -151,9 +151,9 @@ public static void main(String[] args) {
.argName("my_new_shard")
.build();

final Option disableShardingOption = Option.builder("ns")
.longOpt("no-sharding")
.desc("Sharding not used when creating collection")
final Option implicitRoutingOption = Option.builder("ir")
.longOpt("implicit-routing")
.desc("Use implicit routing when creating a collection")
.build();

final Option zkConnectStringOption = Option.builder("z")
@@ -350,7 +350,7 @@ public static void main(String[] args) {
options.addOption(routerNameOption);
options.addOption(routerFieldOption);
options.addOption(shardNameOption);
options.addOption(disableShardingOption);
options.addOption(implicitRoutingOption);
options.addOption(createCollectionOption);
options.addOption(downloadConfigOption);
options.addOption(uploadConfigurationOption);
@@ -445,7 +445,7 @@ public static void main(String[] args) {
String routerName = cli.hasOption("rn") ? cli.getOptionValue("rn") : null;
String routerField = cli.hasOption("rf") ? cli.getOptionValue("rf") : null;
String shardName = cli.hasOption("sn") ? cli.getOptionValue("sn") : null;
boolean isSplitting = !cli.hasOption("ns");
boolean implicitRouting = cli.hasOption("ir");
String jaasFile = cli.hasOption("jf") ? cli.getOptionValue("jf") : null;
String keyStoreLocation = cli.hasOption("ksl") ? cli.getOptionValue("ksl") : null;
String keyStorePassword = cli.hasOption("ksp") ? cli.getOptionValue("ksp") : null;
@@ -472,7 +472,7 @@ public static void main(String[] args) {
.withRouterName(routerName)
.withRouterField(routerField)
.withJaasFile(jaasFile) // call before creating SolrClient
.withSplitting(isSplitting)
.isImplicitRouting(implicitRouting)
.withSolrZkClient(ZK_CLIENT_TIMEOUT, ZK_CLIENT_CONNECT_TIMEOUT)
.withKeyStoreLocation(keyStoreLocation)
.withKeyStorePassword(keyStorePassword)
@@ -64,7 +64,7 @@ public class AmbariSolrCloudClient {
private final int maxShardsPerNode;
private final String routerName;
private final String routerField;
private final boolean splitting;
private final boolean implicitRouting;
private final String jaasFile;
private final String znode;
private final String saslUsers;
@@ -88,7 +88,7 @@ public AmbariSolrCloudClient(AmbariSolrCloudClientBuilder builder) {
this.maxShardsPerNode = builder.maxShardsPerNode;
this.routerName = builder.routerName;
this.routerField = builder.routerField;
this.splitting = builder.splitting;
this.implicitRouting = builder.implicitRouting;
this.znode = builder.znode;
this.saslUsers = builder.saslUsers;
this.propName = builder.propName;
@@ -114,7 +114,7 @@ public String createCollection() throws Exception {
LOG.info("Collection '{}' creation request sent.", collection);
} else {
LOG.info("Collection '{}' already exits.", getCollection());
if (this.isSplitting()) {
if (this.isImplicitRouting()) {
createShard(null);
}
}
@@ -317,8 +317,8 @@ public String getRouterField() {
return routerField;
}

public boolean isSplitting() {
return splitting;
public boolean isImplicitRouting() {
return implicitRouting;
}

public String getJaasFile() {
@@ -20,8 +20,7 @@
package org.apache.ambari.infra.solr;

import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
import org.apache.solr.common.cloud.SolrZkClient;

public class AmbariSolrCloudClientBuilder {
@@ -31,7 +30,9 @@ public class AmbariSolrCloudClientBuilder {
private static final String TRUSTSTORE_LOCATION_ARG = "javax.net.ssl.trustStore";
private static final String TRUSTSTORE_PASSWORD_ARG = "javax.net.ssl.trustStorePassword";
private static final String TRUSTSTORE_TYPE_ARG = "javax.net.ssl.trustStoreType";

private static final String JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config";
private static final String SOLR_HTTPCLIENT_BUILDER_FACTORY = "solr.httpclient.builder.factory";

String zkConnectString;
String collection;
String configSet;
@@ -45,7 +46,7 @@ public class AmbariSolrCloudClientBuilder {
String routerField = "_router_field_";
CloudSolrClient solrCloudClient;
SolrZkClient solrZkClient;
boolean splitting;
boolean implicitRouting;
String jaasFile;
String znode;
String saslUsers;
@@ -113,8 +114,8 @@ public AmbariSolrCloudClientBuilder withRouterField(String routerField) {
return this;
}

public AmbariSolrCloudClientBuilder withSplitting(boolean splitting) {
this.splitting = splitting;
public AmbariSolrCloudClientBuilder isImplicitRouting(boolean implicitRouting) {
this.implicitRouting = implicitRouting;
return this;
}

@@ -208,8 +209,8 @@ public AmbariSolrCloudClientBuilder withSecure(boolean isSecure) {

private void setupSecurity(String jaasFile) {
if (jaasFile != null) {
System.setProperty("java.security.auth.login.config", jaasFile);
HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer());
System.setProperty(JAVA_SECURITY_AUTH_LOGIN_CONFIG, jaasFile);
System.setProperty(SOLR_HTTPCLIENT_BUILDER_FACTORY, Krb5HttpClientBuilder.class.getCanonicalName());
}
}
}
@@ -38,12 +38,10 @@ public String handleResponse(CollectionAdminResponse response, AmbariSolrCloudCl
public CollectionAdminRequest.Create createRequest(AmbariSolrCloudClient client) {
CollectionAdminRequest.Create request =
CollectionAdminRequest.createCollection(client.getCollection(), client.getConfigSet(), client.getShards(), client.getReplication());
request.setMaxShardsPerNode(client.getMaxShardsPerNode());;
if (client.getRouterField() != null && client.getRouterName()!= null) {
request.setMaxShardsPerNode(client.getMaxShardsPerNode());
if (client.isImplicitRouting()) {
request.setRouterName(client.getRouterName());
request.setRouterField(client.getRouterField());
}
if (client.isSplitting()) {
request.setShards(ShardUtils.generateShardListStr(client.getMaxShardsPerNode()));
}
return request;
@@ -41,31 +41,39 @@ function print_help() {
-d, --index-data-dir <DIRECTORY> Location of the solr cores (e.g.: /opt/ambari_infra_solr/data)
-c, --core-filter <FILTER1,FILTER2> Comma separated name filters of core directoies (default: hadoop_logs,audit_logs,history)
-f, --force Force to start index upgrade, even is the version is at least 6.
-v, --version Lucene version to upgrade (default: 6.6.0, available: 6.6.0, 7.2.1)
run-check-index-tool arguments:
-v, --version Lucene version to upgrade (default: 6.6.0, available: 6.6.0, 7.2.1)
run-upgrade-index-tool arguments:
-v, --version Lucene version to upgrade (default: 6.6.0, available: 6.6.0, 7.2.1)
EOF
}

function upgrade_core() {
local INDEX_DIR=${1:?"usage: <index_base_dir> e.g.: /opt/ambari_infra_solr/data"}
local FORCE_UPDATE=${2:?"usage <force_update_flag> e.g.: true"}
local SOLR_CORE_FILTERS=${3:?"usage: <comma separated core filters> e.g.: hadoop_logs,audit_logs,history"}

local LUCENE_VERSION=${4:?"usage <lucene_index_version> e.g.: 7.2.1"}
SOLR_CORE_FILTER_ARR=$(echo $SOLR_CORE_FILTERS | sed "s/,/ /g")

local version_prefix="$(echo $LUCENE_VERSION | head -c 1)"

for coll in $SOLR_CORE_FILTER_ARR; do
if [[ "$1" == *"$coll"* ]]; then
echo "Core '$1' dir name contains $coll (core filter)'";
version=$(PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/libs/lucene-core-6.6.0.jar:$DIR/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.CheckIndex -fast $1|grep " version="|sed -e 's/.*=//g'|head -1)
version=$(PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/migrate/lucene-core-$LUCENE_VERSION.jar:$DIR/migrate/lucene-backward-codecs-$LUCENE_VERSION.jar" org.apache.lucene.index.CheckIndex -fast $1|grep " version="|sed -e 's/.*=//g'|head -1)
if [ -z $version ] ; then
echo "Core '$1' - Empty index?"
return
fi
majorVersion=$(echo $version|cut -c 1)
if [ $majorVersion -ge 6 ] && [ $FORCE_UPDATE == "false" ] ; then
if [ $majorVersion -ge $version_prefix ] && [ $FORCE_UPDATE == "false" ] ; then
echo "Core '$1' - Already on version $version, not upgrading. Use -f or --force option to run upgrade anyway."
else
echo "Core '$1' - Index version is $version, upgrading ..."
PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/libs/lucene-core-6.6.0.jar:$DIR/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.IndexUpgrader -delete-prior-commits $1
PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/migrate/lucene-core-$LUCENE_VERSION.jar:$DIR/migrate/lucene-backward-codecs-$LUCENE_VERSION.jar" org.apache.lucene.index.IndexUpgrader -delete-prior-commits $1
echo "Upgrading core '$1' has finished"
fi
fi
@@ -89,6 +97,10 @@ function upgrade_index() {
local INDEX_DIR="$2"
shift 2
;;
-v|--version)
local LUCENE_VERSION="$2"
shift 2
;;
*)
echo "Unknown option: $1"
exit 1
@@ -104,6 +116,10 @@ function upgrade_index() {
SOLR_CORE_FILTERS="hadoop_logs,audit_logs,history"
fi

if [[ -z "$LUCENE_VERSION" ]] ; then
LUCENE_VERSION="6.6.0"
fi

if [[ -z "$FORCE_UPDATE" ]] ; then
FORCE_UPDATE="false"
else
@@ -120,7 +136,7 @@ function upgrade_index() {
abspath=$(cd "$(dirname "$c")"; pwd)/$(basename "$c")
find $c/data -maxdepth 1 -type d -name 'index*' | while read indexDir; do
echo "Checking core $name - $abspath"
upgrade_core "$indexDir" "$FORCE_UPDATE" "$SOLR_CORE_FILTERS"
upgrade_core "$indexDir" "$FORCE_UPDATE" "$SOLR_CORE_FILTERS" "$LUCENE_VERSION"
done
else
echo "No index folder found for $name"
@@ -132,11 +148,41 @@ function upgrade_index() {

function upgrade_index_tool() {
# see: https://cwiki.apache.org/confluence/display/solr/IndexUpgrader+Tool
PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/libs/lucene-core-6.6.0.jar:$DIR/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.IndexUpgrader ${@}
luc_version = "6.6.0"
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-v|--version)
luc_version="$2"
shift 2
;;
*)
echo "Unknown option: $1"
exit 1
;;
esac
done
PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/migrate/lucene-core-$luc_version.jar:$DIR/migrate/lucene-backward-codecs-$luc_version.jar" org.apache.lucene.index.IndexUpgrader ${@}
}

function check_index_tool() {
PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/libs/lucene-core-6.6.0.jar:$DIR/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.CheckIndex ${@}
luc_version = "6.6.0"
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-v|--version)
luc_version="$2"
shift 2
;;
*)
echo "Unknown option: $1"
exit 1
;;
esac
done
PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR/migrate/lucene-core-$luc_version.jar:$DIR/migrate/lucene-backward-codecs-$luc_version.jar" org.apache.lucene.index.CheckIndex ${@}
}

function main() {

0 comments on commit 88ce547

Please sign in to comment.