diff --git a/solr/core/src/java/org/apache/solr/util/ExportTool.java b/solr/core/src/java/org/apache/solr/util/ExportTool.java index 1f611a8e03ef..b8bfbf13dc6d 100644 --- a/solr/core/src/java/org/apache/solr/util/ExportTool.java +++ b/solr/core/src/java/org/apache/solr/util/ExportTool.java @@ -80,10 +80,11 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.common.util.StrUtils; +import org.apache.solr.util.cli.ToolBase; import org.noggit.CharArr; import org.noggit.JSONWriter; -public class ExportTool extends SolrCLI.ToolBase { +public class ExportTool extends ToolBase { @Override public String getName() { return "export"; @@ -176,7 +177,7 @@ public void streamDocListInfo(long numFound, long start, Float maxScore) {} static Set formats = Set.of(JAVABIN, "jsonl"); @Override - protected void runImpl(CommandLine cli) throws Exception { + public void runImpl(CommandLine cli) throws Exception { String url = cli.getOptionValue("url"); Info info = new MultiThreadedRunner(url); info.query = cli.getOptionValue("query", "*:*"); diff --git a/solr/core/src/java/org/apache/solr/util/PackageTool.java b/solr/core/src/java/org/apache/solr/util/PackageTool.java index 9605af748d38..5e388fa875f6 100644 --- a/solr/core/src/java/org/apache/solr/util/PackageTool.java +++ b/solr/core/src/java/org/apache/solr/util/PackageTool.java @@ -29,7 +29,6 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.apache.lucene.util.SuppressForbidden; -import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; @@ -40,11 +39,12 @@ import org.apache.solr.packagemanager.SolrPackage; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; import org.apache.solr.packagemanager.SolrPackageInstance; -import org.apache.solr.util.SolrCLI.StatusTool; +import org.apache.solr.util.cli.StatusTool; +import org.apache.solr.util.cli.ToolBase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PackageTool extends SolrCLI.ToolBase { +public class PackageTool extends ToolBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -71,7 +71,7 @@ public String getName() { "We really need to print the stacktrace here, otherwise " + "there shall be little else information to debug problems. Other SolrCLI tools " + "don't print stack traces, hence special treatment is needed here.") - protected void runImpl(CommandLine cli) throws Exception { + public void runImpl(CommandLine cli) throws Exception { try { solrUrl = cli.getOptionValues("solrUrl")[cli.getOptionValues("solrUrl").length - 1]; solrBaseUrl = solrUrl.replaceAll("\\/solr$", ""); // strip out ending "/solr" @@ -364,8 +364,7 @@ private String getZkHost(CommandLine cli) throws Exception { if (zkHost != null) return zkHost; String systemInfoUrl = solrUrl + "/admin/info/system"; - CloseableHttpClient httpClient = SolrCLI.getHttpClient(); - try { + try (CloseableHttpClient httpClient = SolrCLI.getHttpClient()) { // hit Solr to get system info Map systemInfo = SolrCLI.getJson(httpClient, systemInfoUrl, 2, true); @@ -381,8 +380,6 @@ private String getZkHost(CommandLine cli) throws Exception { } zkHost = zookeeper; } - } finally { - HttpClientUtil.close(httpClient); } return zkHost; diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java index 48b1eb409c8f..bdcaf583fade 100755 --- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java @@ -18,63 +18,36 @@ import static org.apache.solr.common.SolrException.ErrorCode.FORBIDDEN; import static org.apache.solr.common.SolrException.ErrorCode.UNAUTHORIZED; -import static org.apache.solr.common.params.CommonParams.DISTRIB; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.util.Utils.fromJSONString; import com.google.common.annotations.VisibleForTesting; -import java.io.Console; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; import java.lang.invoke.MethodHandles; import java.net.ConnectException; -import java.net.Socket; import java.net.SocketException; import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileOwnerAttributeView; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; -import java.util.Collection; import java.util.Collections; import java.util.Enumeration; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; -import java.util.Scanner; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import javax.net.ssl.SSLPeerUnverifiedException; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.commons.exec.DefaultExecuteResultHandler; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.ExecuteException; -import org.apache.commons.exec.Executor; -import org.apache.commons.exec.OS; -import org.apache.commons.exec.environment.EnvironmentUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.file.PathUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.SystemUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.NoHttpResponseException; @@ -90,34 +63,37 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCoreNodeProps; -import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.core.ConfigSetService; -import org.apache.solr.security.Sha256AuthenticationProvider; +import org.apache.solr.util.cli.ApiTool; +import org.apache.solr.util.cli.AssertTool; +import org.apache.solr.util.cli.AuthTool; +import org.apache.solr.util.cli.ConfigSetDownloadTool; +import org.apache.solr.util.cli.ConfigSetUploadTool; +import org.apache.solr.util.cli.ConfigTool; +import org.apache.solr.util.cli.CreateCollectionTool; +import org.apache.solr.util.cli.CreateCoreTool; +import org.apache.solr.util.cli.CreateTool; +import org.apache.solr.util.cli.DeleteTool; +import org.apache.solr.util.cli.HealthcheckTool; +import org.apache.solr.util.cli.RunExampleTool; +import org.apache.solr.util.cli.StatusTool; +import org.apache.solr.util.cli.Tool; +import org.apache.solr.util.cli.ZkCpTool; +import org.apache.solr.util.cli.ZkLsTool; +import org.apache.solr.util.cli.ZkMkrootTool; +import org.apache.solr.util.cli.ZkMvTool; +import org.apache.solr.util.cli.ZkRmTool; import org.apache.solr.util.configuration.SSLConfigurationsFactory; import org.noggit.CharArr; import org.noggit.JSONParser; @@ -130,107 +106,17 @@ public class SolrCLI implements CLIO { private static final long MAX_WAIT_FOR_CORE_LOAD_NANOS = TimeUnit.NANOSECONDS.convert(1, TimeUnit.MINUTES); - /** Defines the interface to a Solr tool that can be run from this command-line app. */ - public interface Tool { - String getName(); - - Option[] getOptions(); - - int runTool(CommandLine cli) throws Exception; - } - - public abstract static class ToolBase implements Tool { - protected PrintStream stdout; - protected boolean verbose = false; - - protected ToolBase() { - this(CLIO.getOutStream()); - } - - protected ToolBase(PrintStream stdout) { - this.stdout = stdout; - } - - protected void echoIfVerbose(final String msg, CommandLine cli) { - if (cli.hasOption(OPTION_VERBOSE.getOpt())) { - echo(msg); - } - } - - protected void echo(final String msg) { - stdout.println(msg); - } - - @Override - public int runTool(CommandLine cli) throws Exception { - verbose = cli.hasOption(OPTION_VERBOSE.getOpt()); - - int toolExitStatus = 0; - try { - runImpl(cli); - } catch (Exception exc) { - // since this is a CLI, spare the user the stacktrace - String excMsg = exc.getMessage(); - if (excMsg != null) { - CLIO.err("\nERROR: " + excMsg + "\n"); - if (verbose) { - exc.printStackTrace(CLIO.getErrStream()); - } - toolExitStatus = 1; - } else { - throw exc; - } - } - return toolExitStatus; - } - - protected abstract void runImpl(CommandLine cli) throws Exception; - } - /** - * Helps build SolrCloud aware tools by initializing a CloudSolrClient instance before running the - * tool. - */ - public abstract static class SolrCloudTool extends ToolBase { - - protected SolrCloudTool(PrintStream stdout) { - super(stdout); - } - - @Override - public Option[] getOptions() { - return cloudOptions; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String zkHost = cli.getOptionValue(OPTION_ZKHOST.getOpt(), ZK_HOST); - - log.debug("Connecting to Solr cluster: {}", zkHost); - try (var cloudSolrClient = - new CloudLegacySolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) - .build()) { - - cloudSolrClient.connect(); - runCloudTool(cloudSolrClient, cli); - } - } - - /** Runs a SolrCloud tool with CloudSolrClient initialized */ - protected abstract void runCloudTool(CloudLegacySolrClient cloudSolrClient, CommandLine cli) - throws Exception; - } - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String DEFAULT_SOLR_URL = "http://localhost:8983/solr"; - public static final String ZK_HOST = "localhost:9983"; + public static final String DEFAULT_ZK_HOST = "localhost:9983"; + public static final String DEFAULT_CONFIG_SET = "_default"; public static final Option OPTION_ZKHOST = Option.builder("z") .argName("HOST") .hasArg() .required(false) - .desc("Address of the ZooKeeper ensemble; defaults to: " + ZK_HOST) + .desc("Address of the ZooKeeper ensemble; defaults to: " + DEFAULT_ZK_HOST) .longOpt("zkHost") .build(); public static final Option OPTION_SOLRURL = @@ -254,7 +140,7 @@ protected abstract void runCloudTool(CloudLegacySolrClient cloudSolrClient, Comm // .type(Boolean.class) .build(); - public static final Option[] cloudOptions = + public static final Option[] CLOUD_OPTIONS = new Option[] { OPTION_ZKHOST, Option.builder("c") @@ -267,10 +153,57 @@ protected abstract void runCloudTool(CloudLegacySolrClient cloudSolrClient, Comm OPTION_VERBOSE }; - private static void exit(int exitStatus) { + public static final Option[] CREATE_COLLECTION_OPTIONS = + new Option[] { + OPTION_ZKHOST, + OPTION_SOLRURL, + Option.builder(NAME) + .argName("NAME") + .hasArg() + .required(true) + .desc("Name of collection to create.") + .build(), + Option.builder("shards") + .argName("#") + .hasArg() + .required(false) + .desc("Number of shards; default is 1.") + .build(), + Option.builder("replicationFactor") + .argName("#") + .hasArg() + .required(false) + .desc( + "Number of copies of each document across the collection (replicas per shard); default is 1.") + .build(), + Option.builder("confdir") + .argName("NAME") + .hasArg() + .required(false) + .desc( + "Configuration directory to copy when creating the new collection; default is " + + DEFAULT_CONFIG_SET + + '.') + .build(), + Option.builder("confname") + .argName("NAME") + .hasArg() + .required(false) + .desc("Configuration name; default is the collection name.") + .build(), + Option.builder("configsetsDir") + .argName("DIR") + .hasArg() + .required(true) + .desc("Path to configsets directory on the local system.") + .build(), + OPTION_VERBOSE + }; + + public static void exit(int exitStatus) { try { System.exit(exitStatus); - } catch (java.lang.SecurityException secExc) { + } catch (SecurityException secExc) { if (exitStatus != 0) throw new RuntimeException("SolrCLI failed to exit with status " + exitStatus); } @@ -310,14 +243,6 @@ public static Tool findTool(String[] args) throws Exception { return newTool(toolType); } - /** - * @deprecated Use the method that takes a tool name as the first argument instead. - */ - @Deprecated - public static CommandLine parseCmdLine(String[] args, Option[] toolOptions) throws Exception { - return parseCmdLine(SolrCLI.class.getName(), args, toolOptions); - } - public static CommandLine parseCmdLine(String toolName, String[] args, Option[] toolOptions) throws Exception { // the parser doesn't like -D props @@ -334,8 +259,7 @@ public static CommandLine parseCmdLine(String toolName, String[] args, Option[] String[] toolArgs = toolArgList.toArray(new String[0]); // process command-line args to configure this application - CommandLine cli = - processCommandLineArgs(toolName, joinCommonAndToolOptions(toolOptions), toolArgs); + CommandLine cli = processCommandLineArgs(toolName, toolOptions, toolArgs); List argList = cli.getArgList(); argList.addAll(dashDList); @@ -373,17 +297,12 @@ protected static void checkSslStoreSysProp(String solrInstallDir, String key) { } } - private static void raiseLogLevelUnlessVerbose(CommandLine cli) { + public static void raiseLogLevelUnlessVerbose(CommandLine cli) { if (!cli.hasOption(OPTION_VERBOSE.getOpt())) { StartupLoggingUtils.changeLogLevel("WARN"); } } - /** Support options common to all tools. */ - public static Option[] getCommonToolOptions() { - return new Option[0]; - } - // Creates an instance of the requested tool, using classpath scanning if necessary private static Tool newTool(String toolType) throws Exception { if ("healthcheck".equals(toolType)) return new HealthcheckTool(); @@ -410,7 +329,8 @@ private static Tool newTool(String toolType) throws Exception { // If you add a built-in tool to this class, add it here to avoid // classpath scanning - for (Class next : findToolClassesInPackage("org.apache.solr.util")) { + for (Class next : + findToolClassesInPackage(List.of("org.apache.solr.util", "org.apache.solr.util.cli"))) { Tool tool = next.getConstructor().newInstance(); if (toolType.equals(tool.getName())) return tool; } @@ -432,32 +352,34 @@ private static void displayToolOptions() throws Exception { formatter.printHelp("upconfig", getToolOptions(new ConfigSetUploadTool())); formatter.printHelp("downconfig", getToolOptions(new ConfigSetDownloadTool())); formatter.printHelp("rm", getToolOptions(new ZkRmTool())); - formatter.printHelp("cp", getToolOptions(new ZkCpTool())); formatter.printHelp("mv", getToolOptions(new ZkMvTool())); + formatter.printHelp("cp", getToolOptions(new ZkCpTool())); formatter.printHelp("ls", getToolOptions(new ZkLsTool())); + formatter.printHelp("mkroot", getToolOptions(new ZkMkrootTool())); + formatter.printHelp("assert", getToolOptions(new AssertTool())); + formatter.printHelp("auth", getToolOptions(new AuthTool())); formatter.printHelp("export", getToolOptions(new ExportTool())); formatter.printHelp("package", getToolOptions(new PackageTool())); - List> toolClasses = findToolClassesInPackage("org.apache.solr.util"); + List> toolClasses = + findToolClassesInPackage((List.of("org.apache.solr.util", "org.apache.solr.util.cli"))); for (Class next : toolClasses) { Tool tool = next.getConstructor().newInstance(); formatter.printHelp(tool.getName(), getToolOptions(tool)); } } - private static Options getToolOptions(Tool tool) { + public static Options getToolOptions(Tool tool) { Options options = new Options(); options.addOption("help", false, "Print this message"); options.addOption(OPTION_VERBOSE); - Option[] toolOpts = joinCommonAndToolOptions(tool.getOptions()); - for (int i = 0; i < toolOpts.length; i++) options.addOption(toolOpts[i]); + Option[] toolOpts = tool.getOptions(); + for (int i = 0; i < toolOpts.length; i++) { + options.addOption(toolOpts[i]); + } return options; } - public static Option[] joinCommonAndToolOptions(Option[] toolOpts) { - return joinOptions(getCommonToolOptions(), toolOpts); - } - public static Option[] joinOptions(Option[] lhs, Option[] rhs) { if (lhs == null) { return rhs == null ? new Option[0] : rhs; @@ -474,14 +396,6 @@ public static Option[] joinOptions(Option[] lhs, Option[] rhs) { return options; } - /** - * @deprecated Use the method that takes a tool name as the first argument instead. - */ - @Deprecated - public static CommandLine processCommandLineArgs(Option[] customOptions, String[] args) { - return processCommandLineArgs(SolrCLI.class.getName(), customOptions, args); - } - /** Parses the command-line arguments passed by the user. */ public static CommandLine processCommandLineArgs( String toolName, Option[] customOptions, String[] args) { @@ -525,25 +439,28 @@ public static CommandLine processCommandLineArgs( } /** Scans Jar files on the classpath for Tool implementations to activate. */ - private static List> findToolClassesInPackage(String packageName) { + private static List> findToolClassesInPackage(List packageNames) { List> toolClasses = new ArrayList<>(); - try { - ClassLoader classLoader = SolrCLI.class.getClassLoader(); - String path = packageName.replace('.', '/'); - Enumeration resources = classLoader.getResources(path); - Set classes = new TreeSet<>(); - while (resources.hasMoreElements()) { - URL resource = resources.nextElement(); - classes.addAll(findClasses(resource.getFile(), packageName)); - } + for (String packageName : packageNames) { + try { + ClassLoader classLoader = SolrCLI.class.getClassLoader(); + String path = packageName.replace('.', '/'); + Enumeration resources = classLoader.getResources(path); + Set classes = new TreeSet<>(); + while (resources.hasMoreElements()) { + URL resource = resources.nextElement(); + classes.addAll(findClasses(resource.getFile(), packageName)); + } - for (String classInPackage : classes) { - Class theClass = Class.forName(classInPackage); - if (Tool.class.isAssignableFrom(theClass)) toolClasses.add(theClass.asSubclass(Tool.class)); + for (String classInPackage : classes) { + Class theClass = Class.forName(classInPackage); + if (Tool.class.isAssignableFrom(theClass)) + toolClasses.add(theClass.asSubclass(Tool.class)); + } + } catch (Exception e) { + // safe to squelch this as it's just looking for tools to run + log.debug("Failed to find Tool impl classes in {}, due to: ", packageName, e); } - } catch (Exception e) { - // safe to squelch this as it's just looking for tools to run - log.debug("Failed to find Tool impl classes in {}, due to: ", packageName, e); } return toolClasses; } @@ -594,7 +511,7 @@ public static boolean checkCommunicationError(Exception exc) { * @throws SolrException if auth/autz problems * @throws IOException if connection failure */ - private static int attemptHttpHead(String url, HttpClient httpClient) + public static int attemptHttpHead(String url, HttpClient httpClient) throws SolrException, IOException { HttpResponse response = httpClient.execute(new HttpHead(url), HttpClientUtil.createNewHttpClientRequestContext()); @@ -610,7 +527,7 @@ private static int attemptHttpHead(String url, HttpClient httpClient) return code; } - private static boolean exceptionIsAuthRelated(Exception exc) { + public static boolean exceptionIsAuthRelated(Exception exc) { return (exc instanceof SolrException && Arrays.asList(UNAUTHORIZED.code, FORBIDDEN.code).contains(((SolrException) exc).code())); } @@ -623,17 +540,6 @@ public static CloseableHttpClient getHttpClient() { return HttpClientUtil.createClient(params); } - @SuppressWarnings("deprecation") - public static void closeHttpClient(CloseableHttpClient httpClient) { - if (httpClient != null) { - try { - HttpClientUtil.close(httpClient); - } catch (Exception exc) { - // safe to ignore, we're just shutting things down - } - } - } - public static final String JSON_CONTENT_TYPE = "application/json"; public static NamedList postJsonToSolr( @@ -648,11 +554,9 @@ public static NamedList postJsonToSolr( /** Useful when a tool just needs to send one request to Solr. */ public static Map getJson(String getUrl) throws Exception { Map json = null; - CloseableHttpClient httpClient = getHttpClient(); - try { + ; + try (CloseableHttpClient httpClient = getHttpClient()) { json = getJson(httpClient, getUrl, 2, true); - } finally { - closeHttpClient(httpClient); } return json; } @@ -869,219 +773,16 @@ public static Object atPath(String jsonPath, Map json) { return result; } - /** Get the status of a Solr server. */ - public static class StatusTool extends ToolBase { - - public StatusTool() { - this(CLIO.getOutStream()); - } - - public StatusTool(PrintStream stdout) { - super(stdout); - } - - @Override - public String getName() { - return "status"; - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("solr") - .argName("URL") - .hasArg() - .required(false) - .desc("Address of the Solr Web application, defaults to: " + DEFAULT_SOLR_URL + '.') - .build(), - Option.builder("maxWaitSecs") - .argName("SECS") - .hasArg() - .required(false) - .desc("Wait up to the specified number of seconds to see Solr running.") - .build() - }; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - int maxWaitSecs = Integer.parseInt(cli.getOptionValue("maxWaitSecs", "0")); - String solrUrl = cli.getOptionValue("solr", DEFAULT_SOLR_URL); - if (maxWaitSecs > 0) { - int solrPort = (new URL(solrUrl)).getPort(); - echo("Waiting up to " + maxWaitSecs + " seconds to see Solr running on port " + solrPort); - try { - waitToSeeSolrUp(solrUrl, maxWaitSecs); - echo("Started Solr server on port " + solrPort + ". Happy searching!"); - } catch (TimeoutException timeout) { - throw new Exception( - "Solr at " + solrUrl + " did not come online within " + maxWaitSecs + " seconds!"); - } - } else { - try { - CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(getStatus(solrUrl)); - echo(arr.toString()); - } catch (Exception exc) { - if (exceptionIsAuthRelated(exc)) { - throw exc; - } - if (checkCommunicationError(exc)) { - // this is not actually an error from the tool as it's ok if Solr is not online. - CLIO.err("Solr at " + solrUrl + " not online."); - } else { - throw new Exception( - "Failed to get system information from " + solrUrl + " due to: " + exc); - } - } - } - } - - public Map waitToSeeSolrUp(String solrUrl, int maxWaitSecs) throws Exception { - long timeout = - System.nanoTime() + TimeUnit.NANOSECONDS.convert(maxWaitSecs, TimeUnit.SECONDS); - while (System.nanoTime() < timeout) { - try { - return getStatus(solrUrl); - } catch (SSLPeerUnverifiedException exc) { - throw exc; - } catch (Exception exc) { - if (exceptionIsAuthRelated(exc)) { - throw exc; - } - try { - Thread.sleep(2000L); - } catch (InterruptedException interrupted) { - timeout = 0; // stop looping - } - } - } - throw new TimeoutException( - "Did not see Solr at " + solrUrl + " come online within " + maxWaitSecs + " seconds!"); - } - - public Map getStatus(String solrUrl) throws Exception { - Map status = null; - - if (!solrUrl.endsWith("/")) solrUrl += "/"; - - String systemInfoUrl = solrUrl + "admin/info/system"; - CloseableHttpClient httpClient = getHttpClient(); - try { - // hit Solr to get system info - Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); - // convert raw JSON into user-friendly output - status = reportStatus(solrUrl, systemInfo, httpClient); - } finally { - closeHttpClient(httpClient); - } - - return status; - } - - public Map reportStatus( - String solrUrl, Map info, HttpClient httpClient) throws Exception { - Map status = new LinkedHashMap<>(); - - String solrHome = (String) info.get("solr_home"); - status.put("solr_home", solrHome != null ? solrHome : "?"); - status.put("version", asString("/lucene/solr-impl-version", info)); - status.put("startTime", asString("/jvm/jmx/startTime", info)); - status.put("uptime", uptime(asLong("/jvm/jmx/upTimeMS", info))); - - String usedMemory = asString("/jvm/memory/used", info); - String totalMemory = asString("/jvm/memory/total", info); - status.put("memory", usedMemory + " of " + totalMemory); - - // if this is a Solr in solrcloud mode, gather some basic cluster info - if ("solrcloud".equals(info.get("mode"))) { - String zkHost = (String) info.get("zkHost"); - status.put("cloud", getCloudStatus(httpClient, solrUrl, zkHost)); - } - - return status; - } - - /** - * Calls the CLUSTERSTATUS endpoint in Solr to get basic status information about the SolrCloud - * cluster. - */ - protected Map getCloudStatus( - HttpClient httpClient, String solrUrl, String zkHost) throws Exception { - Map cloudStatus = new LinkedHashMap<>(); - cloudStatus.put("ZooKeeper", (zkHost != null) ? zkHost : "?"); - - String clusterStatusUrl = solrUrl + "admin/collections?action=CLUSTERSTATUS"; - Map json = getJson(httpClient, clusterStatusUrl, 2, true); - - List liveNodes = asList("/cluster/live_nodes", json); - cloudStatus.put("liveNodes", String.valueOf(liveNodes.size())); - - Map collections = asMap("/cluster/collections", json); - cloudStatus.put("collections", String.valueOf(collections.size())); - - return cloudStatus; - } - } // end StatusTool class - - /** Used to send an arbitrary HTTP request to a Solr API endpoint. */ - public static class ApiTool extends ToolBase { - - public ApiTool() { - this(CLIO.getOutStream()); - } - - public ApiTool(PrintStream stdout) { - super(stdout); - } - - @Override - public String getName() { - return "api"; - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("get") - .argName("URL") - .hasArg() - .required(true) - .desc("Send a GET request to a Solr API endpoint.") - .build() - }; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - String getUrl = cli.getOptionValue("get"); - if (getUrl != null) { - Map json = getJson(getUrl); - - // pretty-print the response to stdout - CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(json); - echo(arr.toString()); - } - } - } // end ApiTool class - - private static final String DEFAULT_CONFIG_SET = "_default"; - - private static final long MS_IN_MIN = 60 * 1000L; - private static final long MS_IN_HOUR = MS_IN_MIN * 60L; - private static final long MS_IN_DAY = MS_IN_HOUR * 24L; - @VisibleForTesting - static final String uptime(long uptimeMs) { + public static final String uptime(long uptimeMs) { if (uptimeMs <= 0L) return "?"; - long numDays = (uptimeMs >= MS_IN_DAY) ? (uptimeMs / MS_IN_DAY) : 0L; - long rem = uptimeMs - (numDays * MS_IN_DAY); - long numHours = (rem >= MS_IN_HOUR) ? (rem / MS_IN_HOUR) : 0L; - rem = rem - (numHours * MS_IN_HOUR); - long numMinutes = (rem >= MS_IN_MIN) ? (rem / MS_IN_MIN) : 0L; - rem = rem - (numMinutes * MS_IN_MIN); + long numDays = TimeUnit.MILLISECONDS.toDays(uptimeMs); + long rem = uptimeMs - TimeUnit.DAYS.toMillis(numDays); + long numHours = TimeUnit.MILLISECONDS.toHours(rem); + rem = rem - TimeUnit.HOURS.toMillis(numHours); + long numMinutes = TimeUnit.MILLISECONDS.toMinutes(rem); + rem = rem - TimeUnit.MINUTES.toMillis(numMinutes); long numSeconds = Math.round(rem / 1000.0); return String.format( Locale.ROOT, @@ -1092,7 +793,7 @@ static final String uptime(long uptimeMs) { numSeconds); } - static class ReplicaHealth implements Comparable { + public static class ReplicaHealth implements Comparable { String shard; String name; String url; @@ -1102,7 +803,7 @@ static class ReplicaHealth implements Comparable { String uptime; String memory; - ReplicaHealth( + public ReplicaHealth( String shard, String name, String url, @@ -1172,18 +873,18 @@ public int compareTo(ReplicaHealth other) { } } - enum ShardState { + public enum ShardState { healthy, degraded, down, no_leader } - static class ShardHealth { + public static class ShardHealth { String shard; List replicas; - ShardHealth(String shard, List replicas) { + public ShardHealth(String shard, List replicas) { this.shard = shard; this.replicas = replicas; } @@ -1227,270 +928,75 @@ public String toString() { } } - /** Requests health information about a specific collection in SolrCloud. */ - public static class HealthcheckTool extends SolrCloudTool { + /** + * Get the base URL of a live Solr instance from either the solrUrl command-line option from + * ZooKeeper. + */ + public static String resolveSolrUrl(CommandLine cli) throws Exception { + String solrUrl = cli.getOptionValue("solrUrl"); + if (solrUrl == null) { + String zkHost = cli.getOptionValue("zkHost"); + if (zkHost == null) + throw new IllegalStateException( + "Must provide either the '-solrUrl' or '-zkHost' parameters!"); - public HealthcheckTool() { - this(CLIO.getOutStream()); - } + try (CloudSolrClient cloudSolrClient = + new CloudLegacySolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) + .build()) { + cloudSolrClient.connect(); + Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); + if (liveNodes.isEmpty()) + throw new IllegalStateException( + "No live nodes found! Cannot determine 'solrUrl' from ZooKeeper: " + zkHost); - public HealthcheckTool(PrintStream stdout) { - super(stdout); + String firstLiveNode = liveNodes.iterator().next(); + solrUrl = ZkStateReader.from(cloudSolrClient).getBaseUrlForNodeName(firstLiveNode); + } } + return solrUrl; + } - @Override - public String getName() { - return "healthcheck"; + /** + * Get the ZooKeeper connection string from either the zkHost command-line option or by looking it + * up from a running Solr instance based on the solrUrl option. + */ + public static String getZkHost(CommandLine cli) throws Exception { + String zkHost = cli.getOptionValue("zkHost"); + if (zkHost != null) { + return zkHost; } - @Override - protected void runCloudTool(CloudLegacySolrClient cloudSolrClient, CommandLine cli) - throws Exception { - raiseLogLevelUnlessVerbose(cli); - String collection = cli.getOptionValue("collection"); - if (collection == null) - throw new IllegalArgumentException( - "Must provide a collection to run a healthcheck against!"); - - log.debug("Running healthcheck for {}", collection); - - ZkStateReader zkStateReader = ZkStateReader.from(cloudSolrClient); + // find it using the localPort + String solrUrl = cli.getOptionValue("solrUrl"); + if (solrUrl == null) + throw new IllegalStateException( + "Must provide either the -zkHost or -solrUrl parameters to use this command!"); - ClusterState clusterState = zkStateReader.getClusterState(); - Set liveNodes = clusterState.getLiveNodes(); - final DocCollection docCollection = clusterState.getCollectionOrNull(collection); - if (docCollection == null || docCollection.getSlices() == null) - throw new IllegalArgumentException("Collection " + collection + " not found!"); + if (!solrUrl.endsWith("/")) { + solrUrl += "/"; + } - Collection slices = docCollection.getSlices(); - // Test http code using a HEAD request first, fail fast if authentication failure - String urlForColl = - zkStateReader.getLeaderUrl(collection, slices.stream().findFirst().get().getName(), 1000); - attemptHttpHead(urlForColl, cloudSolrClient.getHttpClient()); + String systemInfoUrl = solrUrl + "admin/info/system"; + try (CloseableHttpClient httpClient = getHttpClient()) { + // hit Solr to get system info + Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); - SolrQuery q = new SolrQuery("*:*"); - q.setRows(0); - QueryResponse qr = cloudSolrClient.query(collection, q); - String collErr = null; - long docCount = -1; - try { - docCount = qr.getResults().getNumFound(); - } catch (Exception exc) { - collErr = String.valueOf(exc); + // convert raw JSON into user-friendly output + StatusTool statusTool = new StatusTool(); + Map status = statusTool.reportStatus(solrUrl, systemInfo, httpClient); + @SuppressWarnings("unchecked") + Map cloud = (Map) status.get("cloud"); + if (cloud != null) { + String zookeeper = (String) cloud.get("ZooKeeper"); + if (zookeeper.endsWith("(embedded)")) { + zookeeper = zookeeper.substring(0, zookeeper.length() - "(embedded)".length()); + } + zkHost = zookeeper; } + } - List shardList = new ArrayList<>(); - boolean collectionIsHealthy = (docCount != -1); - - for (Slice slice : slices) { - String shardName = slice.getName(); - // since we're reporting health of this shard, there's no guarantee of a leader - String leaderUrl = null; - try { - leaderUrl = zkStateReader.getLeaderUrl(collection, shardName, 1000); - } catch (Exception exc) { - log.warn("Failed to get leader for shard {} due to: {}", shardName, exc); - } - - List replicaList = new ArrayList<>(); - for (Replica r : slice.getReplicas()) { - - String uptime = null; - String memory = null; - String replicaStatus = null; - long numDocs = -1L; - - ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r); - String coreUrl = replicaCoreProps.getCoreUrl(); - boolean isLeader = coreUrl.equals(leaderUrl); - - // if replica's node is not live, its status is DOWN - String nodeName = replicaCoreProps.getNodeName(); - if (nodeName == null || !liveNodes.contains(nodeName)) { - replicaStatus = Replica.State.DOWN.toString(); - } else { - // query this replica directly to get doc count and assess health - q = new SolrQuery("*:*"); - q.setRows(0); - q.set(DISTRIB, "false"); - try (HttpSolrClient solr = new HttpSolrClient.Builder(coreUrl).build()) { - - String solrUrl = solr.getBaseURL(); - - qr = solr.query(q); - numDocs = qr.getResults().getNumFound(); - - int lastSlash = solrUrl.lastIndexOf('/'); - String systemInfoUrl = solrUrl.substring(0, lastSlash) + "/admin/info/system"; - Map info = getJson(solr.getHttpClient(), systemInfoUrl, 2, true); - uptime = uptime(asLong("/jvm/jmx/upTimeMS", info)); - String usedMemory = asString("/jvm/memory/used", info); - String totalMemory = asString("/jvm/memory/total", info); - memory = usedMemory + " of " + totalMemory; - - // if we get here, we can trust the state - replicaStatus = replicaCoreProps.getState(); - } catch (Exception exc) { - log.error("ERROR: {} when trying to reach: {}", exc, coreUrl); - - if (checkCommunicationError(exc)) { - replicaStatus = Replica.State.DOWN.toString(); - } else { - replicaStatus = "error: " + exc; - } - } - } - - replicaList.add( - new ReplicaHealth( - shardName, - r.getName(), - coreUrl, - replicaStatus, - numDocs, - isLeader, - uptime, - memory)); - } - - ShardHealth shardHealth = new ShardHealth(shardName, replicaList); - if (ShardState.healthy != shardHealth.getShardState()) - collectionIsHealthy = false; // at least one shard is un-healthy - - shardList.add(shardHealth.asMap()); - } - - Map report = new LinkedHashMap<>(); - report.put("collection", collection); - report.put("status", collectionIsHealthy ? "healthy" : "degraded"); - if (collErr != null) { - report.put("error", collErr); - } - report.put("numDocs", docCount); - report.put("numShards", slices.size()); - report.put("shards", shardList); - - CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(report); - echo(arr.toString()); - } - } // end HealthcheckTool - - private static final Option[] CREATE_COLLECTION_OPTIONS = - new Option[] { - OPTION_ZKHOST, - OPTION_SOLRURL, - Option.builder(NAME) - .argName("NAME") - .hasArg() - .required(true) - .desc("Name of collection to create.") - .build(), - Option.builder("shards") - .argName("#") - .hasArg() - .required(false) - .desc("Number of shards; default is 1.") - .build(), - Option.builder("replicationFactor") - .argName("#") - .hasArg() - .required(false) - .desc( - "Number of copies of each document across the collection (replicas per shard); default is 1.") - .build(), - Option.builder("confdir") - .argName("NAME") - .hasArg() - .required(false) - .desc( - "Configuration directory to copy when creating the new collection; default is " - + DEFAULT_CONFIG_SET - + '.') - .build(), - Option.builder("confname") - .argName("NAME") - .hasArg() - .required(false) - .desc("Configuration name; default is the collection name.") - .build(), - Option.builder("configsetsDir") - .argName("DIR") - .hasArg() - .required(true) - .desc("Path to configsets directory on the local system.") - .build(), - OPTION_VERBOSE - }; - - /** - * Get the base URL of a live Solr instance from either the solrUrl command-line option from - * ZooKeeper. - */ - public static String resolveSolrUrl(CommandLine cli) throws Exception { - String solrUrl = cli.getOptionValue("solrUrl"); - if (solrUrl == null) { - String zkHost = cli.getOptionValue("zkHost"); - if (zkHost == null) - throw new IllegalStateException( - "Must provide either the '-solrUrl' or '-zkHost' parameters!"); - - try (CloudSolrClient cloudSolrClient = - new CloudLegacySolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) - .build()) { - cloudSolrClient.connect(); - Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); - if (liveNodes.isEmpty()) - throw new IllegalStateException( - "No live nodes found! Cannot determine 'solrUrl' from ZooKeeper: " + zkHost); - - String firstLiveNode = liveNodes.iterator().next(); - solrUrl = ZkStateReader.from(cloudSolrClient).getBaseUrlForNodeName(firstLiveNode); - } - } - return solrUrl; - } - - /** - * Get the ZooKeeper connection string from either the zkHost command-line option or by looking it - * up from a running Solr instance based on the solrUrl option. - */ - public static String getZkHost(CommandLine cli) throws Exception { - String zkHost = cli.getOptionValue("zkHost"); - if (zkHost != null) return zkHost; - - // find it using the localPort - String solrUrl = cli.getOptionValue("solrUrl"); - if (solrUrl == null) - throw new IllegalStateException( - "Must provide either the -zkHost or -solrUrl parameters to use the create_collection command!"); - - if (!solrUrl.endsWith("/")) solrUrl += "/"; - - String systemInfoUrl = solrUrl + "admin/info/system"; - CloseableHttpClient httpClient = getHttpClient(); - try { - // hit Solr to get system info - Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); - - // convert raw JSON into user-friendly output - StatusTool statusTool = new StatusTool(); - Map status = statusTool.reportStatus(solrUrl, systemInfo, httpClient); - @SuppressWarnings("unchecked") - Map cloud = (Map) status.get("cloud"); - if (cloud != null) { - String zookeeper = (String) cloud.get("ZooKeeper"); - if (zookeeper.endsWith("(embedded)")) { - zookeeper = zookeeper.substring(0, zookeeper.length() - "(embedded)".length()); - } - zkHost = zookeeper; - } - } finally { - HttpClientUtil.close(httpClient); - } - - return zkHost; - } + return zkHost; + } public static boolean safeCheckCollectionExists(String url, String collection) { boolean exists = false; @@ -1534,3194 +1040,9 @@ public static boolean safeCheckCoreExists(String coreStatusUrl, String coreName) return exists; } - /** Supports create_collection command in the bin/solr script. */ - public static class CreateCollectionTool extends ToolBase { - - public CreateCollectionTool() { - this(CLIO.getOutStream()); - } - - public CreateCollectionTool(PrintStream stdout) { - super(stdout); - } - - @Override - public String getName() { - return "create_collection"; - } - - @Override - public Option[] getOptions() { - return CREATE_COLLECTION_OPTIONS; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String zkHost = getZkHost(cli); - if (zkHost == null) { - throw new IllegalStateException( - "Solr at " - + cli.getOptionValue("solrUrl") - + " is running in standalone server mode, please use the create_core command instead;\n" - + "create_collection can only be used when running in SolrCloud mode.\n"); - } - - try (CloudSolrClient cloudSolrClient = - new CloudLegacySolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) - .build()) { - echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); - cloudSolrClient.connect(); - runCloudTool(cloudSolrClient, cli); - } - } - - protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) throws Exception { - - Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); - if (liveNodes.isEmpty()) - throw new IllegalStateException( - "No live nodes found! Cannot create a collection until " - + "there is at least 1 live node in the cluster."); - - String baseUrl = cli.getOptionValue("solrUrl"); - if (baseUrl == null) { - String firstLiveNode = liveNodes.iterator().next(); - baseUrl = ZkStateReader.from(cloudSolrClient).getBaseUrlForNodeName(firstLiveNode); - } - - String collectionName = cli.getOptionValue(NAME); - - // build a URL to create the collection - int numShards = optionAsInt(cli, "shards", 1); - int replicationFactor = optionAsInt(cli, "replicationFactor", 1); - - String confname = cli.getOptionValue("confname"); - String confdir = cli.getOptionValue("confdir"); - String configsetsDir = cli.getOptionValue("configsetsDir"); - - boolean configExistsInZk = - confname != null - && !"".equals(confname.trim()) - && ZkStateReader.from(cloudSolrClient) - .getZkClient() - .exists("/configs/" + confname, true); - - if (CollectionAdminParams.SYSTEM_COLL.equals(collectionName)) { - // do nothing - } else if (configExistsInZk) { - echo("Re-using existing configuration directory " + confname); - } else if (confdir != null && !"".equals(confdir.trim())) { - if (confname == null || "".equals(confname.trim())) { - confname = collectionName; - } - Path confPath = ConfigSetService.getConfigsetPath(confdir, configsetsDir); - - echoIfVerbose( - "Uploading " - + confPath.toAbsolutePath() - + " for config " - + confname - + " to ZooKeeper at " - + cloudSolrClient.getClusterStateProvider().getQuorumHosts(), - cli); - ZkMaintenanceUtils.uploadToZK( - ZkStateReader.from(cloudSolrClient).getZkClient(), - confPath, - ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confname, - ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN); - } - - // since creating a collection is a heavy-weight operation, check for existence first - String collectionListUrl = baseUrl + "/admin/collections?action=list"; - if (safeCheckCollectionExists(collectionListUrl, collectionName)) { - throw new IllegalStateException( - "\nCollection '" - + collectionName - + "' already exists!\nChecked collection existence using Collections API command:\n" - + collectionListUrl); - } - - // doesn't seem to exist ... try to create - String createCollectionUrl = - String.format( - Locale.ROOT, - "%s/admin/collections?action=CREATE&name=%s&numShards=%d&replicationFactor=%d", - baseUrl, - collectionName, - numShards, - replicationFactor); - if (confname != null && !"".equals(confname.trim())) { - createCollectionUrl = - createCollectionUrl + String.format(Locale.ROOT, "&collection.configName=%s", confname); - } - - echoIfVerbose( - "\nCreating new collection '" - + collectionName - + "' using command:\n" - + createCollectionUrl - + "\n", - cli); - - Map json = null; - try { - json = getJson(createCollectionUrl); - } catch (SolrServerException sse) { - throw new Exception( - "Failed to create collection '" + collectionName + "' due to: " + sse.getMessage()); - } - - if (cli.hasOption(OPTION_VERBOSE.getOpt())) { - CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(json); - echo(arr.toString()); - } else { - String endMessage = - String.format( - Locale.ROOT, - "Created collection '%s' with %d shard(s), %d replica(s)", - collectionName, - numShards, - replicationFactor); - if (confname != null && !"".equals(confname.trim())) { - endMessage += String.format(Locale.ROOT, " with config-set '%s'", confname); - } - - echo(endMessage); - } - } - - protected int optionAsInt(CommandLine cli, String option, int defaultVal) { - return Integer.parseInt(cli.getOptionValue(option, String.valueOf(defaultVal))); - } - } // end CreateCollectionTool class - - public static class CreateCoreTool extends ToolBase { - - public CreateCoreTool() { - this(CLIO.getOutStream()); - } - - public CreateCoreTool(PrintStream stdout) { - super(stdout); - } - - @Override - public String getName() { - return "create_core"; - } - - @Override - public Option[] getOptions() { - return new Option[] { - OPTION_SOLRURL, - Option.builder(NAME) - .argName("NAME") - .hasArg() - .required(true) - .desc("Name of the core to create.") - .build(), - Option.builder("confdir") - .argName("CONFIG") - .hasArg() - .required(false) - .desc( - "Configuration directory to copy when creating the new core; default is " - + DEFAULT_CONFIG_SET - + '.') - .build(), - Option.builder("configsetsDir") - .argName("DIR") - .hasArg() - .required(true) - .desc("Path to configsets directory on the local system.") - .build(), - OPTION_VERBOSE - }; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - String solrUrl = cli.getOptionValue("solrUrl", DEFAULT_SOLR_URL); - if (!solrUrl.endsWith("/")) solrUrl += "/"; - - File configsetsDir = new File(cli.getOptionValue("configsetsDir")); - if (!configsetsDir.isDirectory()) - throw new FileNotFoundException(configsetsDir.getAbsolutePath() + " not found!"); - - String configSet = cli.getOptionValue("confdir", DEFAULT_CONFIG_SET); - File configSetDir = new File(configsetsDir, configSet); - if (!configSetDir.isDirectory()) { - // we allow them to pass a directory instead of a configset name - File possibleConfigDir = new File(configSet); - if (possibleConfigDir.isDirectory()) { - configSetDir = possibleConfigDir; - } else { - throw new FileNotFoundException( - "Specified config directory " - + configSet - + " not found in " - + configsetsDir.getAbsolutePath()); - } - } - - String coreName = cli.getOptionValue(NAME); - - String systemInfoUrl = solrUrl + "admin/info/system"; - CloseableHttpClient httpClient = getHttpClient(); - String coreRootDirectory = null; // usually same as solr home, but not always - try { - Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); - if ("solrcloud".equals(systemInfo.get("mode"))) { - throw new IllegalStateException( - "Solr at " - + solrUrl - + " is running in SolrCloud mode, please use create_collection command instead."); - } - - // convert raw JSON into user-friendly output - coreRootDirectory = (String) systemInfo.get("core_root"); - - // Fall back to solr_home, in case we are running against older server that does not return - // the property - if (coreRootDirectory == null) coreRootDirectory = (String) systemInfo.get("solr_home"); - if (coreRootDirectory == null) - coreRootDirectory = configsetsDir.getParentFile().getAbsolutePath(); - - } finally { - closeHttpClient(httpClient); - } - - String coreStatusUrl = solrUrl + "admin/cores?action=STATUS&core=" + coreName; - if (safeCheckCoreExists(coreStatusUrl, coreName)) { - throw new IllegalArgumentException( - "\nCore '" - + coreName - + "' already exists!\nChecked core existence using Core API command:\n" - + coreStatusUrl); - } - - File coreInstanceDir = new File(coreRootDirectory, coreName); - File confDir = new File(configSetDir, "conf"); - if (!coreInstanceDir.isDirectory()) { - coreInstanceDir.mkdirs(); - if (!coreInstanceDir.isDirectory()) - throw new IOException( - "Failed to create new core instance directory: " + coreInstanceDir.getAbsolutePath()); - - if (confDir.isDirectory()) { - FileUtils.copyDirectoryToDirectory(confDir, coreInstanceDir); - } else { - // hmmm ... the configset we're cloning doesn't have a conf sub-directory, - // we'll just assume it is OK if it has solrconfig.xml - if ((new File(configSetDir, "solrconfig.xml")).isFile()) { - FileUtils.copyDirectory(configSetDir, new File(coreInstanceDir, "conf")); - } else { - throw new IllegalArgumentException( - "\n" - + configSetDir.getAbsolutePath() - + " doesn't contain a conf subdirectory or solrconfig.xml\n"); - } - } - echoIfVerbose( - "\nCopying configuration to new core instance directory:\n" - + coreInstanceDir.getAbsolutePath(), - cli); - } - - String createCoreUrl = - String.format( - Locale.ROOT, - "%sadmin/cores?action=CREATE&name=%s&instanceDir=%s", - solrUrl, - coreName, - coreName); - - echoIfVerbose( - "\nCreating new core '" + coreName + "' using command:\n" + createCoreUrl + "\n", cli); - - try { - Map json = getJson(createCoreUrl); - if (cli.hasOption(OPTION_VERBOSE.getOpt())) { - CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(json); - echo(arr.toString()); - echo("\n"); - } else { - echo(String.format(Locale.ROOT, "\nCreated new core '%s'", coreName)); - } - } catch (Exception e) { - /* create-core failed, cleanup the copied configset before propagating the error. */ - PathUtils.deleteDirectory(coreInstanceDir.toPath()); - throw e; - } - } - } // end CreateCoreTool class - - public static class CreateTool extends ToolBase { - - public CreateTool() { - this(CLIO.getOutStream()); - } - - public CreateTool(PrintStream stdout) { - super(stdout); + public static class AssertionFailureException extends Exception { + public AssertionFailureException(String message) { + super(message); } - - @Override - public String getName() { - return "create"; - } - - @Override - public Option[] getOptions() { - return CREATE_COLLECTION_OPTIONS; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String solrUrl = cli.getOptionValue("solrUrl", DEFAULT_SOLR_URL); - if (!solrUrl.endsWith("/")) solrUrl += "/"; - - String systemInfoUrl = solrUrl + "admin/info/system"; - CloseableHttpClient httpClient = getHttpClient(); - - ToolBase tool = null; - try { - Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); - if ("solrcloud".equals(systemInfo.get("mode"))) { - tool = new CreateCollectionTool(stdout); - } else { - tool = new CreateCoreTool(stdout); - } - tool.runImpl(cli); - } finally { - closeHttpClient(httpClient); - } - } - } // end CreateTool class - - public static class ConfigSetUploadTool extends ToolBase { - - public ConfigSetUploadTool() { - this(CLIO.getOutStream()); - } - - public ConfigSetUploadTool(PrintStream stdout) { - super(stdout); - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("confname") - .argName("confname") // Comes out in help message - .hasArg() // Has one sub-argument - .required(true) // confname argument must be present - .desc("Configset name in ZooKeeper.") - .build(), // passed as -confname value - Option.builder("confdir") - .argName("confdir") - .hasArg() - .required(true) - .desc("Local directory with configs.") - .build(), - Option.builder("configsetsDir") - .argName("configsetsDir") - .hasArg() - .required(false) - .desc("Parent directory of example configsets.") - .build(), - OPTION_ZKHOST, - OPTION_VERBOSE - }; - } - - @Override - public String getName() { - return "upconfig"; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String zkHost = getZkHost(cli); - if (zkHost == null) { - throw new IllegalStateException( - "Solr at " - + cli.getOptionValue("solrUrl") - + " is running in standalone server mode, upconfig can only be used when running in SolrCloud mode.\n"); - } - - String confName = cli.getOptionValue("confname"); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); - Path confPath = - ConfigSetService.getConfigsetPath( - cli.getOptionValue("confdir"), cli.getOptionValue("configsetsDir")); - - echo( - "Uploading " - + confPath.toAbsolutePath().toString() - + " for config " - + cli.getOptionValue("confname") - + " to ZooKeeper at " - + zkHost); - ZkMaintenanceUtils.uploadToZK( - zkClient, - confPath, - ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName, - ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN); - - } catch (Exception e) { - log.error("Could not complete upconfig operation for reason: ", e); - throw (e); - } - } - } - - public static class ConfigSetDownloadTool extends ToolBase { - - public ConfigSetDownloadTool() { - this(CLIO.getOutStream()); - } - - public ConfigSetDownloadTool(PrintStream stdout) { - super(stdout); - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("confname") - .argName("confname") - .hasArg() - .required(true) - .desc("Configset name in ZooKeeper.") - .build(), - Option.builder("confdir") - .argName("confdir") - .hasArg() - .required(true) - .desc("Local directory with configs.") - .build(), - OPTION_ZKHOST, - OPTION_VERBOSE - }; - } - - @Override - public String getName() { - return "downconfig"; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String zkHost = getZkHost(cli); - if (zkHost == null) { - throw new IllegalStateException( - "Solr at " - + cli.getOptionValue("solrUrl") - + " is running in standalone server mode, downconfig can only be used when running in SolrCloud mode.\n"); - } - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); - String confName = cli.getOptionValue("confname"); - String confDir = cli.getOptionValue("confdir"); - Path configSetPath = Paths.get(confDir); - // we try to be nice about having the "conf" in the directory, and we create it if it's not - // there. - if (configSetPath.endsWith("/conf") == false) { - configSetPath = Paths.get(configSetPath.toString(), "conf"); - } - Files.createDirectories(configSetPath); - echo( - "Downloading configset " - + confName - + " from ZooKeeper at " - + zkHost - + " to directory " - + configSetPath.toAbsolutePath()); - - zkClient.downConfig(confName, configSetPath); - } catch (Exception e) { - log.error("Could not complete downconfig operation for reason: ", e); - throw (e); - } - } - } // End ConfigSetDownloadTool class - - public static class ZkRmTool extends ToolBase { - - public ZkRmTool() { - this(CLIO.getOutStream()); - } - - public ZkRmTool(PrintStream stdout) { - super(stdout); - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("path") - .argName("path") - .hasArg() - .required(true) - .desc("Path to remove.") - .build(), - OPTION_RECURSE, - OPTION_ZKHOST, - OPTION_VERBOSE - }; - } - - @Override - public String getName() { - return "rm"; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String zkHost = getZkHost(cli); - - if (zkHost == null) { - throw new IllegalStateException( - "Solr at " - + cli.getOptionValue("zkHost") - + " is running in standalone server mode, 'zk rm' can only be used when running in SolrCloud mode.\n"); - } - String target = cli.getOptionValue("path"); - Boolean recurse = Boolean.parseBoolean(cli.getOptionValue("recurse")); - - String znode = target; - if (target.toLowerCase(Locale.ROOT).startsWith("zk:")) { - znode = target.substring(3); - } - if (znode.equals("/")) { - throw new SolrServerException("You may not remove the root ZK node ('/')!"); - } - echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - if (recurse == false && zkClient.getChildren(znode, null, true).size() != 0) { - throw new SolrServerException( - "ZooKeeper node " + znode + " has children and recurse has NOT been specified."); - } - echo( - "Removing ZooKeeper node " - + znode - + " from ZooKeeper at " - + zkHost - + " recurse: " - + Boolean.toString(recurse)); - zkClient.clean(znode); - } catch (Exception e) { - log.error("Could not complete rm operation for reason: ", e); - throw (e); - } - } - } // End RmTool class - - public static class ZkLsTool extends ToolBase { - - public ZkLsTool() { - this(CLIO.getOutStream()); - } - - public ZkLsTool(PrintStream stdout) { - super(stdout); - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("path") - .argName("path") - .hasArg() - .required(true) - .desc("Path to list.") - .build(), - OPTION_RECURSE, - OPTION_ZKHOST, - OPTION_VERBOSE - }; - } - - @Override - public String getName() { - return "ls"; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String zkHost = getZkHost(cli); - - if (zkHost == null) { - throw new IllegalStateException( - "Solr at " - + cli.getOptionValue("zkHost") - + " is running in standalone server mode, 'zk ls' can only be used when running in SolrCloud mode.\n"); - } - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); - - String znode = cli.getOptionValue("path"); - Boolean recurse = Boolean.parseBoolean(cli.getOptionValue("recurse")); - echoIfVerbose( - "Getting listing for ZooKeeper node " - + znode - + " from ZooKeeper at " - + zkHost - + " recurse: " - + Boolean.toString(recurse), - cli); - stdout.print(zkClient.listZnode(znode, recurse)); - } catch (Exception e) { - log.error("Could not complete ls operation for reason: ", e); - throw (e); - } - } - } // End zkLsTool class - - public static class ZkMkrootTool extends ToolBase { - - public ZkMkrootTool() { - this(CLIO.getOutStream()); - } - - public ZkMkrootTool(PrintStream stdout) { - super(stdout); - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("path") - .argName("path") - .hasArg() - .required(true) - .desc("Path to create.") - .build(), - OPTION_ZKHOST, - OPTION_VERBOSE - }; - } - - @Override - public String getName() { - return "mkroot"; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String zkHost = getZkHost(cli); - - if (zkHost == null) { - throw new IllegalStateException( - "Solr at " - + cli.getOptionValue("zkHost") - + " is running in standalone server mode, 'zk mkroot' can only be used when running in SolrCloud mode.\n"); - } - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); - - String znode = cli.getOptionValue("path"); - echo("Creating ZooKeeper path " + znode + " on ZooKeeper at " + zkHost); - zkClient.makePath(znode, true); - } catch (Exception e) { - log.error("Could not complete mkroot operation for reason: ", e); - throw (e); - } - } - } // End zkMkrootTool class - - public static class ZkCpTool extends ToolBase { - - public ZkCpTool() { - this(CLIO.getOutStream()); - } - - public ZkCpTool(PrintStream stdout) { - super(stdout); - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("src") - .argName("src") - .hasArg() - .required(true) - .desc("Source file or directory, may be local or a Znode.") - .build(), - Option.builder("dst") - .argName("dst") - .hasArg() - .required(true) - .desc("Destination of copy, may be local or a Znode.") - .build(), - OPTION_RECURSE, - OPTION_ZKHOST, - OPTION_VERBOSE - }; - } - - @Override - public String getName() { - return "cp"; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String zkHost = getZkHost(cli); - if (zkHost == null) { - throw new IllegalStateException( - "Solr at " - + cli.getOptionValue("solrUrl") - + " is running in standalone server mode, cp can only be used when running in SolrCloud mode.\n"); - } - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); - String src = cli.getOptionValue("src"); - String dst = cli.getOptionValue("dst"); - Boolean recurse = Boolean.parseBoolean(cli.getOptionValue("recurse")); - echo("Copying from '" + src + "' to '" + dst + "'. ZooKeeper at " + zkHost); - - boolean srcIsZk = src.toLowerCase(Locale.ROOT).startsWith("zk:"); - boolean dstIsZk = dst.toLowerCase(Locale.ROOT).startsWith("zk:"); - - String srcName = src; - if (srcIsZk) { - srcName = src.substring(3); - } else if (srcName.toLowerCase(Locale.ROOT).startsWith("file:")) { - srcName = srcName.substring(5); - } - - String dstName = dst; - if (dstIsZk) { - dstName = dst.substring(3); - } else { - if (dstName.toLowerCase(Locale.ROOT).startsWith("file:")) { - dstName = dstName.substring(5); - } - } - zkClient.zkTransfer(srcName, srcIsZk, dstName, dstIsZk, recurse); - } catch (Exception e) { - log.error("Could not complete the zk operation for reason: ", e); - throw (e); - } - } - } // End CpTool class - - public static class ZkMvTool extends ToolBase { - - public ZkMvTool() { - this(CLIO.getOutStream()); - } - - public ZkMvTool(PrintStream stdout) { - super(stdout); - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("src") - .argName("src") - .hasArg() - .required(true) - .desc("Source Znode to move from.") - .build(), - Option.builder("dst") - .argName("dst") - .hasArg() - .required(true) - .desc("Destination Znode to move to.") - .build(), - OPTION_ZKHOST, - OPTION_VERBOSE - }; - } - - @Override - public String getName() { - return "mv"; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String zkHost = getZkHost(cli); - if (zkHost == null) { - throw new IllegalStateException( - "Solr at " - + cli.getOptionValue("solrUrl") - + " is running in standalone server mode, downconfig can only be used when running in SolrCloud mode.\n"); - } - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); - String src = cli.getOptionValue("src"); - String dst = cli.getOptionValue("dst"); - - if (src.toLowerCase(Locale.ROOT).startsWith("file:") - || dst.toLowerCase(Locale.ROOT).startsWith("file:")) { - throw new SolrServerException( - "mv command operates on znodes and 'file:' has been specified."); - } - String source = src; - if (src.toLowerCase(Locale.ROOT).startsWith("zk")) { - source = src.substring(3); - } - - String dest = dst; - if (dst.toLowerCase(Locale.ROOT).startsWith("zk")) { - dest = dst.substring(3); - } - - echo("Moving Znode " + source + " to " + dest + " on ZooKeeper at " + zkHost); - zkClient.moveZnode(source, dest); - } catch (Exception e) { - log.error("Could not complete mv operation for reason: ", e); - throw (e); - } - } - } // End MvTool class - - public static class DeleteTool extends ToolBase { - - public DeleteTool() { - this(CLIO.getOutStream()); - } - - public DeleteTool(PrintStream stdout) { - super(stdout); - } - - @Override - public String getName() { - return "delete"; - } - - @Override - public Option[] getOptions() { - return new Option[] { - OPTION_SOLRURL, - Option.builder(NAME) - .argName("NAME") - .hasArg() - .required(true) - .desc("Name of the core / collection to delete.") - .build(), - Option.builder("deleteConfig") - .argName("true|false") - .hasArg() - .required(false) - .desc( - "Flag to indicate if the underlying configuration directory for a collection should also be deleted; default is true.") - .build(), - Option.builder("forceDeleteConfig") - .required(false) - .desc( - "Skip safety checks when deleting the configuration directory used by a collection.") - .build(), - OPTION_ZKHOST, - OPTION_VERBOSE - }; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - String solrUrl = cli.getOptionValue("solrUrl", DEFAULT_SOLR_URL); - if (!solrUrl.endsWith("/")) solrUrl += "/"; - - String systemInfoUrl = solrUrl + "admin/info/system"; - CloseableHttpClient httpClient = getHttpClient(); - try { - Map systemInfo = getJson(httpClient, systemInfoUrl, 2, true); - if ("solrcloud".equals(systemInfo.get("mode"))) { - deleteCollection(cli); - } else { - deleteCore(cli, httpClient, solrUrl); - } - } finally { - closeHttpClient(httpClient); - } - } - - protected void deleteCollection(CommandLine cli) throws Exception { - String zkHost = getZkHost(cli); - try (CloudSolrClient cloudSolrClient = - new CloudLegacySolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) - .withSocketTimeout(30000, TimeUnit.MILLISECONDS) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .build()) { - echoIfVerbose("Connecting to ZooKeeper at " + zkHost, cli); - cloudSolrClient.connect(); - deleteCollection(cloudSolrClient, cli); - } - } - - protected void deleteCollection(CloudSolrClient cloudSolrClient, CommandLine cli) - throws Exception { - Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); - if (liveNodes.isEmpty()) - throw new IllegalStateException( - "No live nodes found! Cannot delete a collection until " - + "there is at least 1 live node in the cluster."); - - String firstLiveNode = liveNodes.iterator().next(); - ZkStateReader zkStateReader = ZkStateReader.from(cloudSolrClient); - String baseUrl = zkStateReader.getBaseUrlForNodeName(firstLiveNode); - String collectionName = cli.getOptionValue(NAME); - if (!zkStateReader.getClusterState().hasCollection(collectionName)) { - throw new IllegalArgumentException("Collection " + collectionName + " not found!"); - } - - String configName = - zkStateReader.getClusterState().getCollection(collectionName).getConfigName(); - boolean deleteConfig = "true".equals(cli.getOptionValue("deleteConfig", "true")); - if (deleteConfig && configName != null) { - if (cli.hasOption("forceDeleteConfig")) { - log.warn( - "Skipping safety checks, configuration directory {} will be deleted with impunity.", - configName); - } else { - // need to scan all Collections to see if any are using the config - Set collections = zkStateReader.getClusterState().getCollectionsMap().keySet(); - - // give a little note to the user if there are many collections in case it takes a while - if (collections.size() > 50) - if (log.isInfoEnabled()) { - log.info( - "Scanning {} to ensure no other collections are using config {}", - collections.size(), - configName); - } - - Optional inUse = - collections.stream() - .filter(name -> !name.equals(collectionName)) // ignore this collection - .filter( - name -> - configName.equals( - zkStateReader.getClusterState().getCollection(name).getConfigName())) - .findFirst(); - if (inUse.isPresent()) { - deleteConfig = false; - log.warn( - "Configuration directory {} is also being used by {}{}", - configName, - inUse.get(), - "; configuration will not be deleted from ZooKeeper. You can pass the -forceDeleteConfig flag to force delete."); - } - } - } - - String deleteCollectionUrl = - String.format( - Locale.ROOT, "%s/admin/collections?action=DELETE&name=%s", baseUrl, collectionName); - - echoIfVerbose( - "\nDeleting collection '" - + collectionName - + "' using command:\n" - + deleteCollectionUrl - + "\n", - cli); - - Map json = null; - try { - json = getJson(deleteCollectionUrl); - } catch (SolrServerException sse) { - throw new Exception( - "Failed to delete collection '" + collectionName + "' due to: " + sse.getMessage()); - } - - if (deleteConfig) { - String configZnode = "/configs/" + configName; - try { - zkStateReader.getZkClient().clean(configZnode); - } catch (Exception exc) { - echo( - "\nWARNING: Failed to delete configuration directory " - + configZnode - + " in ZooKeeper due to: " - + exc.getMessage() - + "\nYou'll need to manually delete this znode using the zkcli script."); - } - } - - if (json != null) { - CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(json); - echo(arr.toString()); - echo("\n"); - } - - echo("Deleted collection '" + collectionName + "' using command:\n" + deleteCollectionUrl); - } - - protected void deleteCore(CommandLine cli, CloseableHttpClient httpClient, String solrUrl) - throws Exception { - String coreName = cli.getOptionValue(NAME); - String deleteCoreUrl = - String.format( - Locale.ROOT, - "%sadmin/cores?action=UNLOAD&core=%s&deleteIndex=true&deleteDataDir=true&deleteInstanceDir=true", - solrUrl, - coreName); - - echo("\nDeleting core '" + coreName + "' using command:\n" + deleteCoreUrl + "\n"); - - Map json = null; - try { - json = getJson(deleteCoreUrl); - } catch (SolrServerException sse) { - throw new Exception("Failed to delete core '" + coreName + "' due to: " + sse.getMessage()); - } - - if (json != null) { - CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(json); - echoIfVerbose(arr.toString(), cli); - echoIfVerbose("\n", cli); - } - } - } // end DeleteTool class - - /** Sends a POST to the Config API to perform a specified action. */ - public static class ConfigTool extends ToolBase { - - public ConfigTool() { - this(CLIO.getOutStream()); - } - - public ConfigTool(PrintStream stdout) { - super(stdout); - } - - @Override - public String getName() { - return "config"; - } - - @Override - public Option[] getOptions() { - Option[] configOptions = - new Option[] { - Option.builder("action") - .argName("ACTION") - .hasArg() - .required(false) - .desc( - "Config API action, one of: set-property, unset-property; default is 'set-property'.") - .build(), - Option.builder("property") - .argName("PROP") - .hasArg() - .required(true) - .desc( - "Name of the Config API property to apply the action to, such as: 'updateHandler.autoSoftCommit.maxTime'.") - .build(), - Option.builder("value") - .argName("VALUE") - .hasArg() - .required(false) - .desc("Set the property to this value; accepts JSON objects and strings.") - .build(), - OPTION_SOLRURL, - OPTION_ZKHOST, - Option.builder("p") - .argName("PORT") - .hasArg() - .required(false) - .desc("The port of the Solr node to use when applying configuration change.") - .longOpt("port") - .build(), - Option.builder("s") - .argName("SCHEME") - .hasArg() - .required(false) - .desc( - "The scheme for accessing Solr. Accepted values: http or https. Default is 'http'") - .longOpt("scheme") - .build() - }; - return joinOptions(configOptions, cloudOptions); - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - String solrUrl; - try { - solrUrl = resolveSolrUrl(cli); - } catch (IllegalStateException e) { - // Fallback to using the provided scheme and port - final String scheme = cli.getOptionValue("scheme", "http"); - if (cli.hasOption("port")) { - solrUrl = scheme + "://localhost:" + cli.getOptionValue("port", "8983") + "/solr"; - } else { - throw e; - } - } - - String action = cli.getOptionValue("action", "set-property"); - String collection = cli.getOptionValue("collection", "gettingstarted"); - String property = cli.getOptionValue("property"); - String value = cli.getOptionValue("value"); - - Map jsonObj = new HashMap<>(); - if (value != null) { - Map setMap = new HashMap<>(); - setMap.put(property, value); - jsonObj.put(action, setMap); - } else { - jsonObj.put(action, property); - } - - CharArr arr = new CharArr(); - (new JSONWriter(arr, 0)).write(jsonObj); - String jsonBody = arr.toString(); - - String updatePath = "/" + collection + "/config"; - - echo("\nPOSTing request to Config API: " + solrUrl + updatePath); - echo(jsonBody); - - try (SolrClient solrClient = new HttpSolrClient.Builder(solrUrl).build()) { - NamedList result = postJsonToSolr(solrClient, updatePath, jsonBody); - Integer statusCode = (Integer) ((NamedList) result.get("responseHeader")).get("status"); - if (statusCode == 0) { - if (value != null) { - echo("Successfully " + action + " " + property + " to " + value); - } else { - echo("Successfully " + action + " " + property); - } - } else { - throw new Exception("Failed to " + action + " property due to:\n" + result); - } - } - } - } // end ConfigTool class - - /** Supports an interactive session with the user to launch (or relaunch the -e cloud example) */ - public static class RunExampleTool extends ToolBase { - - private static final String PROMPT_FOR_NUMBER = "Please enter %s [%d]: "; - private static final String PROMPT_FOR_NUMBER_IN_RANGE = - "Please enter %s between %d and %d [%d]: "; - private static final String PROMPT_NUMBER_TOO_SMALL = - "%d is too small! " + PROMPT_FOR_NUMBER_IN_RANGE; - private static final String PROMPT_NUMBER_TOO_LARGE = - "%d is too large! " + PROMPT_FOR_NUMBER_IN_RANGE; - - protected InputStream userInput; - protected Executor executor; - protected String script; - protected File serverDir; - protected File exampleDir; - protected String urlScheme; - - /** Default constructor used by the framework when running as a command-line application. */ - public RunExampleTool() { - this(null, System.in, CLIO.getOutStream()); - } - - public RunExampleTool(Executor executor, InputStream userInput, PrintStream stdout) { - super(stdout); - this.executor = (executor != null) ? executor : new DefaultExecutor(); - this.userInput = userInput; - } - - @Override - public String getName() { - return "run_example"; - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("noprompt") - .required(false) - .desc( - "Don't prompt for input; accept all defaults when running examples that accept user input.") - .build(), - Option.builder("e") - .argName("NAME") - .hasArg() - .required(true) - .desc("Name of the example to launch, one of: cloud, techproducts, schemaless, films.") - .longOpt("example") - .build(), - Option.builder("script") - .argName("PATH") - .hasArg() - .required(false) - .desc("Path to the bin/solr script.") - .build(), - Option.builder("d") - .argName("DIR") - .hasArg() - .required(true) - .desc("Path to the Solr server directory.") - .longOpt("serverDir") - .build(), - Option.builder("force") - .argName("FORCE") - .desc("Force option in case Solr is run as root.") - .build(), - Option.builder("exampleDir") - .argName("DIR") - .hasArg() - .required(false) - .desc( - "Path to the Solr example directory; if not provided, ${serverDir}/../example is expected to exist.") - .build(), - Option.builder("urlScheme") - .argName("SCHEME") - .hasArg() - .required(false) - .desc("Solr URL scheme: http or https, defaults to http if not specified.") - .build(), - Option.builder("p") - .argName("PORT") - .hasArg() - .required(false) - .desc("Specify the port to start the Solr HTTP listener on; default is 8983.") - .longOpt("port") - .build(), - Option.builder("h") - .argName("HOSTNAME") - .hasArg() - .required(false) - .desc("Specify the hostname for this Solr instance.") - .longOpt("host") - .build(), - Option.builder("z") - .argName("ZKHOST") - .hasArg() - .required(false) - .desc("ZooKeeper connection string; only used when running in SolrCloud mode using -c.") - .longOpt("zkhost") - .build(), - Option.builder("c") - .required(false) - .desc( - "Start Solr in SolrCloud mode; if -z not supplied, an embedded ZooKeeper instance is started on Solr port+1000, such as 9983 if Solr is bound to 8983.") - .longOpt("cloud") - .build(), - Option.builder("m") - .argName("MEM") - .hasArg() - .required(false) - .desc( - "Sets the min (-Xms) and max (-Xmx) heap size for the JVM, such as: -m 4g results in: -Xms4g -Xmx4g; by default, this script sets the heap size to 512m.") - .longOpt("memory") - .build(), - Option.builder("a") - .argName("OPTS") - .hasArg() - .required(false) - .desc( - "Additional options to be passed to the JVM when starting example Solr server(s).") - .longOpt("addlopts") - .build() - }; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - this.urlScheme = cli.getOptionValue("urlScheme", "http"); - - serverDir = new File(cli.getOptionValue("serverDir")); - if (!serverDir.isDirectory()) - throw new IllegalArgumentException( - "Value of -serverDir option is invalid! " - + serverDir.getAbsolutePath() - + " is not a directory!"); - - script = cli.getOptionValue("script"); - if (script != null) { - if (!(new File(script)).isFile()) - throw new IllegalArgumentException( - "Value of -script option is invalid! " + script + " not found"); - } else { - File scriptFile = new File(serverDir.getParentFile(), "bin/solr"); - if (scriptFile.isFile()) { - script = scriptFile.getAbsolutePath(); - } else { - scriptFile = new File(serverDir.getParentFile(), "bin/solr.cmd"); - if (scriptFile.isFile()) { - script = scriptFile.getAbsolutePath(); - } else { - throw new IllegalArgumentException( - "Cannot locate the bin/solr script! Please pass -script to this application."); - } - } - } - - exampleDir = - (cli.hasOption("exampleDir")) - ? new File(cli.getOptionValue("exampleDir")) - : new File(serverDir.getParent(), "example"); - if (!exampleDir.isDirectory()) - throw new IllegalArgumentException( - "Value of -exampleDir option is invalid! " - + exampleDir.getAbsolutePath() - + " is not a directory!"); - - echoIfVerbose( - "Running with\nserverDir=" - + serverDir.getAbsolutePath() - + ",\nexampleDir=" - + exampleDir.getAbsolutePath() - + "\nscript=" - + script, - cli); - - String exampleType = cli.getOptionValue("example"); - if ("cloud".equals(exampleType)) { - runCloudExample(cli); - } else if ("techproducts".equals(exampleType) - || "schemaless".equals(exampleType) - || "films".equals(exampleType)) { - runExample(cli, exampleType); - } else { - throw new IllegalArgumentException( - "Unsupported example " - + exampleType - + "! Please choose one of: cloud, schemaless, techproducts, or films"); - } - } - - protected void runExample(CommandLine cli, String exampleName) throws Exception { - File exDir = setupExampleDir(serverDir, exampleDir, exampleName); - String collectionName = "schemaless".equals(exampleName) ? "gettingstarted" : exampleName; - String configSet = - "techproducts".equals(exampleName) ? "sample_techproducts_configs" : "_default"; - - boolean isCloudMode = cli.hasOption('c'); - String zkHost = cli.getOptionValue('z'); - int port = Integer.parseInt(cli.getOptionValue('p', "8983")); - Map nodeStatus = - startSolr(new File(exDir, "solr"), isCloudMode, cli, port, zkHost, 30); - - // invoke the CreateTool - File configsetsDir = new File(serverDir, "solr/configsets"); - - String solrUrl = (String) nodeStatus.get("baseUrl"); - - // safe check if core / collection already exists - boolean alreadyExists = false; - if (nodeStatus.get("cloud") != null) { - String collectionListUrl = solrUrl + "/admin/collections?action=list"; - if (safeCheckCollectionExists(collectionListUrl, collectionName)) { - alreadyExists = true; - echo( - "\nWARNING: Collection '" - + collectionName - + "' already exists!\nChecked collection existence using Collections API command:\n" - + collectionListUrl - + "\n"); - } - } else { - String coreName = collectionName; - String coreStatusUrl = solrUrl + "/admin/cores?action=STATUS&core=" + coreName; - if (safeCheckCoreExists(coreStatusUrl, coreName)) { - alreadyExists = true; - echo( - "\nWARNING: Core '" - + coreName - + "' already exists!\nChecked core existence using Core API command:\n" - + coreStatusUrl - + "\n"); - } - } - - if (!alreadyExists) { - String[] createArgs = - new String[] { - "-name", collectionName, - "-shards", "1", - "-replicationFactor", "1", - "-confname", collectionName, - "-confdir", configSet, - "-configsetsDir", configsetsDir.getAbsolutePath(), - "-solrUrl", solrUrl - }; - CreateTool createTool = new CreateTool(stdout); - int createCode = - createTool.runTool( - processCommandLineArgs( - createTool.getName(), - joinCommonAndToolOptions(createTool.getOptions()), - createArgs)); - if (createCode != 0) - throw new Exception( - "Failed to create " - + collectionName - + " using command: " - + Arrays.asList(createArgs)); - } - - if ("techproducts".equals(exampleName) && !alreadyExists) { - - File exampledocsDir = new File(exampleDir, "exampledocs"); - if (!exampledocsDir.isDirectory()) { - File readOnlyExampleDir = new File(serverDir.getParentFile(), "example"); - if (readOnlyExampleDir.isDirectory()) { - exampledocsDir = new File(readOnlyExampleDir, "exampledocs"); - } - } - - if (exampledocsDir.isDirectory()) { - String updateUrl = String.format(Locale.ROOT, "%s/%s/update", solrUrl, collectionName); - echo("Indexing tech product example docs from " + exampledocsDir.getAbsolutePath()); - - String currentPropVal = System.getProperty("url"); - System.setProperty("url", updateUrl); - SimplePostTool.main(new String[] {exampledocsDir.getAbsolutePath() + "/*.xml"}); - if (currentPropVal != null) { - System.setProperty("url", currentPropVal); // reset - } else { - System.clearProperty("url"); - } - } else { - echo( - "exampledocs directory not found, skipping indexing step for the techproducts example"); - } - } else if ("films".equals(exampleName) && !alreadyExists) { - SolrClient solrClient = new HttpSolrClient.Builder(solrUrl).build(); - - echo("Adding dense vector field type to films schema \"_default\""); - try { - SolrCLI.postJsonToSolr( - solrClient, - "/" + collectionName + "/schema", - "{\n" - + " \"add-field-type\" : {\n" - + " \"name\":\"knn_vector_10\",\n" - + " \"class\":\"solr.DenseVectorField\",\n" - + " \"vectorDimension\":10,\n" - + " \"similarityFunction\":cosine\n" - + " \"knnAlgorithm\":hnsw\n" - + " }\n" - + " }"); - } catch (Exception ex) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, ex); - } - - echo( - "Adding name, initial_release_date, and film_vector fields to films schema \"_default\""); - try { - SolrCLI.postJsonToSolr( - solrClient, - "/" + collectionName + "/schema", - "{\n" - + " \"add-field\" : {\n" - + " \"name\":\"name\",\n" - + " \"type\":\"text_general\",\n" - + " \"multiValued\":false,\n" - + " \"stored\":true\n" - + " },\n" - + " \"add-field\" : {\n" - + " \"name\":\"initial_release_date\",\n" - + " \"type\":\"pdate\",\n" - + " \"stored\":true\n" - + " },\n" - + " \"add-field\" : {\n" - + " \"name\":\"film_vector\",\n" - + " \"type\":\"knn_vector_10\",\n" - + " \"indexed\":true\n" - + " \"stored\":true\n" - + " }\n" - + " }"); - } catch (Exception ex) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, ex); - } - - echo( - "Adding paramsets \"algo\" and \"algo_b\" to films configuration for relevancy tuning"); - try { - SolrCLI.postJsonToSolr( - solrClient, - "/" + collectionName + "/config/params", - "{\n" - + " \"set\": {\n" - + " \"algo_a\":{\n" - + " \"defType\":\"dismax\",\n" - + " \"qf\":\"name\"\n" - + " }\n" - + " },\n" - + " \"set\": {\n" - + " \"algo_b\":{\n" - + " \"defType\":\"dismax\",\n" - + " \"qf\":\"name\",\n" - + " \"mm\":\"100%\"\n" - + " }\n" - + " }\n" - + " }\n"); - } catch (Exception ex) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, ex); - } - - File filmsJsonFile = new File(exampleDir, "films/films.json"); - String updateUrl = String.format(Locale.ROOT, "%s/%s/update/json", solrUrl, collectionName); - echo("Indexing films example docs from " + filmsJsonFile.getAbsolutePath()); - String currentPropVal = System.getProperty("url"); - System.setProperty("url", updateUrl); - SimplePostTool.main(new String[] {filmsJsonFile.getAbsolutePath()}); - if (currentPropVal != null) { - System.setProperty("url", currentPropVal); // reset - } else { - System.clearProperty("url"); - } - } - - echo( - "\nSolr " - + exampleName - + " example launched successfully. Direct your Web browser to " - + solrUrl - + " to visit the Solr Admin UI"); - } - - protected void runCloudExample(CommandLine cli) throws Exception { - - boolean prompt = !cli.hasOption("noprompt"); - int numNodes = 2; - int[] cloudPorts = new int[] {8983, 7574, 8984, 7575}; - File cloudDir = new File(exampleDir, "cloud"); - if (!cloudDir.isDirectory()) cloudDir.mkdir(); - - echo("\nWelcome to the SolrCloud example!\n"); - - Scanner readInput = prompt ? new Scanner(userInput, StandardCharsets.UTF_8.name()) : null; - if (prompt) { - echo( - "This interactive session will help you launch a SolrCloud cluster on your local workstation."); - - // get the number of nodes to start - numNodes = - promptForInt( - readInput, - "To begin, how many Solr nodes would you like to run in your local cluster? (specify 1-4 nodes) [2]: ", - "a number", - numNodes, - 1, - 4); - - echo("Ok, let's start up " + numNodes + " Solr nodes for your example SolrCloud cluster."); - - // get the ports for each port - for (int n = 0; n < numNodes; n++) { - String promptMsg = - String.format( - Locale.ROOT, "Please enter the port for node%d [%d]: ", (n + 1), cloudPorts[n]); - int port = promptForPort(readInput, n + 1, promptMsg, cloudPorts[n]); - while (!isPortAvailable(port)) { - port = - promptForPort( - readInput, - n + 1, - "Oops! Looks like port " - + port - + " is already being used by another process. Please choose a different port.", - cloudPorts[n]); - } - - cloudPorts[n] = port; - echoIfVerbose("Using port " + port + " for node " + (n + 1), cli); - } - } else { - echo("Starting up " + numNodes + " Solr nodes for your example SolrCloud cluster.\n"); - } - - // setup a unique solr.solr.home directory for each node - File node1Dir = setupExampleDir(serverDir, cloudDir, "node1"); - for (int n = 2; n <= numNodes; n++) { - File nodeNDir = new File(cloudDir, "node" + n); - if (!nodeNDir.isDirectory()) { - echo("Cloning " + node1Dir.getAbsolutePath() + " into\n " + nodeNDir.getAbsolutePath()); - FileUtils.copyDirectory(node1Dir, nodeNDir); - } else { - echo(nodeNDir.getAbsolutePath() + " already exists."); - } - } - - // deal with extra args passed to the script to run the example - String zkHost = cli.getOptionValue('z'); - - // start the first node (most likely with embedded ZK) - Map nodeStatus = - startSolr(new File(node1Dir, "solr"), true, cli, cloudPorts[0], zkHost, 30); - - if (zkHost == null) { - @SuppressWarnings("unchecked") - Map cloudStatus = (Map) nodeStatus.get("cloud"); - if (cloudStatus != null) { - String zookeeper = (String) cloudStatus.get("ZooKeeper"); - if (zookeeper != null) zkHost = zookeeper; - } - if (zkHost == null) - throw new Exception("Could not get the ZooKeeper connection string for node1!"); - } - - if (numNodes > 1) { - // start the other nodes - for (int n = 1; n < numNodes; n++) - startSolr( - new File(cloudDir, "node" + (n + 1) + "/solr"), true, cli, cloudPorts[n], zkHost, 30); - } - - String solrUrl = (String) nodeStatus.get("baseUrl"); - if (solrUrl.endsWith("/")) solrUrl = solrUrl.substring(0, solrUrl.length() - 1); - - // wait until live nodes == numNodes - waitToSeeLiveNodes(10 /* max wait */, zkHost, numNodes); - - // create the collection - String collectionName = createCloudExampleCollection(numNodes, readInput, prompt, solrUrl); - - // update the config to enable soft auto-commit - echo("\nEnabling auto soft-commits with maxTime 3 secs using the Config API"); - setCollectionConfigProperty( - solrUrl, collectionName, "updateHandler.autoSoftCommit.maxTime", "3000"); - - echo("\n\nSolrCloud example running, please visit: " + solrUrl + " \n"); - } - - protected void setCollectionConfigProperty( - String solrUrl, String collectionName, String propName, String propValue) { - ConfigTool configTool = new ConfigTool(stdout); - String[] configArgs = - new String[] { - "-collection", - collectionName, - "-property", - propName, - "-value", - propValue, - "-solrUrl", - solrUrl - }; - - // let's not fail if we get this far ... just report error and finish up - try { - configTool.runTool( - processCommandLineArgs( - configTool.getName(), - joinCommonAndToolOptions(configTool.getOptions()), - configArgs)); - } catch (Exception exc) { - CLIO.err("Failed to update '" + propName + "' property due to: " + exc); - } - } - - protected void waitToSeeLiveNodes(int maxWaitSecs, String zkHost, int numNodes) { - CloudSolrClient cloudClient = null; - try { - cloudClient = - new CloudSolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) - .build(); - cloudClient.connect(); - Set liveNodes = cloudClient.getClusterState().getLiveNodes(); - int numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0; - long timeout = - System.nanoTime() + TimeUnit.NANOSECONDS.convert(maxWaitSecs, TimeUnit.SECONDS); - while (System.nanoTime() < timeout && numLiveNodes < numNodes) { - echo( - "\nWaiting up to " - + maxWaitSecs - + " seconds to see " - + (numNodes - numLiveNodes) - + " more nodes join the SolrCloud cluster ..."); - try { - Thread.sleep(2000); - } catch (InterruptedException ie) { - Thread.interrupted(); - } - liveNodes = cloudClient.getClusterState().getLiveNodes(); - numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0; - } - if (numLiveNodes < numNodes) { - echo( - "\nWARNING: Only " - + numLiveNodes - + " of " - + numNodes - + " are active in the cluster after " - + maxWaitSecs - + " seconds! Please check the solr.log for each node to look for errors.\n"); - } - } catch (Exception exc) { - CLIO.err("Failed to see if " + numNodes + " joined the SolrCloud cluster due to: " + exc); - } finally { - if (cloudClient != null) { - try { - cloudClient.close(); - } catch (Exception ignore) { - } - } - } - } - - protected Map startSolr( - File solrHomeDir, - boolean cloudMode, - CommandLine cli, - int port, - String zkHost, - int maxWaitSecs) - throws Exception { - - String extraArgs = readExtraArgs(cli.getArgs()); - - String host = cli.getOptionValue('h'); - String memory = cli.getOptionValue('m'); - - String hostArg = (host != null && !"localhost".equals(host)) ? " -h " + host : ""; - String zkHostArg = (zkHost != null) ? " -z " + zkHost : ""; - String memArg = (memory != null) ? " -m " + memory : ""; - String cloudModeArg = cloudMode ? "-cloud " : ""; - String forceArg = cli.hasOption("force") ? " -force" : ""; - - String addlOpts = cli.getOptionValue('a'); - String addlOptsArg = (addlOpts != null) ? " -a \"" + addlOpts + "\"" : ""; - - File cwd = new File(System.getProperty("user.dir")); - File binDir = (new File(script)).getParentFile(); - - boolean isWindows = (OS.isFamilyDOS() || OS.isFamilyWin9x() || OS.isFamilyWindows()); - String callScript = (!isWindows && cwd.equals(binDir.getParentFile())) ? "bin/solr" : script; - - String cwdPath = cwd.getAbsolutePath(); - String solrHome = solrHomeDir.getAbsolutePath(); - - // don't display a huge path for solr home if it is relative to the cwd - if (!isWindows && cwdPath.length() > 1 && solrHome.startsWith(cwdPath)) - solrHome = solrHome.substring(cwdPath.length() + 1); - - String startCmd = - String.format( - Locale.ROOT, - "\"%s\" start %s -p %d -s \"%s\" %s %s %s %s %s %s", - callScript, - cloudModeArg, - port, - solrHome, - hostArg, - zkHostArg, - memArg, - forceArg, - extraArgs, - addlOptsArg); - startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing - - echo("\nStarting up Solr on port " + port + " using command:"); - echo(startCmd + "\n"); - - String solrUrl = - String.format( - Locale.ROOT, "%s://%s:%d/solr", urlScheme, (host != null ? host : "localhost"), port); - - Map nodeStatus = checkPortConflict(solrUrl, solrHomeDir, port, cli); - if (nodeStatus != null) - return nodeStatus; // the server they are trying to start is already running - - int code = 0; - if (isWindows) { - // On Windows, the execution doesn't return, so we have to execute async - // and when calling the script, it seems to be inheriting the environment that launched this - // app so we have to prune out env vars that may cause issues - Map startEnv = new HashMap<>(); - Map procEnv = EnvironmentUtils.getProcEnvironment(); - if (procEnv != null) { - for (Map.Entry entry : procEnv.entrySet()) { - String envVar = entry.getKey(); - String envVarVal = entry.getValue(); - if (envVarVal != null && !"EXAMPLE".equals(envVar) && !envVar.startsWith("SOLR_")) { - startEnv.put(envVar, envVarVal); - } - } - } - DefaultExecuteResultHandler handler = new DefaultExecuteResultHandler(); - executor.execute(org.apache.commons.exec.CommandLine.parse(startCmd), startEnv, handler); - - // wait for execution. - try { - handler.waitFor(3000); - } catch (InterruptedException ie) { - // safe to ignore ... - Thread.interrupted(); - } - if (handler.hasResult() && handler.getExitValue() != 0) { - throw new Exception( - "Failed to start Solr using command: " - + startCmd - + " Exception : " - + handler.getException()); - } - } else { - try { - code = executor.execute(org.apache.commons.exec.CommandLine.parse(startCmd)); - } catch (ExecuteException e) { - throw new Exception( - "Failed to start Solr using command: " + startCmd + " Exception : " + e); - } - } - if (code != 0) throw new Exception("Failed to start Solr using command: " + startCmd); - - return getNodeStatus(solrUrl, maxWaitSecs); - } - - protected Map checkPortConflict( - String solrUrl, File solrHomeDir, int port, CommandLine cli) { - // quickly check if the port is in use - if (isPortAvailable(port)) return null; // not in use ... try to start - - Map nodeStatus = null; - try { - nodeStatus = (new StatusTool()).getStatus(solrUrl); - } catch (Exception ignore) { - /* just trying to determine if this example is already running. */ - } - - if (nodeStatus != null) { - String solr_home = (String) nodeStatus.get("solr_home"); - if (solr_home != null) { - String solrHomePath = solrHomeDir.getAbsolutePath(); - if (!solrHomePath.endsWith("/")) solrHomePath += "/"; - if (!solr_home.endsWith("/")) solr_home += "/"; - - if (solrHomePath.equals(solr_home)) { - CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(nodeStatus); - echo( - "Solr is already setup and running on port " - + port - + " with status:\n" - + arr.toString()); - echo( - "\nIf this is not the example node you are trying to start, please choose a different port."); - nodeStatus.put("baseUrl", solrUrl); - return nodeStatus; - } - } - } - - throw new IllegalStateException( - "Port " + port + " is already being used by another process."); - } - - protected String readExtraArgs(String[] extraArgsArr) { - String extraArgs = ""; - if (extraArgsArr != null && extraArgsArr.length > 0) { - StringBuilder sb = new StringBuilder(); - int app = 0; - for (int e = 0; e < extraArgsArr.length; e++) { - String arg = extraArgsArr[e]; - if ("e".equals(arg) || "example".equals(arg)) { - e++; // skip over the example arg - continue; - } - - if (app > 0) sb.append(" "); - sb.append(arg); - ++app; - } - extraArgs = sb.toString().trim(); - } - return extraArgs; - } - - protected String createCloudExampleCollection( - int numNodes, Scanner readInput, boolean prompt, String solrUrl) throws Exception { - // yay! numNodes SolrCloud nodes running - int numShards = 2; - int replicationFactor = 2; - String cloudConfig = "_default"; - String collectionName = "gettingstarted"; - - File configsetsDir = new File(serverDir, "solr/configsets"); - String collectionListUrl = solrUrl + "/admin/collections?action=list"; - - if (prompt) { - echo( - "\nNow let's create a new collection for indexing documents in your " - + numNodes - + "-node cluster."); - - while (true) { - collectionName = - prompt( - readInput, - "Please provide a name for your new collection: [" + collectionName + "] ", - collectionName); - - // Test for existence and then prompt to either create another or skip the create step - if (safeCheckCollectionExists(collectionListUrl, collectionName)) { - echo("\nCollection '" + collectionName + "' already exists!"); - int oneOrTwo = - promptForInt( - readInput, - "Do you want to re-use the existing collection or create a new one? Enter 1 to reuse, 2 to create new [1]: ", - "a 1 or 2", - 1, - 1, - 2); - if (oneOrTwo == 1) { - return collectionName; - } else { - continue; - } - } else { - break; // user selected a collection that doesn't exist ... proceed on - } - } - - numShards = - promptForInt( - readInput, - "How many shards would you like to split " + collectionName + " into? [2]", - "a shard count", - 2, - 1, - 4); - - replicationFactor = - promptForInt( - readInput, - "How many replicas per shard would you like to create? [2] ", - "a replication factor", - 2, - 1, - 4); - - echo( - "Please choose a configuration for the " - + collectionName - + " collection, available options are:"); - String validConfigs = "_default or sample_techproducts_configs [" + cloudConfig + "] "; - cloudConfig = prompt(readInput, validConfigs, cloudConfig); - - // validate the cloudConfig name - while (!isValidConfig(configsetsDir, cloudConfig)) { - echo( - cloudConfig - + " is not a valid configuration directory! Please choose a configuration for the " - + collectionName - + " collection, available options are:"); - cloudConfig = prompt(readInput, validConfigs, cloudConfig); - } - } else { - // must verify if default collection exists - if (safeCheckCollectionExists(collectionListUrl, collectionName)) { - echo( - "\nCollection '" - + collectionName - + "' already exists! Skipping collection creation step."); - return collectionName; - } - } - - // invoke the CreateCollectionTool - String[] createArgs = - new String[] { - "-name", collectionName, - "-shards", String.valueOf(numShards), - "-replicationFactor", String.valueOf(replicationFactor), - "-confname", collectionName, - "-confdir", cloudConfig, - "-configsetsDir", configsetsDir.getAbsolutePath(), - "-solrUrl", solrUrl - }; - - CreateCollectionTool createCollectionTool = new CreateCollectionTool(stdout); - int createCode = - createCollectionTool.runTool( - processCommandLineArgs( - createCollectionTool.getName(), - joinCommonAndToolOptions(createCollectionTool.getOptions()), - createArgs)); - - if (createCode != 0) - throw new Exception( - "Failed to create collection using command: " + Arrays.asList(createArgs)); - - return collectionName; - } - - protected boolean isValidConfig(File configsetsDir, String config) { - File configDir = new File(configsetsDir, config); - if (configDir.isDirectory()) return true; - - // not a built-in configset ... maybe it's a custom directory? - configDir = new File(config); - if (configDir.isDirectory()) return true; - - return false; - } - - protected Map getNodeStatus(String solrUrl, int maxWaitSecs) throws Exception { - StatusTool statusTool = new StatusTool(); - if (verbose) echo("\nChecking status of Solr at " + solrUrl + " ..."); - - URL solrURL = new URL(solrUrl); - Map nodeStatus = statusTool.waitToSeeSolrUp(solrUrl, maxWaitSecs); - nodeStatus.put("baseUrl", solrUrl); - CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(nodeStatus); - String mode = (nodeStatus.get("cloud") != null) ? "cloud" : "standalone"; - if (verbose) - echo( - "\nSolr is running on " - + solrURL.getPort() - + " in " - + mode - + " mode with status:\n" - + arr.toString()); - - return nodeStatus; - } - - protected File setupExampleDir(File serverDir, File exampleParentDir, String dirName) - throws IOException { - File solrXml = new File(serverDir, "solr/solr.xml"); - if (!solrXml.isFile()) - throw new IllegalArgumentException( - "Value of -serverDir option is invalid! " + solrXml.getAbsolutePath() + " not found!"); - - File zooCfg = new File(serverDir, "solr/zoo.cfg"); - if (!zooCfg.isFile()) - throw new IllegalArgumentException( - "Value of -serverDir option is invalid! " + zooCfg.getAbsolutePath() + " not found!"); - - File solrHomeDir = new File(exampleParentDir, dirName + "/solr"); - if (!solrHomeDir.isDirectory()) { - echo("Creating Solr home directory " + solrHomeDir); - solrHomeDir.mkdirs(); - } else { - echo("Solr home directory " + solrHomeDir.getAbsolutePath() + " already exists."); - } - - copyIfNeeded(solrXml, new File(solrHomeDir, "solr.xml")); - copyIfNeeded(zooCfg, new File(solrHomeDir, "zoo.cfg")); - - return solrHomeDir.getParentFile(); - } - - protected void copyIfNeeded(File src, File dest) throws IOException { - if (!dest.isFile()) FileUtils.copyFile(src, dest); - - if (!dest.isFile()) - throw new IllegalStateException("Required file " + dest.getAbsolutePath() + " not found!"); - } - - protected boolean isPortAvailable(int port) { - Socket s = null; - try { - s = new Socket("localhost", port); - return false; - } catch (IOException e) { - return true; - } finally { - if (s != null) { - try { - s.close(); - } catch (IOException ignore) { - } - } - } - } - - protected Integer promptForPort(Scanner s, int node, String prompt, Integer defVal) { - return promptForInt(s, prompt, "a port for node " + node, defVal, null, null); - } - - protected Integer promptForInt( - Scanner s, String prompt, String label, Integer defVal, Integer min, Integer max) { - Integer inputAsInt = null; - - String value = prompt(s, prompt, null /* default is null since we handle that here */); - if (value != null) { - int attempts = 3; - while (value != null && --attempts > 0) { - try { - inputAsInt = Integer.valueOf(value); - - if (min != null) { - if (inputAsInt < min) { - value = - prompt( - s, - String.format( - Locale.ROOT, - PROMPT_NUMBER_TOO_SMALL, - inputAsInt, - label, - min, - max, - defVal)); - inputAsInt = null; - continue; - } - } - - if (max != null) { - if (inputAsInt > max) { - value = - prompt( - s, - String.format( - Locale.ROOT, - PROMPT_NUMBER_TOO_LARGE, - inputAsInt, - label, - min, - max, - defVal)); - inputAsInt = null; - continue; - } - } - - } catch (NumberFormatException nfe) { - if (verbose) echo(value + " is not a number!"); - - if (min != null && max != null) { - value = - prompt( - s, - String.format( - Locale.ROOT, PROMPT_FOR_NUMBER_IN_RANGE, label, min, max, defVal)); - } else { - value = prompt(s, String.format(Locale.ROOT, PROMPT_FOR_NUMBER, label, defVal)); - } - } - } - if (attempts == 0 && value != null && inputAsInt == null) - echo("Too many failed attempts! Going with default value " + defVal); - } - - return (inputAsInt != null) ? inputAsInt : defVal; - } - - protected String prompt(Scanner s, String prompt) { - return prompt(s, prompt, null); - } - - protected String prompt(Scanner s, String prompt, String defaultValue) { - echo(prompt); - String nextInput = s.nextLine(); - if (nextInput != null) { - nextInput = nextInput.trim(); - if (nextInput.isEmpty()) nextInput = null; - } - return (nextInput != null) ? nextInput : defaultValue; - } - } // end RunExampleTool class - - /** - * Asserts various conditions and exists with error code if fails, else continues with no output - */ - public static class AssertTool extends ToolBase { - - private static String message = null; - private static boolean useExitCode = false; - private static Optional timeoutMs = Optional.empty(); - - public AssertTool() { - this(CLIO.getOutStream()); - } - - public AssertTool(PrintStream stdout) { - super(stdout); - } - - @Override - public String getName() { - return "assert"; - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("R") - .desc("Asserts that we are NOT the root user.") - .longOpt("not-root") - .build(), - Option.builder("r").desc("Asserts that we are the root user.").longOpt("root").build(), - Option.builder("S") - .desc("Asserts that Solr is NOT running on a certain URL. Default timeout is 1000ms.") - .longOpt("not-started") - .hasArg(true) - .argName("url") - .build(), - Option.builder("s") - .desc("Asserts that Solr is running on a certain URL. Default timeout is 1000ms.") - .longOpt("started") - .hasArg(true) - .argName("url") - .build(), - Option.builder("u") - .desc("Asserts that we run as same user that owns .") - .longOpt("same-user") - .hasArg(true) - .argName("directory") - .build(), - Option.builder("x") - .desc("Asserts that directory exists.") - .longOpt("exists") - .hasArg(true) - .argName("directory") - .build(), - Option.builder("X") - .desc("Asserts that directory does NOT exist.") - .longOpt("not-exists") - .hasArg(true) - .argName("directory") - .build(), - Option.builder("c") - .desc( - "Asserts that Solr is running in cloud mode. Also fails if Solr not running. URL should be for root Solr path.") - .longOpt("cloud") - .hasArg(true) - .argName("url") - .build(), - Option.builder("C") - .desc( - "Asserts that Solr is not running in cloud mode. Also fails if Solr not running. URL should be for root Solr path.") - .longOpt("not-cloud") - .hasArg(true) - .argName("url") - .build(), - Option.builder("m") - .desc("Exception message to be used in place of the default error message.") - .longOpt("message") - .hasArg(true) - .argName("message") - .build(), - Option.builder("t") - .desc("Timeout in ms for commands supporting a timeout.") - .longOpt("timeout") - .hasArg(true) - .type(Long.class) - .argName("ms") - .build(), - Option.builder("e") - .desc("Return an exit code instead of printing error message on assert fail.") - .longOpt("exitcode") - .build() - }; - } - - @Override - public int runTool(CommandLine cli) throws Exception { - verbose = cli.hasOption(OPTION_VERBOSE.getOpt()); - - int toolExitStatus = 0; - try { - toolExitStatus = runAssert(cli); - } catch (Exception exc) { - // since this is a CLI, spare the user the stacktrace - String excMsg = exc.getMessage(); - if (excMsg != null) { - if (verbose) { - CLIO.err("\nERROR: " + exc + "\n"); - } else { - CLIO.err("\nERROR: " + excMsg + "\n"); - } - toolExitStatus = 100; // Exit >= 100 means error, else means number of tests that failed - } else { - throw exc; - } - } - return toolExitStatus; - } - - @Override - protected void runImpl(CommandLine cli) throws Exception { - runAssert(cli); - } - - /** - * Custom run method which may return exit code - * - * @param cli the command line object - * @return 0 on success, or a number corresponding to number of tests that failed - * @throws Exception if a tool failed, e.g. authentication failure - */ - protected int runAssert(CommandLine cli) throws Exception { - if (cli.getOptions().length == 0 || cli.getArgs().length > 0 || cli.hasOption("h")) { - new HelpFormatter() - .printHelp( - "bin/solr assert [-m ] [-e] [-rR] [-s ] [-S ] [-c ] [-C ] [-u ] [-x ] [-X ]", - getToolOptions(this)); - return 1; - } - if (cli.hasOption("m")) { - message = cli.getOptionValue("m"); - } - if (cli.hasOption("t")) { - timeoutMs = Optional.of(Long.parseLong(cli.getOptionValue("t"))); - } - if (cli.hasOption("e")) { - useExitCode = true; - } - - int ret = 0; - if (cli.hasOption("r")) { - ret += assertRootUser(); - } - if (cli.hasOption("R")) { - ret += assertNotRootUser(); - } - if (cli.hasOption("x")) { - ret += assertFileExists(cli.getOptionValue("x")); - } - if (cli.hasOption("X")) { - ret += assertFileNotExists(cli.getOptionValue("X")); - } - if (cli.hasOption("u")) { - ret += sameUser(cli.getOptionValue("u")); - } - if (cli.hasOption("s")) { - ret += assertSolrRunning(cli.getOptionValue("s")); - } - if (cli.hasOption("S")) { - ret += assertSolrNotRunning(cli.getOptionValue("S")); - } - if (cli.hasOption("c")) { - ret += assertSolrRunningInCloudMode(cli.getOptionValue("c")); - } - if (cli.hasOption("C")) { - ret += assertSolrNotRunningInCloudMode(cli.getOptionValue("C")); - } - return ret; - } - - public static int assertSolrRunning(String url) throws Exception { - StatusTool status = new StatusTool(); - try { - status.waitToSeeSolrUp(url, timeoutMs.orElse(1000L).intValue() / 1000); - } catch (Exception se) { - if (exceptionIsAuthRelated(se)) { - throw se; - } - return exitOrException( - "Solr is not running on url " + url + " after " + timeoutMs.orElse(1000L) / 1000 + "s"); - } - return 0; - } - - public static int assertSolrNotRunning(String url) throws Exception { - StatusTool status = new StatusTool(); - long timeout = - System.nanoTime() - + TimeUnit.NANOSECONDS.convert(timeoutMs.orElse(1000L), TimeUnit.MILLISECONDS); - try { - attemptHttpHead(url, getHttpClient()); - } catch (SolrException se) { - throw se; // Auth error - } catch (IOException e) { - log.debug("Opening connection to {} failed, Solr does not seem to be running", url, e); - return 0; - } - while (System.nanoTime() < timeout) { - try { - status.waitToSeeSolrUp(url, 1); - try { - log.debug("Solr still up. Waiting before trying again to see if it was stopped"); - Thread.sleep(1000L); - } catch (InterruptedException interrupted) { - timeout = 0; // stop looping - } - } catch (Exception se) { - if (exceptionIsAuthRelated(se)) { - throw se; - } - return exitOrException(se.getMessage()); - } - } - return exitOrException( - "Solr is still running at " + url + " after " + timeoutMs.orElse(1000L) / 1000 + "s"); - } - - public static int assertSolrRunningInCloudMode(String url) throws Exception { - if (!isSolrRunningOn(url)) { - return exitOrException( - "Solr is not running on url " + url + " after " + timeoutMs.orElse(1000L) / 1000 + "s"); - } - - if (!runningSolrIsCloud(url)) { - return exitOrException("Solr is not running in cloud mode on " + url); - } - return 0; - } - - public static int assertSolrNotRunningInCloudMode(String url) throws Exception { - if (!isSolrRunningOn(url)) { - return exitOrException( - "Solr is not running on url " + url + " after " + timeoutMs.orElse(1000L) / 1000 + "s"); - } - - if (runningSolrIsCloud(url)) { - return exitOrException("Solr is not running in standalone mode on " + url); - } - return 0; - } - - public static int sameUser(String directory) throws Exception { - if (Files.exists(Paths.get(directory))) { - String userForDir = userForDir(Paths.get(directory)); - if (!currentUser().equals(userForDir)) { - return exitOrException("Must run as user " + userForDir + ". We are " + currentUser()); - } - } else { - return exitOrException("Directory " + directory + " does not exist."); - } - return 0; - } - - public static int assertFileExists(String directory) throws Exception { - if (!Files.exists(Paths.get(directory))) { - return exitOrException("Directory " + directory + " does not exist."); - } - return 0; - } - - public static int assertFileNotExists(String directory) throws Exception { - if (Files.exists(Paths.get(directory))) { - return exitOrException("Directory " + directory + " should not exist."); - } - return 0; - } - - public static int assertRootUser() throws Exception { - if (!currentUser().equals("root")) { - return exitOrException("Must run as root user"); - } - return 0; - } - - public static int assertNotRootUser() throws Exception { - if (currentUser().equals("root")) { - return exitOrException("Not allowed to run as root user"); - } - return 0; - } - - public static String currentUser() { - return System.getProperty("user.name"); - } - - public static String userForDir(Path pathToDir) { - try { - FileOwnerAttributeView ownerAttributeView = - Files.getFileAttributeView(pathToDir, FileOwnerAttributeView.class); - return ownerAttributeView.getOwner().getName(); - } catch (IOException e) { - return "N/A"; - } - } - - private static int exitOrException(String msg) throws AssertionFailureException { - if (useExitCode) { - return 1; - } else { - throw new AssertionFailureException(message != null ? message : msg); - } - } - - private static boolean isSolrRunningOn(String url) throws Exception { - StatusTool status = new StatusTool(); - try { - status.waitToSeeSolrUp(url, timeoutMs.orElse(1000L).intValue() / 1000); - return true; - } catch (Exception se) { - if (exceptionIsAuthRelated(se)) { - throw se; - } - return false; - } - } - - private static boolean runningSolrIsCloud(String url) throws Exception { - try (final SolrClient client = new HttpSolrClient.Builder(url).build()) { - final SolrRequest request = - new CollectionAdminRequest.ClusterStatus(); - final CollectionAdminResponse response = request.process(client); - return response != null; - } catch (Exception e) { - if (exceptionIsAuthRelated(e)) { - throw e; - } - return false; - } - } - } // end AssertTool class - - public static class AssertionFailureException extends Exception { - public AssertionFailureException(String message) { - super(message); - } - } - - // Authentication tool - public static class AuthTool extends ToolBase { - public AuthTool() { - this(CLIO.getOutStream()); - } - - public AuthTool(PrintStream stdout) { - super(stdout); - } - - @Override - public String getName() { - return "auth"; - } - - List authenticationVariables = - Arrays.asList( - "SOLR_AUTHENTICATION_CLIENT_BUILDER", "SOLR_AUTH_TYPE", "SOLR_AUTHENTICATION_OPTS"); - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("type") - .argName("type") - .hasArg() - .desc( - "The authentication mechanism to enable (basicAuth or kerberos). Defaults to 'basicAuth'.") - .build(), - Option.builder("credentials") - .argName("credentials") - .hasArg() - .desc( - "Credentials in the format username:password. Example: -credentials solr:SolrRocks") - .build(), - Option.builder("prompt") - .argName("prompt") - .hasArg() - .desc( - "Prompts the user to provide the credentials. Use either -credentials or -prompt, not both.") - .build(), - Option.builder("config") - .argName("config") - .hasArgs() - .desc( - "Configuration parameters (Solr startup parameters). Required for Kerberos authentication.") - .build(), - Option.builder("blockUnknown") - .argName("blockUnknown") - .desc( - "Blocks all access for unknown users (requires authentication for all endpoints).") - .hasArg() - .build(), - Option.builder("solrIncludeFile") - .argName("solrIncludeFile") - .hasArg() - .desc( - "The Solr include file which contains overridable environment variables for configuring Solr configurations.") - .build(), - Option.builder("updateIncludeFileOnly") - .argName("updateIncludeFileOnly") - .desc( - "Only update the solr.in.sh or solr.in.cmd file, and skip actual enabling/disabling" - + " authentication (i.e. don't update security.json).") - .hasArg() - .build(), - Option.builder("authConfDir") - .argName("authConfDir") - .hasArg() - .required() - .desc( - "This is where any authentication related configuration files, if any, would be placed.") - .build(), - Option.builder("solrUrl").argName("solrUrl").hasArg().desc("Solr URL.").build(), - Option.builder("zkHost") - .argName("zkHost") - .hasArg() - .desc("ZooKeeper host to connect to.") - .build(), - OPTION_VERBOSE - }; - } - - private void ensureArgumentIsValidBooleanIfPresent(CommandLine cli, String argName) { - if (cli.hasOption(argName)) { - final String value = cli.getOptionValue(argName); - final Boolean parsedBoolean = BooleanUtils.toBooleanObject(value); - if (parsedBoolean == null) { - echo("Argument [" + argName + "] must be either true or false, but was [" + value + "]"); - exit(1); - } - } - } - - @Override - public int runTool(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - if (cli.getOptions().length == 0 - || cli.getArgs().length == 0 - || cli.getArgs().length > 1 - || cli.hasOption("h")) { - new HelpFormatter() - .printHelp("bin/solr auth [OPTIONS]", getToolOptions(this)); - return 1; - } - - ensureArgumentIsValidBooleanIfPresent(cli, "blockUnknown"); - ensureArgumentIsValidBooleanIfPresent(cli, "updateIncludeFileOnly"); - - String type = cli.getOptionValue("type", "basicAuth"); - switch (type) { - case "basicAuth": - return handleBasicAuth(cli); - case "kerberos": - return handleKerberos(cli); - default: - CLIO.out("Only type=basicAuth or kerberos supported at the moment."); - exit(1); - } - return 1; - } - - private int handleKerberos(CommandLine cli) throws Exception { - String cmd = cli.getArgs()[0]; - boolean updateIncludeFileOnly = - Boolean.parseBoolean(cli.getOptionValue("updateIncludeFileOnly", "false")); - String securityJson = - "{" - + "\n \"authentication\":{" - + "\n \"class\":\"solr.KerberosPlugin\"" - + "\n }" - + "\n}"; - - switch (cmd) { - case "enable": - String zkHost = null; - boolean zkInaccessible = false; - - if (!updateIncludeFileOnly) { - try { - zkHost = getZkHost(cli); - } catch (Exception ex) { - CLIO.out( - "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" - + securityJson - + "\n"); - zkInaccessible = true; - } - if (zkHost == null) { - if (zkInaccessible == false) { - CLIO.out( - "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" - + securityJson - + "\n"); - zkInaccessible = true; - } - } - - // check if security is already enabled or not - if (!zkInaccessible) { - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) - .build()) { - if (zkClient.exists("/security.json", true)) { - byte oldSecurityBytes[] = zkClient.getData("/security.json", null, null, true); - if (!"{}".equals(new String(oldSecurityBytes, StandardCharsets.UTF_8).trim())) { - CLIO.out( - "Security is already enabled. You can disable it with 'bin/solr auth disable'. Existing security.json: \n" - + new String(oldSecurityBytes, StandardCharsets.UTF_8)); - exit(1); - } - } - } catch (Exception ex) { - if (zkInaccessible == false) { - CLIO.out( - "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" - + securityJson - + "\n"); - zkInaccessible = true; - } - } - } - } - - if (!updateIncludeFileOnly) { - if (!zkInaccessible) { - echoIfVerbose("Uploading following security.json: " + securityJson, cli); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) - .build()) { - zkClient.setData( - "/security.json", securityJson.getBytes(StandardCharsets.UTF_8), true); - } catch (Exception ex) { - if (zkInaccessible == false) { - CLIO.out( - "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" - + securityJson); - zkInaccessible = true; - } - } - } - } - - String config = StrUtils.join(Arrays.asList(cli.getOptionValues("config")), ' '); - // config is base64 encoded (to get around parsing problems), decode it - config = config.replaceAll(" ", ""); - config = - new String( - Base64.getDecoder().decode(config.getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8); - config = config.replaceAll("\n", "").replaceAll("\r", ""); - - String solrIncludeFilename = cli.getOptionValue("solrIncludeFile"); - File includeFile = new File(solrIncludeFilename); - if (includeFile.exists() == false || includeFile.canWrite() == false) { - CLIO.out( - "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); - printAuthEnablingInstructions(config); - System.exit(0); - } - - // update the solr.in.sh file to contain the necessary authentication lines - updateIncludeFileEnableAuth(includeFile, null, config, cli); - echo( - "Successfully enabled Kerberos authentication; please restart any running Solr nodes."); - return 0; - - case "disable": - if (!updateIncludeFileOnly) { - zkHost = getZkHost(cli); - if (zkHost == null) { - stdout.print("ZK Host not found. Solr should be running in cloud mode."); - exit(1); - } - - echoIfVerbose("Uploading following security.json: {}", cli); - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) - .build()) { - zkClient.setData("/security.json", "{}".getBytes(StandardCharsets.UTF_8), true); - } - } - - solrIncludeFilename = cli.getOptionValue("solrIncludeFile"); - includeFile = new File(solrIncludeFilename); - if (!includeFile.exists() || !includeFile.canWrite()) { - CLIO.out( - "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); - CLIO.out( - "Security has been disabled. Please remove any SOLR_AUTH_TYPE or SOLR_AUTHENTICATION_OPTS configuration from solr.in.sh/solr.in.cmd.\n"); - System.exit(0); - } - - // update the solr.in.sh file to comment out the necessary authentication lines - updateIncludeFileDisableAuth(includeFile, cli); - return 0; - - default: - CLIO.out("Valid auth commands are: enable, disable."); - exit(1); - } - - CLIO.out("Options not understood."); - new HelpFormatter() - .printHelp("bin/solr auth [OPTIONS]", getToolOptions(this)); - return 1; - } - - private int handleBasicAuth(CommandLine cli) throws Exception { - String cmd = cli.getArgs()[0]; - boolean prompt = Boolean.parseBoolean(cli.getOptionValue("prompt", "false")); - boolean updateIncludeFileOnly = - Boolean.parseBoolean(cli.getOptionValue("updateIncludeFileOnly", "false")); - switch (cmd) { - case "enable": - if (!prompt && !cli.hasOption("credentials")) { - CLIO.out("Option -credentials or -prompt is required with enable."); - new HelpFormatter() - .printHelp("bin/solr auth [OPTIONS]", getToolOptions(this)); - exit(1); - } else if (!prompt - && (cli.getOptionValue("credentials") == null - || !cli.getOptionValue("credentials").contains(":"))) { - CLIO.out("Option -credentials is not in correct format."); - new HelpFormatter() - .printHelp("bin/solr auth [OPTIONS]", getToolOptions(this)); - exit(1); - } - - String zkHost = null; - - if (!updateIncludeFileOnly) { - try { - zkHost = getZkHost(cli); - } catch (Exception ex) { - if (cli.hasOption("zkHost")) { - CLIO.out( - "Couldn't get ZooKeeper host. Please make sure that ZooKeeper is running and the correct zkHost has been passed in."); - } else { - CLIO.out( - "Couldn't get ZooKeeper host. Please make sure Solr is running in cloud mode, or a zkHost has been passed in."); - } - exit(1); - } - if (zkHost == null) { - if (cli.hasOption("zkHost")) { - CLIO.out( - "Couldn't get ZooKeeper host. Please make sure that ZooKeeper is running and the correct zkHost has been passed in."); - } else { - CLIO.out( - "Couldn't get ZooKeeper host. Please make sure Solr is running in cloud mode, or a zkHost has been passed in."); - } - exit(1); - } - - // check if security is already enabled or not - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) - .build()) { - if (zkClient.exists("/security.json", true)) { - byte oldSecurityBytes[] = zkClient.getData("/security.json", null, null, true); - if (!"{}".equals(new String(oldSecurityBytes, StandardCharsets.UTF_8).trim())) { - CLIO.out( - "Security is already enabled. You can disable it with 'bin/solr auth disable'. Existing security.json: \n" - + new String(oldSecurityBytes, StandardCharsets.UTF_8)); - exit(1); - } - } - } - } - - String username, password; - if (cli.hasOption("credentials")) { - String credentials = cli.getOptionValue("credentials"); - username = credentials.split(":")[0]; - password = credentials.split(":")[1]; - } else { - Console console = System.console(); - // keep prompting until they've entered a non-empty username & password - do { - username = console.readLine("Enter username: "); - } while (username == null || username.trim().length() == 0); - username = username.trim(); - - do { - password = new String(console.readPassword("Enter password: ")); - } while (password.length() == 0); - } - - boolean blockUnknown = Boolean.valueOf(cli.getOptionValue("blockUnknown", "true")); - - String securityJson = - "{" - + "\n \"authentication\":{" - + "\n \"blockUnknown\": " - + blockUnknown - + "," - + "\n \"class\":\"solr.BasicAuthPlugin\"," - + "\n \"credentials\":{\"" - + username - + "\":\"" - + Sha256AuthenticationProvider.getSaltedHashedValue(password) - + "\"}" - + "\n }," - + "\n \"authorization\":{" - + "\n \"class\":\"solr.RuleBasedAuthorizationPlugin\"," - + "\n \"permissions\":[" - + "\n {\"name\":\"security-edit\", \"role\":\"admin\"}," - + "\n {\"name\":\"security-read\", \"role\":\"admin\"}," - + "\n {\"name\":\"config-edit\", \"role\":\"admin\"}," - + "\n {\"name\":\"config-read\", \"role\":\"admin\"}," - + "\n {\"name\":\"collection-admin-edit\", \"role\":\"admin\"}," - + "\n {\"name\":\"collection-admin-read\", \"role\":\"admin\"}," - + "\n {\"name\":\"core-admin-edit\", \"role\":\"admin\"}," - + "\n {\"name\":\"core-admin-read\", \"role\":\"admin\"}," - + "\n {\"name\":\"all\", \"role\":\"admin\"}" - + "\n ]," - + "\n \"user-role\":{\"" - + username - + "\":\"admin\"}" - + "\n }" - + "\n}"; - - if (!updateIncludeFileOnly) { - echoIfVerbose("Uploading following security.json: " + securityJson, cli); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) - .build()) { - zkClient.setData( - "/security.json", securityJson.getBytes(StandardCharsets.UTF_8), true); - } - } - - String solrIncludeFilename = cli.getOptionValue("solrIncludeFile"); - File includeFile = new File(solrIncludeFilename); - if (includeFile.exists() == false || includeFile.canWrite() == false) { - CLIO.out( - "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); - printAuthEnablingInstructions(username, password); - System.exit(0); - } - String authConfDir = cli.getOptionValue("authConfDir"); - File basicAuthConfFile = new File(authConfDir + File.separator + "basicAuth.conf"); - - if (basicAuthConfFile.getParentFile().canWrite() == false) { - CLIO.out("Cannot write to file: " + basicAuthConfFile.getAbsolutePath()); - printAuthEnablingInstructions(username, password); - System.exit(0); - } - - FileUtils.writeStringToFile( - basicAuthConfFile, - "httpBasicAuthUser=" + username + "\nhttpBasicAuthPassword=" + password, - StandardCharsets.UTF_8); - - // update the solr.in.sh file to contain the necessary authentication lines - updateIncludeFileEnableAuth(includeFile, basicAuthConfFile.getAbsolutePath(), null, cli); - final String successMessage = - String.format( - Locale.ROOT, - "Successfully enabled basic auth with username [%s] and password [%s].", - username, - password); - echo(successMessage); - return 0; - - case "disable": - if (!updateIncludeFileOnly) { - zkHost = getZkHost(cli); - if (zkHost == null) { - stdout.print("ZK Host not found. Solr should be running in cloud mode."); - exit(1); - } - - echoIfVerbose("Uploading following security.json: {}", cli); - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) - .build()) { - zkClient.setData("/security.json", "{}".getBytes(StandardCharsets.UTF_8), true); - } - } - - solrIncludeFilename = cli.getOptionValue("solrIncludeFile"); - includeFile = new File(solrIncludeFilename); - if (!includeFile.exists() || !includeFile.canWrite()) { - CLIO.out( - "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); - CLIO.out( - "Security has been disabled. Please remove any SOLR_AUTH_TYPE or SOLR_AUTHENTICATION_OPTS configuration from solr.in.sh/solr.in.cmd.\n"); - System.exit(0); - } - - // update the solr.in.sh file to comment out the necessary authentication lines - updateIncludeFileDisableAuth(includeFile, cli); - return 0; - - default: - CLIO.out("Valid auth commands are: enable, disable."); - exit(1); - } - - CLIO.out("Options not understood."); - new HelpFormatter() - .printHelp("bin/solr auth [OPTIONS]", getToolOptions(this)); - return 1; - } - - private void printAuthEnablingInstructions(String username, String password) { - if (SystemUtils.IS_OS_WINDOWS) { - CLIO.out( - "\nAdd the following lines to the solr.in.cmd file so that the solr.cmd script can use subsequently.\n"); - CLIO.out( - "set SOLR_AUTH_TYPE=basic\n" - + "set SOLR_AUTHENTICATION_OPTS=\"-Dbasicauth=" - + username - + ":" - + password - + "\"\n"); - } else { - CLIO.out( - "\nAdd the following lines to the solr.in.sh file so that the ./solr script can use subsequently.\n"); - CLIO.out( - "SOLR_AUTH_TYPE=\"basic\"\n" - + "SOLR_AUTHENTICATION_OPTS=\"-Dbasicauth=" - + username - + ":" - + password - + "\"\n"); - } - } - - private void printAuthEnablingInstructions(String kerberosConfig) { - if (SystemUtils.IS_OS_WINDOWS) { - CLIO.out( - "\nAdd the following lines to the solr.in.cmd file so that the solr.cmd script can use subsequently.\n"); - CLIO.out( - "set SOLR_AUTH_TYPE=kerberos\n" - + "set SOLR_AUTHENTICATION_OPTS=\"" - + kerberosConfig - + "\"\n"); - } else { - CLIO.out( - "\nAdd the following lines to the solr.in.sh file so that the ./solr script can use subsequently.\n"); - CLIO.out( - "SOLR_AUTH_TYPE=\"kerberos\"\n" - + "SOLR_AUTHENTICATION_OPTS=\"" - + kerberosConfig - + "\"\n"); - } - } - - /** - * This will update the include file (e.g. solr.in.sh / solr.in.cmd) with the authentication - * parameters. - * - * @param includeFile The include file - * @param basicAuthConfFile If basicAuth, the path of the file containing credentials. If not, - * null. - * @param kerberosConfig If kerberos, the config string containing startup parameters. If not, - * null. - */ - private void updateIncludeFileEnableAuth( - File includeFile, String basicAuthConfFile, String kerberosConfig, CommandLine cli) - throws IOException { - assert !(basicAuthConfFile != null - && kerberosConfig != null); // only one of the two needs to be populated - List includeFileLines = FileUtils.readLines(includeFile, StandardCharsets.UTF_8); - for (int i = 0; i < includeFileLines.size(); i++) { - String line = includeFileLines.get(i); - if (authenticationVariables.contains(line.trim().split("=")[0].trim())) { // Non-Windows - includeFileLines.set(i, "# " + line); - } - if (line.trim().split("=")[0].trim().startsWith("set ") - && authenticationVariables.contains( - line.trim().split("=")[0].trim().substring(4))) { // Windows - includeFileLines.set(i, "REM " + line); - } - } - includeFileLines.add(""); // blank line - - if (basicAuthConfFile != null) { // for basicAuth - if (SystemUtils.IS_OS_WINDOWS) { - includeFileLines.add("REM The following lines added by solr.cmd for enabling BasicAuth"); - includeFileLines.add("set SOLR_AUTH_TYPE=basic"); - includeFileLines.add( - "set SOLR_AUTHENTICATION_OPTS=\"-Dsolr.httpclient.config=" - + basicAuthConfFile - + "\""); - } else { - includeFileLines.add("# The following lines added by ./solr for enabling BasicAuth"); - includeFileLines.add("SOLR_AUTH_TYPE=\"basic\""); - includeFileLines.add( - "SOLR_AUTHENTICATION_OPTS=\"-Dsolr.httpclient.config=" + basicAuthConfFile + "\""); - } - } else { // for kerberos - if (SystemUtils.IS_OS_WINDOWS) { - includeFileLines.add("REM The following lines added by solr.cmd for enabling BasicAuth"); - includeFileLines.add("set SOLR_AUTH_TYPE=kerberos"); - includeFileLines.add( - "set SOLR_AUTHENTICATION_OPTS=\"-Dsolr.httpclient.config=" - + basicAuthConfFile - + "\""); - } else { - includeFileLines.add("# The following lines added by ./solr for enabling BasicAuth"); - includeFileLines.add("SOLR_AUTH_TYPE=\"kerberos\""); - includeFileLines.add("SOLR_AUTHENTICATION_OPTS=\"" + kerberosConfig + "\""); - } - } - FileUtils.writeLines(includeFile, StandardCharsets.UTF_8.name(), includeFileLines); - - if (basicAuthConfFile != null) { - echoIfVerbose("Written out credentials file: " + basicAuthConfFile, cli); - } - echoIfVerbose("Updated Solr include file: " + includeFile.getAbsolutePath(), cli); - } - - private void updateIncludeFileDisableAuth(File includeFile, CommandLine cli) - throws IOException { - List includeFileLines = FileUtils.readLines(includeFile, StandardCharsets.UTF_8); - boolean hasChanged = false; - for (int i = 0; i < includeFileLines.size(); i++) { - String line = includeFileLines.get(i); - if (authenticationVariables.contains(line.trim().split("=")[0].trim())) { // Non-Windows - includeFileLines.set(i, "# " + line); - hasChanged = true; - } - if (line.trim().split("=")[0].trim().startsWith("set ") - && authenticationVariables.contains( - line.trim().split("=")[0].trim().substring(4))) { // Windows - includeFileLines.set(i, "REM " + line); - hasChanged = true; - } - } - if (hasChanged) { - FileUtils.writeLines(includeFile, StandardCharsets.UTF_8.name(), includeFileLines); - echoIfVerbose("Commented out necessary lines from " + includeFile.getAbsolutePath(), cli); - } - } - - @Override - protected void runImpl(CommandLine cli) throws Exception {} } } diff --git a/solr/core/src/java/org/apache/solr/util/cli/ApiTool.java b/solr/core/src/java/org/apache/solr/util/cli/ApiTool.java new file mode 100644 index 000000000000..391160aaf803 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/ApiTool.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.util.Map; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.noggit.CharArr; +import org.noggit.JSONWriter; + +/** Used to send an arbitrary HTTP request to a Solr API endpoint. */ +public class ApiTool extends ToolBase { + + public ApiTool() { + this(CLIO.getOutStream()); + } + + public ApiTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "api"; + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("get") + .argName("URL") + .hasArg() + .required(true) + .desc("Send a GET request to a Solr API endpoint.") + .build() + }; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + String getUrl = cli.getOptionValue("get"); + if (getUrl != null) { + Map json = SolrCLI.getJson(getUrl); + + // pretty-print the response to stdout + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(json); + echo(arr.toString()); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/AssertTool.java b/solr/core/src/java/org/apache/solr/util/cli/AssertTool.java new file mode 100644 index 000000000000..e8d45cfd58bd --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/AssertTool.java @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.IOException; +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileOwnerAttributeView; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Asserts various conditions and exists with error code if fails, else continues with no output */ +public class AssertTool extends ToolBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static String message = null; + private static boolean useExitCode = false; + private static Optional timeoutMs = Optional.empty(); + + public AssertTool() { + this(CLIO.getOutStream()); + } + + public AssertTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "assert"; + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("R") + .desc("Asserts that we are NOT the root user.") + .longOpt("not-root") + .build(), + Option.builder("r").desc("Asserts that we are the root user.").longOpt("root").build(), + Option.builder("S") + .desc("Asserts that Solr is NOT running on a certain URL. Default timeout is 1000ms.") + .longOpt("not-started") + .hasArg(true) + .argName("url") + .build(), + Option.builder("s") + .desc("Asserts that Solr is running on a certain URL. Default timeout is 1000ms.") + .longOpt("started") + .hasArg(true) + .argName("url") + .build(), + Option.builder("u") + .desc("Asserts that we run as same user that owns .") + .longOpt("same-user") + .hasArg(true) + .argName("directory") + .build(), + Option.builder("x") + .desc("Asserts that directory exists.") + .longOpt("exists") + .hasArg(true) + .argName("directory") + .build(), + Option.builder("X") + .desc("Asserts that directory does NOT exist.") + .longOpt("not-exists") + .hasArg(true) + .argName("directory") + .build(), + Option.builder("c") + .desc( + "Asserts that Solr is running in cloud mode. Also fails if Solr not running. URL should be for root Solr path.") + .longOpt("cloud") + .hasArg(true) + .argName("url") + .build(), + Option.builder("C") + .desc( + "Asserts that Solr is not running in cloud mode. Also fails if Solr not running. URL should be for root Solr path.") + .longOpt("not-cloud") + .hasArg(true) + .argName("url") + .build(), + Option.builder("m") + .desc("Exception message to be used in place of the default error message.") + .longOpt("message") + .hasArg(true) + .argName("message") + .build(), + Option.builder("t") + .desc("Timeout in ms for commands supporting a timeout.") + .longOpt("timeout") + .hasArg(true) + .type(Long.class) + .argName("ms") + .build(), + Option.builder("e") + .desc("Return an exit code instead of printing error message on assert fail.") + .longOpt("exitcode") + .build() + }; + } + + @Override + public int runTool(CommandLine cli) throws Exception { + verbose = cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt()); + + int toolExitStatus; + try { + toolExitStatus = runAssert(cli); + } catch (Exception exc) { + // since this is a CLI, spare the user the stacktrace + String excMsg = exc.getMessage(); + if (excMsg != null) { + if (verbose) { + CLIO.err("\nERROR: " + exc + "\n"); + } else { + CLIO.err("\nERROR: " + excMsg + "\n"); + } + toolExitStatus = 100; // Exit >= 100 means error, else means number of tests that failed + } else { + throw exc; + } + } + return toolExitStatus; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + runAssert(cli); + } + + /** + * Custom run method which may return exit code + * + * @param cli the command line object + * @return 0 on success, or a number corresponding to number of tests that failed + * @throws Exception if a tool failed, e.g. authentication failure + */ + protected int runAssert(CommandLine cli) throws Exception { + if (cli.getOptions().length == 0 || cli.getArgs().length > 0 || cli.hasOption("h")) { + new HelpFormatter() + .printHelp( + "bin/solr assert [-m ] [-e] [-rR] [-s ] [-S ] [-c ] [-C ] [-u ] [-x ] [-X ]", + SolrCLI.getToolOptions(this)); + return 1; + } + if (cli.hasOption("m")) { + message = cli.getOptionValue("m"); + } + if (cli.hasOption("t")) { + timeoutMs = Optional.of(Long.parseLong(cli.getOptionValue("t"))); + } + if (cli.hasOption("e")) { + useExitCode = true; + } + + int ret = 0; + if (cli.hasOption("r")) { + ret += assertRootUser(); + } + if (cli.hasOption("R")) { + ret += assertNotRootUser(); + } + if (cli.hasOption("x")) { + ret += assertFileExists(cli.getOptionValue("x")); + } + if (cli.hasOption("X")) { + ret += assertFileNotExists(cli.getOptionValue("X")); + } + if (cli.hasOption("u")) { + ret += sameUser(cli.getOptionValue("u")); + } + if (cli.hasOption("s")) { + ret += assertSolrRunning(cli.getOptionValue("s")); + } + if (cli.hasOption("S")) { + ret += assertSolrNotRunning(cli.getOptionValue("S")); + } + if (cli.hasOption("c")) { + ret += assertSolrRunningInCloudMode(cli.getOptionValue("c")); + } + if (cli.hasOption("C")) { + ret += assertSolrNotRunningInCloudMode(cli.getOptionValue("C")); + } + return ret; + } + + public static int assertSolrRunning(String url) throws Exception { + StatusTool status = new StatusTool(); + try { + status.waitToSeeSolrUp(url, timeoutMs.orElse(1000L), TimeUnit.MILLISECONDS); + } catch (Exception se) { + if (SolrCLI.exceptionIsAuthRelated(se)) { + throw se; + } + return exitOrException( + "Solr is not running on url " + url + " after " + timeoutMs.orElse(1000L) / 1000 + "s"); + } + return 0; + } + + public static int assertSolrNotRunning(String url) throws Exception { + StatusTool status = new StatusTool(); + long timeout = + System.nanoTime() + + TimeUnit.NANOSECONDS.convert(timeoutMs.orElse(1000L), TimeUnit.MILLISECONDS); + try (CloseableHttpClient httpClient = SolrCLI.getHttpClient()) { + SolrCLI.attemptHttpHead(url, httpClient); + } catch (SolrException se) { + throw se; // Auth error + } catch (IOException e) { + log.debug("Opening connection to {} failed, Solr does not seem to be running", url, e); + return 0; + } + while (System.nanoTime() < timeout) { + try { + status.waitToSeeSolrUp(url, 1, TimeUnit.SECONDS); + try { + log.debug("Solr still up. Waiting before trying again to see if it was stopped"); + Thread.sleep(1000L); + } catch (InterruptedException interrupted) { + timeout = 0; // stop looping + } + } catch (Exception se) { + if (SolrCLI.exceptionIsAuthRelated(se)) { + throw se; + } + return exitOrException(se.getMessage()); + } + } + return exitOrException( + "Solr is still running at " + url + " after " + timeoutMs.orElse(1000L) / 1000 + "s"); + } + + public static int assertSolrRunningInCloudMode(String url) throws Exception { + if (!isSolrRunningOn(url)) { + return exitOrException( + "Solr is not running on url " + url + " after " + timeoutMs.orElse(1000L) / 1000 + "s"); + } + + if (!runningSolrIsCloud(url)) { + return exitOrException("Solr is not running in cloud mode on " + url); + } + return 0; + } + + public static int assertSolrNotRunningInCloudMode(String url) throws Exception { + if (!isSolrRunningOn(url)) { + return exitOrException( + "Solr is not running on url " + url + " after " + timeoutMs.orElse(1000L) / 1000 + "s"); + } + + if (runningSolrIsCloud(url)) { + return exitOrException("Solr is not running in standalone mode on " + url); + } + return 0; + } + + public static int sameUser(String directory) throws Exception { + if (Files.exists(Paths.get(directory))) { + String userForDir = userForDir(Paths.get(directory)); + if (!currentUser().equals(userForDir)) { + return exitOrException("Must run as user " + userForDir + ". We are " + currentUser()); + } + } else { + return exitOrException("Directory " + directory + " does not exist."); + } + return 0; + } + + public static int assertFileExists(String directory) throws Exception { + if (!Files.exists(Paths.get(directory))) { + return exitOrException("Directory " + directory + " does not exist."); + } + return 0; + } + + public static int assertFileNotExists(String directory) throws Exception { + if (Files.exists(Paths.get(directory))) { + return exitOrException("Directory " + directory + " should not exist."); + } + return 0; + } + + public static int assertRootUser() throws Exception { + if (!currentUser().equals("root")) { + return exitOrException("Must run as root user"); + } + return 0; + } + + public static int assertNotRootUser() throws Exception { + if (currentUser().equals("root")) { + return exitOrException("Not allowed to run as root user"); + } + return 0; + } + + public static String currentUser() { + return System.getProperty("user.name"); + } + + public static String userForDir(Path pathToDir) { + try { + FileOwnerAttributeView ownerAttributeView = + Files.getFileAttributeView(pathToDir, FileOwnerAttributeView.class); + return ownerAttributeView.getOwner().getName(); + } catch (IOException e) { + return "N/A"; + } + } + + private static int exitOrException(String msg) throws SolrCLI.AssertionFailureException { + if (useExitCode) { + return 1; + } else { + throw new SolrCLI.AssertionFailureException(message != null ? message : msg); + } + } + + private static boolean isSolrRunningOn(String url) throws Exception { + StatusTool status = new StatusTool(); + try { + status.waitToSeeSolrUp(url, timeoutMs.orElse(1000L), TimeUnit.MILLISECONDS); + return true; + } catch (Exception se) { + if (SolrCLI.exceptionIsAuthRelated(se)) { + throw se; + } + return false; + } + } + + private static boolean runningSolrIsCloud(String url) throws Exception { + try (final SolrClient client = new HttpSolrClient.Builder(url).build()) { + final SolrRequest request = + new CollectionAdminRequest.ClusterStatus(); + final CollectionAdminResponse response = request.process(client); + return response != null; + } catch (Exception e) { + if (SolrCLI.exceptionIsAuthRelated(e)) { + throw e; + } + return false; + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/AuthTool.java b/solr/core/src/java/org/apache/solr/util/cli/AuthTool.java new file mode 100644 index 000000000000..43614779cb73 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/AuthTool.java @@ -0,0 +1,630 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.Console; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.SystemUtils; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.security.Sha256AuthenticationProvider; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.apache.zookeeper.KeeperException; + +// Authentication tool +public class AuthTool extends ToolBase { + public AuthTool() { + this(CLIO.getOutStream()); + } + + public AuthTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "auth"; + } + + List authenticationVariables = + Arrays.asList( + "SOLR_AUTHENTICATION_CLIENT_BUILDER", "SOLR_AUTH_TYPE", "SOLR_AUTHENTICATION_OPTS"); + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("type") + .argName("type") + .hasArg() + .desc( + "The authentication mechanism to enable (basicAuth or kerberos). Defaults to 'basicAuth'.") + .build(), + Option.builder("credentials") + .argName("credentials") + .hasArg() + .desc("Credentials in the format username:password. Example: -credentials solr:SolrRocks") + .build(), + Option.builder("prompt") + .argName("prompt") + .hasArg() + .desc( + "Prompts the user to provide the credentials. Use either -credentials or -prompt, not both.") + .build(), + Option.builder("config") + .argName("config") + .hasArgs() + .desc( + "Configuration parameters (Solr startup parameters). Required for Kerberos authentication.") + .build(), + Option.builder("blockUnknown") + .argName("blockUnknown") + .desc("Blocks all access for unknown users (requires authentication for all endpoints).") + .hasArg() + .build(), + Option.builder("solrIncludeFile") + .argName("solrIncludeFile") + .hasArg() + .desc( + "The Solr include file which contains overridable environment variables for configuring Solr configurations.") + .build(), + Option.builder("updateIncludeFileOnly") + .argName("updateIncludeFileOnly") + .desc( + "Only update the solr.in.sh or solr.in.cmd file, and skip actual enabling/disabling" + + " authentication (i.e. don't update security.json).") + .hasArg() + .build(), + Option.builder("authConfDir") + .argName("authConfDir") + .hasArg() + .required() + .desc( + "This is where any authentication related configuration files, if any, would be placed.") + .build(), + Option.builder("solrUrl").argName("solrUrl").hasArg().desc("Solr URL.").build(), + Option.builder("zkHost") + .argName("zkHost") + .hasArg() + .desc("ZooKeeper host to connect to.") + .build(), + SolrCLI.OPTION_VERBOSE + }; + } + + private void ensureArgumentIsValidBooleanIfPresent(CommandLine cli, String argName) { + if (cli.hasOption(argName)) { + final String value = cli.getOptionValue(argName); + final Boolean parsedBoolean = BooleanUtils.toBooleanObject(value); + if (parsedBoolean == null) { + echo("Argument [" + argName + "] must be either true or false, but was [" + value + "]"); + SolrCLI.exit(1); + } + } + } + + @Override + public int runTool(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + if (cli.getOptions().length == 0 + || cli.getArgs().length == 0 + || cli.getArgs().length > 1 + || cli.hasOption("h")) { + new HelpFormatter() + .printHelp("bin/solr auth [OPTIONS]", SolrCLI.getToolOptions(this)); + return 1; + } + + ensureArgumentIsValidBooleanIfPresent(cli, "blockUnknown"); + ensureArgumentIsValidBooleanIfPresent(cli, "updateIncludeFileOnly"); + + String type = cli.getOptionValue("type", "basicAuth"); + switch (type) { + case "basicAuth": + return handleBasicAuth(cli); + case "kerberos": + return handleKerberos(cli); + default: + CLIO.out("Only type=basicAuth or kerberos supported at the moment."); + SolrCLI.exit(1); + } + return 1; + } + + private int handleKerberos(CommandLine cli) throws Exception { + String cmd = cli.getArgs()[0]; + boolean updateIncludeFileOnly = + Boolean.parseBoolean(cli.getOptionValue("updateIncludeFileOnly", "false")); + String securityJson = + "{" + + "\n \"authentication\":{" + + "\n \"class\":\"solr.KerberosPlugin\"" + + "\n }" + + "\n}"; + + switch (cmd) { + case "enable": + String zkHost = null; + boolean zkInaccessible = false; + + if (!updateIncludeFileOnly) { + try { + zkHost = SolrCLI.getZkHost(cli); + } catch (Exception ex) { + CLIO.out( + "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" + + securityJson + + "\n"); + zkInaccessible = true; + } + if (zkHost == null) { + if (!zkInaccessible) { + CLIO.out( + "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" + + securityJson + + "\n"); + zkInaccessible = true; + } + } + + // check if security is already enabled or not + if (!zkInaccessible) { + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(10000, TimeUnit.MILLISECONDS) + .build()) { + checkSecurityConfigExistence(zkClient); + } catch (Exception ex) { + CLIO.out( + "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" + + securityJson + + "\n"); + zkInaccessible = true; + } + } + } + + if (!updateIncludeFileOnly) { + if (!zkInaccessible) { + echoIfVerbose("Uploading following security.json: " + securityJson, cli); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(10000, TimeUnit.MILLISECONDS) + .build()) { + zkClient.setData( + "/security.json", securityJson.getBytes(StandardCharsets.UTF_8), true); + } catch (Exception ex) { + CLIO.out( + "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" + + securityJson); + } + } + } + + String config = StrUtils.join(Arrays.asList(cli.getOptionValues("config")), ' '); + // config is base64 encoded (to get around parsing problems), decode it + config = config.replaceAll(" ", ""); + config = + new String( + Base64.getDecoder().decode(config.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + config = config.replaceAll("\n", "").replaceAll("\r", ""); + + String solrIncludeFilename = cli.getOptionValue("solrIncludeFile"); + File includeFile = new File(solrIncludeFilename); + if (!includeFile.exists() || !includeFile.canWrite()) { + CLIO.out( + "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); + printAuthEnablingInstructions(config); + System.exit(0); + } + + // update the solr.in.sh file to contain the necessary authentication lines + updateIncludeFileEnableAuth(includeFile, null, config, cli); + echo( + "Successfully enabled Kerberos authentication; please restart any running Solr nodes."); + return 0; + + case "disable": + uploadSecurityConfiguration(cli, updateIncludeFileOnly); + + solrIncludeFilename = cli.getOptionValue("solrIncludeFile"); + includeFile = new File(solrIncludeFilename); + if (!includeFile.exists() || !includeFile.canWrite()) { + CLIO.out( + "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); + CLIO.out( + "Security has been disabled. Please remove any SOLR_AUTH_TYPE or SOLR_AUTHENTICATION_OPTS configuration from solr.in.sh/solr.in.cmd.\n"); + System.exit(0); + } + + // update the solr.in.sh file to comment out the necessary authentication lines + updateIncludeFileDisableAuth(includeFile, cli); + return 0; + + default: + CLIO.out("Valid auth commands are: enable, disable."); + SolrCLI.exit(1); + } + + CLIO.out("Options not understood."); + new HelpFormatter() + .printHelp("bin/solr auth [OPTIONS]", SolrCLI.getToolOptions(this)); + return 1; + } + + private void checkSecurityConfigExistence(SolrZkClient zkClient) + throws KeeperException, InterruptedException { + if (zkClient.exists("/security.json", true)) { + byte[] oldSecurityBytes = zkClient.getData("/security.json", null, null, true); + if (!"{}".equals(new String(oldSecurityBytes, StandardCharsets.UTF_8).trim())) { + CLIO.out( + "Security is already enabled. You can disable it with 'bin/solr auth disable'. Existing security.json: \n" + + new String(oldSecurityBytes, StandardCharsets.UTF_8)); + SolrCLI.exit(1); + } + } + } + + private void uploadSecurityConfiguration(CommandLine cli, boolean updateIncludeFileOnly) + throws Exception { + String zkHost; + if (!updateIncludeFileOnly) { + zkHost = SolrCLI.getZkHost(cli); + if (zkHost == null) { + stdout.print("ZK Host not found. Solr should be running in cloud mode."); + SolrCLI.exit(1); + } + + echoIfVerbose("Uploading following security.json: {}", cli); + + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(10000, TimeUnit.MILLISECONDS) + .build()) { + zkClient.setData("/security.json", "{}".getBytes(StandardCharsets.UTF_8), true); + } + } + } + + private int handleBasicAuth(CommandLine cli) throws Exception { + String cmd = cli.getArgs()[0]; + boolean prompt = Boolean.parseBoolean(cli.getOptionValue("prompt", "false")); + boolean updateIncludeFileOnly = + Boolean.parseBoolean(cli.getOptionValue("updateIncludeFileOnly", "false")); + switch (cmd) { + case "enable": + if (!prompt && !cli.hasOption("credentials")) { + CLIO.out("Option -credentials or -prompt is required with enable."); + new HelpFormatter() + .printHelp("bin/solr auth [OPTIONS]", SolrCLI.getToolOptions(this)); + SolrCLI.exit(1); + } else if (!prompt + && (cli.getOptionValue("credentials") == null + || !cli.getOptionValue("credentials").contains(":"))) { + CLIO.out("Option -credentials is not in correct format."); + new HelpFormatter() + .printHelp("bin/solr auth [OPTIONS]", SolrCLI.getToolOptions(this)); + SolrCLI.exit(1); + } + + String zkHost = null; + + if (!updateIncludeFileOnly) { + try { + zkHost = SolrCLI.getZkHost(cli); + } catch (Exception ex) { + if (cli.hasOption("zkHost")) { + CLIO.out( + "Couldn't get ZooKeeper host. Please make sure that ZooKeeper is running and the correct zkHost has been passed in."); + } else { + CLIO.out( + "Couldn't get ZooKeeper host. Please make sure Solr is running in cloud mode, or a zkHost has been passed in."); + } + SolrCLI.exit(1); + } + if (zkHost == null) { + if (cli.hasOption("zkHost")) { + CLIO.out( + "Couldn't get ZooKeeper host. Please make sure that ZooKeeper is running and the correct zkHost has been passed in."); + } else { + CLIO.out( + "Couldn't get ZooKeeper host. Please make sure Solr is running in cloud mode, or a zkHost has been passed in."); + } + SolrCLI.exit(1); + } + + // check if security is already enabled or not + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(10000, TimeUnit.MILLISECONDS) + .build()) { + checkSecurityConfigExistence(zkClient); + } + } + + String username, password; + if (cli.hasOption("credentials")) { + String credentials = cli.getOptionValue("credentials"); + username = credentials.split(":")[0]; + password = credentials.split(":")[1]; + } else { + Console console = System.console(); + // keep prompting until they've entered a non-empty username & password + do { + username = console.readLine("Enter username: "); + } while (username == null || username.trim().length() == 0); + username = username.trim(); + + do { + password = new String(console.readPassword("Enter password: ")); + } while (password.length() == 0); + } + + boolean blockUnknown = Boolean.parseBoolean(cli.getOptionValue("blockUnknown", "true")); + + String securityJson = + "{" + + "\n \"authentication\":{" + + "\n \"blockUnknown\": " + + blockUnknown + + "," + + "\n \"class\":\"solr.BasicAuthPlugin\"," + + "\n \"credentials\":{\"" + + username + + "\":\"" + + Sha256AuthenticationProvider.getSaltedHashedValue(password) + + "\"}" + + "\n }," + + "\n \"authorization\":{" + + "\n \"class\":\"solr.RuleBasedAuthorizationPlugin\"," + + "\n \"permissions\":[" + + "\n {\"name\":\"security-edit\", \"role\":\"admin\"}," + + "\n {\"name\":\"security-read\", \"role\":\"admin\"}," + + "\n {\"name\":\"config-edit\", \"role\":\"admin\"}," + + "\n {\"name\":\"config-read\", \"role\":\"admin\"}," + + "\n {\"name\":\"collection-admin-edit\", \"role\":\"admin\"}," + + "\n {\"name\":\"collection-admin-read\", \"role\":\"admin\"}," + + "\n {\"name\":\"core-admin-edit\", \"role\":\"admin\"}," + + "\n {\"name\":\"core-admin-read\", \"role\":\"admin\"}," + + "\n {\"name\":\"all\", \"role\":\"admin\"}" + + "\n ]," + + "\n \"user-role\":{\"" + + username + + "\":\"admin\"}" + + "\n }" + + "\n}"; + + if (!updateIncludeFileOnly) { + echoIfVerbose("Uploading following security.json: " + securityJson, cli); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(10000, TimeUnit.MILLISECONDS) + .build()) { + zkClient.setData("/security.json", securityJson.getBytes(StandardCharsets.UTF_8), true); + } + } + + String solrIncludeFilename = cli.getOptionValue("solrIncludeFile"); + File includeFile = new File(solrIncludeFilename); + if (!includeFile.exists() || !includeFile.canWrite()) { + CLIO.out( + "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); + printAuthEnablingInstructions(username, password); + System.exit(0); + } + String authConfDir = cli.getOptionValue("authConfDir"); + File basicAuthConfFile = new File(authConfDir + File.separator + "basicAuth.conf"); + + if (!basicAuthConfFile.getParentFile().canWrite()) { + CLIO.out("Cannot write to file: " + basicAuthConfFile.getAbsolutePath()); + printAuthEnablingInstructions(username, password); + System.exit(0); + } + + FileUtils.writeStringToFile( + basicAuthConfFile, + "httpBasicAuthUser=" + username + "\nhttpBasicAuthPassword=" + password, + StandardCharsets.UTF_8); + + // update the solr.in.sh file to contain the necessary authentication lines + updateIncludeFileEnableAuth(includeFile, basicAuthConfFile.getAbsolutePath(), null, cli); + final String successMessage = + String.format( + Locale.ROOT, + "Successfully enabled basic auth with username [%s] and password [%s].", + username, + password); + echo(successMessage); + return 0; + + case "disable": + uploadSecurityConfiguration(cli, updateIncludeFileOnly); + + solrIncludeFilename = cli.getOptionValue("solrIncludeFile"); + includeFile = new File(solrIncludeFilename); + if (!includeFile.exists() || !includeFile.canWrite()) { + CLIO.out( + "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); + CLIO.out( + "Security has been disabled. Please remove any SOLR_AUTH_TYPE or SOLR_AUTHENTICATION_OPTS configuration from solr.in.sh/solr.in.cmd.\n"); + System.exit(0); + } + + // update the solr.in.sh file to comment out the necessary authentication lines + updateIncludeFileDisableAuth(includeFile, cli); + return 0; + + default: + CLIO.out("Valid auth commands are: enable, disable."); + SolrCLI.exit(1); + } + + CLIO.out("Options not understood."); + new HelpFormatter() + .printHelp("bin/solr auth [OPTIONS]", SolrCLI.getToolOptions(this)); + return 1; + } + + private void printAuthEnablingInstructions(String username, String password) { + if (SystemUtils.IS_OS_WINDOWS) { + CLIO.out( + "\nAdd the following lines to the solr.in.cmd file so that the solr.cmd script can use subsequently.\n"); + CLIO.out( + "set SOLR_AUTH_TYPE=basic\n" + + "set SOLR_AUTHENTICATION_OPTS=\"-Dbasicauth=" + + username + + ":" + + password + + "\"\n"); + } else { + CLIO.out( + "\nAdd the following lines to the solr.in.sh file so that the ./solr script can use subsequently.\n"); + CLIO.out( + "SOLR_AUTH_TYPE=\"basic\"\n" + + "SOLR_AUTHENTICATION_OPTS=\"-Dbasicauth=" + + username + + ":" + + password + + "\"\n"); + } + } + + private void printAuthEnablingInstructions(String kerberosConfig) { + if (SystemUtils.IS_OS_WINDOWS) { + CLIO.out( + "\nAdd the following lines to the solr.in.cmd file so that the solr.cmd script can use subsequently.\n"); + CLIO.out( + "set SOLR_AUTH_TYPE=kerberos\n" + + "set SOLR_AUTHENTICATION_OPTS=\"" + + kerberosConfig + + "\"\n"); + } else { + CLIO.out( + "\nAdd the following lines to the solr.in.sh file so that the ./solr script can use subsequently.\n"); + CLIO.out( + "SOLR_AUTH_TYPE=\"kerberos\"\n" + + "SOLR_AUTHENTICATION_OPTS=\"" + + kerberosConfig + + "\"\n"); + } + } + + /** + * This will update the include file (e.g. solr.in.sh / solr.in.cmd) with the authentication + * parameters. + * + * @param includeFile The include file + * @param basicAuthConfFile If basicAuth, the path of the file containing credentials. If not, + * null. + * @param kerberosConfig If kerberos, the config string containing startup parameters. If not, + * null. + */ + private void updateIncludeFileEnableAuth( + File includeFile, String basicAuthConfFile, String kerberosConfig, CommandLine cli) + throws IOException { + assert !(basicAuthConfFile != null + && kerberosConfig != null); // only one of the two needs to be populated + List includeFileLines = FileUtils.readLines(includeFile, StandardCharsets.UTF_8); + for (int i = 0; i < includeFileLines.size(); i++) { + String line = includeFileLines.get(i); + if (authenticationVariables.contains(line.trim().split("=")[0].trim())) { // Non-Windows + includeFileLines.set(i, "# " + line); + } + if (line.trim().split("=")[0].trim().startsWith("set ") + && authenticationVariables.contains( + line.trim().split("=")[0].trim().substring(4))) { // Windows + includeFileLines.set(i, "REM " + line); + } + } + includeFileLines.add(""); // blank line + + if (basicAuthConfFile != null) { // for basicAuth + if (SystemUtils.IS_OS_WINDOWS) { + includeFileLines.add("REM The following lines added by solr.cmd for enabling BasicAuth"); + includeFileLines.add("set SOLR_AUTH_TYPE=basic"); + includeFileLines.add( + "set SOLR_AUTHENTICATION_OPTS=\"-Dsolr.httpclient.config=" + basicAuthConfFile + "\""); + } else { + includeFileLines.add("# The following lines added by ./solr for enabling BasicAuth"); + includeFileLines.add("SOLR_AUTH_TYPE=\"basic\""); + includeFileLines.add( + "SOLR_AUTHENTICATION_OPTS=\"-Dsolr.httpclient.config=" + basicAuthConfFile + "\""); + } + } else { // for kerberos + if (SystemUtils.IS_OS_WINDOWS) { + includeFileLines.add("REM The following lines added by solr.cmd for enabling BasicAuth"); + includeFileLines.add("set SOLR_AUTH_TYPE=kerberos"); + includeFileLines.add( + "set SOLR_AUTHENTICATION_OPTS=\"-Dsolr.httpclient.config=" + basicAuthConfFile + "\""); + } else { + includeFileLines.add("# The following lines added by ./solr for enabling BasicAuth"); + includeFileLines.add("SOLR_AUTH_TYPE=\"kerberos\""); + includeFileLines.add("SOLR_AUTHENTICATION_OPTS=\"" + kerberosConfig + "\""); + } + } + FileUtils.writeLines(includeFile, StandardCharsets.UTF_8.name(), includeFileLines); + + if (basicAuthConfFile != null) { + echoIfVerbose("Written out credentials file: " + basicAuthConfFile, cli); + } + echoIfVerbose("Updated Solr include file: " + includeFile.getAbsolutePath(), cli); + } + + private void updateIncludeFileDisableAuth(File includeFile, CommandLine cli) throws IOException { + List includeFileLines = FileUtils.readLines(includeFile, StandardCharsets.UTF_8); + boolean hasChanged = false; + for (int i = 0; i < includeFileLines.size(); i++) { + String line = includeFileLines.get(i); + if (authenticationVariables.contains(line.trim().split("=")[0].trim())) { // Non-Windows + includeFileLines.set(i, "# " + line); + hasChanged = true; + } + if (line.trim().split("=")[0].trim().startsWith("set ") + && authenticationVariables.contains( + line.trim().split("=")[0].trim().substring(4))) { // Windows + includeFileLines.set(i, "REM " + line); + hasChanged = true; + } + } + if (hasChanged) { + FileUtils.writeLines(includeFile, StandardCharsets.UTF_8.name(), includeFileLines); + echoIfVerbose("Commented out necessary lines from " + includeFile.getAbsolutePath(), cli); + } + } + + @Override + public void runImpl(CommandLine cli) throws Exception {} +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/ConfigSetDownloadTool.java b/solr/core/src/java/org/apache/solr/util/cli/ConfigSetDownloadTool.java new file mode 100644 index 000000000000..0a3505c07ab5 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/ConfigSetDownloadTool.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ConfigSetDownloadTool extends ToolBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public ConfigSetDownloadTool() { + this(CLIO.getOutStream()); + } + + public ConfigSetDownloadTool(PrintStream stdout) { + super(stdout); + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("confname") + .argName("confname") + .hasArg() + .required(true) + .desc("Configset name in ZooKeeper.") + .build(), + Option.builder("confdir") + .argName("confdir") + .hasArg() + .required(true) + .desc("Local directory with configs.") + .build(), + SolrCLI.OPTION_ZKHOST, + SolrCLI.OPTION_VERBOSE + }; + } + + @Override + public String getName() { + return "downconfig"; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String zkHost = SolrCLI.getZkHost(cli); + if (zkHost == null) { + throw new IllegalStateException( + "Solr at " + + cli.getOptionValue("solrUrl") + + " is running in standalone server mode, downconfig can only be used when running in SolrCloud mode.\n"); + } + + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); + String confName = cli.getOptionValue("confname"); + String confDir = cli.getOptionValue("confdir"); + Path configSetPath = Paths.get(confDir); + // we try to be nice about having the "conf" in the directory, and we create it if it's not + // there. + if (!configSetPath.endsWith("/conf")) { + configSetPath = Paths.get(configSetPath.toString(), "conf"); + } + Files.createDirectories(configSetPath); + echo( + "Downloading configset " + + confName + + " from ZooKeeper at " + + zkHost + + " to directory " + + configSetPath.toAbsolutePath()); + + zkClient.downConfig(confName, configSetPath); + } catch (Exception e) { + log.error("Could not complete downconfig operation for reason: ", e); + throw (e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/util/cli/ConfigSetUploadTool.java new file mode 100644 index 000000000000..a7086a3b313a --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/ConfigSetUploadTool.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; +import org.apache.solr.core.ConfigSetService; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ConfigSetUploadTool extends ToolBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public ConfigSetUploadTool() { + this(CLIO.getOutStream()); + } + + public ConfigSetUploadTool(PrintStream stdout) { + super(stdout); + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("confname") + .argName("confname") // Comes out in help message + .hasArg() // Has one sub-argument + .required(true) // confname argument must be present + .desc("Configset name in ZooKeeper.") + .build(), // passed as -confname value + Option.builder("confdir") + .argName("confdir") + .hasArg() + .required(true) + .desc("Local directory with configs.") + .build(), + Option.builder("configsetsDir") + .argName("configsetsDir") + .hasArg() + .required(false) + .desc("Parent directory of example configsets.") + .build(), + SolrCLI.OPTION_ZKHOST, + SolrCLI.OPTION_VERBOSE + }; + } + + @Override + public String getName() { + return "upconfig"; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String zkHost = SolrCLI.getZkHost(cli); + if (zkHost == null) { + throw new IllegalStateException( + "Solr at " + + cli.getOptionValue("solrUrl") + + " is running in standalone server mode, upconfig can only be used when running in SolrCloud mode.\n"); + } + + String confName = cli.getOptionValue("confname"); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); + Path confPath = + ConfigSetService.getConfigsetPath( + cli.getOptionValue("confdir"), cli.getOptionValue("configsetsDir")); + + echo( + "Uploading " + + confPath.toAbsolutePath() + + " for config " + + cli.getOptionValue("confname") + + " to ZooKeeper at " + + zkHost); + ZkMaintenanceUtils.uploadToZK( + zkClient, + confPath, + ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName, + ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN); + + } catch (Exception e) { + log.error("Could not complete upconfig operation for reason: ", e); + throw (e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/ConfigTool.java b/solr/core/src/java/org/apache/solr/util/cli/ConfigTool.java new file mode 100644 index 000000000000..7d3d14e97c68 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/ConfigTool.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.noggit.CharArr; +import org.noggit.JSONWriter; + +/** Sends a POST to the Config API to perform a specified action. */ +public class ConfigTool extends ToolBase { + + public ConfigTool() { + this(CLIO.getOutStream()); + } + + public ConfigTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "config"; + } + + @Override + public Option[] getOptions() { + Option[] configOptions = + new Option[] { + Option.builder("action") + .argName("ACTION") + .hasArg() + .required(false) + .desc( + "Config API action, one of: set-property, unset-property; default is 'set-property'.") + .build(), + Option.builder("property") + .argName("PROP") + .hasArg() + .required(true) + .desc( + "Name of the Config API property to apply the action to, such as: 'updateHandler.autoSoftCommit.maxTime'.") + .build(), + Option.builder("value") + .argName("VALUE") + .hasArg() + .required(false) + .desc("Set the property to this value; accepts JSON objects and strings.") + .build(), + SolrCLI.OPTION_SOLRURL, + SolrCLI.OPTION_ZKHOST, + Option.builder("p") + .argName("PORT") + .hasArg() + .required(false) + .desc("The port of the Solr node to use when applying configuration change.") + .longOpt("port") + .build(), + Option.builder("s") + .argName("SCHEME") + .hasArg() + .required(false) + .desc( + "The scheme for accessing Solr. Accepted values: http or https. Default is 'http'") + .longOpt("scheme") + .build() + }; + return SolrCLI.joinOptions(configOptions, SolrCLI.CLOUD_OPTIONS); + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + String solrUrl; + try { + solrUrl = SolrCLI.resolveSolrUrl(cli); + } catch (IllegalStateException e) { + // Fallback to using the provided scheme and port + final String scheme = cli.getOptionValue("scheme", "http"); + if (cli.hasOption("port")) { + solrUrl = scheme + "://localhost:" + cli.getOptionValue("port", "8983") + "/solr"; + } else { + throw e; + } + } + + String action = cli.getOptionValue("action", "set-property"); + String collection = cli.getOptionValue("collection", "gettingstarted"); + String property = cli.getOptionValue("property"); + String value = cli.getOptionValue("value"); + + Map jsonObj = new HashMap<>(); + if (value != null) { + Map setMap = new HashMap<>(); + setMap.put(property, value); + jsonObj.put(action, setMap); + } else { + jsonObj.put(action, property); + } + + CharArr arr = new CharArr(); + (new JSONWriter(arr, 0)).write(jsonObj); + String jsonBody = arr.toString(); + + String updatePath = "/" + collection + "/config"; + + echo("\nPOSTing request to Config API: " + solrUrl + updatePath); + echo(jsonBody); + + try (SolrClient solrClient = new HttpSolrClient.Builder(solrUrl).build()) { + NamedList result = SolrCLI.postJsonToSolr(solrClient, updatePath, jsonBody); + Integer statusCode = (Integer) ((NamedList) result.get("responseHeader")).get("status"); + if (statusCode == 0) { + if (value != null) { + echo("Successfully " + action + " " + property + " to " + value); + } else { + echo("Successfully " + action + " " + property); + } + } else { + throw new Exception("Failed to " + action + " property due to:\n" + result); + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/CreateCollectionTool.java b/solr/core/src/java/org/apache/solr/util/cli/CreateCollectionTool.java new file mode 100644 index 000000000000..fc786875445d --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/CreateCollectionTool.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import static org.apache.solr.common.params.CommonParams.NAME; + +import java.io.PrintStream; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionAdminParams; +import org.apache.solr.core.ConfigSetService; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.noggit.CharArr; +import org.noggit.JSONWriter; + +/** Supports create_collection command in the bin/solr script. */ +public class CreateCollectionTool extends ToolBase { + + public CreateCollectionTool() { + this(CLIO.getOutStream()); + } + + public CreateCollectionTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "create_collection"; + } + + @Override + public Option[] getOptions() { + return SolrCLI.CREATE_COLLECTION_OPTIONS; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String zkHost = SolrCLI.getZkHost(cli); + if (zkHost == null) { + throw new IllegalStateException( + "Solr at " + + cli.getOptionValue("solrUrl") + + " is running in standalone server mode, please use the create_core command instead;\n" + + "create_collection can only be used when running in SolrCloud mode.\n"); + } + + try (CloudSolrClient cloudSolrClient = + new CloudLegacySolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) + .build()) { + echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); + cloudSolrClient.connect(); + runCloudTool(cloudSolrClient, cli); + } + } + + protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) throws Exception { + + Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); + if (liveNodes.isEmpty()) + throw new IllegalStateException( + "No live nodes found! Cannot create a collection until " + + "there is at least 1 live node in the cluster."); + + String baseUrl = cli.getOptionValue("solrUrl"); + if (baseUrl == null) { + String firstLiveNode = liveNodes.iterator().next(); + baseUrl = ZkStateReader.from(cloudSolrClient).getBaseUrlForNodeName(firstLiveNode); + } + + String collectionName = cli.getOptionValue(NAME); + + // build a URL to create the collection + int numShards = optionAsInt(cli, "shards", 1); + int replicationFactor = optionAsInt(cli, "replicationFactor", 1); + + String confname = cli.getOptionValue("confname"); + String confdir = cli.getOptionValue("confdir"); + String configsetsDir = cli.getOptionValue("configsetsDir"); + + boolean configExistsInZk = + confname != null + && !"".equals(confname.trim()) + && ZkStateReader.from(cloudSolrClient) + .getZkClient() + .exists("/configs/" + confname, true); + + if (CollectionAdminParams.SYSTEM_COLL.equals(collectionName)) { + // do nothing + } else if (configExistsInZk) { + echo("Re-using existing configuration directory " + confname); + } else if (confdir != null && !"".equals(confdir.trim())) { + if (confname == null || "".equals(confname.trim())) { + confname = collectionName; + } + Path confPath = ConfigSetService.getConfigsetPath(confdir, configsetsDir); + + echoIfVerbose( + "Uploading " + + confPath.toAbsolutePath() + + " for config " + + confname + + " to ZooKeeper at " + + cloudSolrClient.getClusterStateProvider().getQuorumHosts(), + cli); + ZkMaintenanceUtils.uploadToZK( + ZkStateReader.from(cloudSolrClient).getZkClient(), + confPath, + ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confname, + ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN); + } + + // since creating a collection is a heavy-weight operation, check for existence first + String collectionListUrl = baseUrl + "/admin/collections?action=list"; + if (SolrCLI.safeCheckCollectionExists(collectionListUrl, collectionName)) { + throw new IllegalStateException( + "\nCollection '" + + collectionName + + "' already exists!\nChecked collection existence using Collections API command:\n" + + collectionListUrl); + } + + // doesn't seem to exist ... try to create + String createCollectionUrl = + String.format( + Locale.ROOT, + "%s/admin/collections?action=CREATE&name=%s&numShards=%d&replicationFactor=%d", + baseUrl, + collectionName, + numShards, + replicationFactor); + if (confname != null && !"".equals(confname.trim())) { + createCollectionUrl = + createCollectionUrl + String.format(Locale.ROOT, "&collection.configName=%s", confname); + } + + echoIfVerbose( + "\nCreating new collection '" + + collectionName + + "' using command:\n" + + createCollectionUrl + + "\n", + cli); + + Map json; + try { + json = SolrCLI.getJson(createCollectionUrl); + } catch (SolrServerException sse) { + throw new Exception( + "Failed to create collection '" + collectionName + "' due to: " + sse.getMessage()); + } + + if (cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt())) { + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(json); + echo(arr.toString()); + } else { + String endMessage = + String.format( + Locale.ROOT, + "Created collection '%s' with %d shard(s), %d replica(s)", + collectionName, + numShards, + replicationFactor); + if (confname != null && !"".equals(confname.trim())) { + endMessage += String.format(Locale.ROOT, " with config-set '%s'", confname); + } + + echo(endMessage); + } + } + + protected int optionAsInt(CommandLine cli, String option, int defaultVal) { + return Integer.parseInt(cli.getOptionValue(option, String.valueOf(defaultVal))); + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/CreateCoreTool.java b/solr/core/src/java/org/apache/solr/util/cli/CreateCoreTool.java new file mode 100644 index 000000000000..4cb1082d0cd4 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/CreateCoreTool.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import static org.apache.solr.common.params.CommonParams.NAME; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Locale; +import java.util.Map; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.file.PathUtils; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.noggit.CharArr; +import org.noggit.JSONWriter; + +public class CreateCoreTool extends ToolBase { + + public CreateCoreTool() { + this(CLIO.getOutStream()); + } + + public CreateCoreTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "create_core"; + } + + @Override + public Option[] getOptions() { + return new Option[] { + SolrCLI.OPTION_SOLRURL, + Option.builder(NAME) + .argName("NAME") + .hasArg() + .required(true) + .desc("Name of the core to create.") + .build(), + Option.builder("confdir") + .argName("CONFIG") + .hasArg() + .required(false) + .desc( + "Configuration directory to copy when creating the new core; default is " + + SolrCLI.DEFAULT_CONFIG_SET + + '.') + .build(), + Option.builder("configsetsDir") + .argName("DIR") + .hasArg() + .required(true) + .desc("Path to configsets directory on the local system.") + .build(), + SolrCLI.OPTION_VERBOSE + }; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + String solrUrl = cli.getOptionValue("solrUrl", SolrCLI.DEFAULT_SOLR_URL); + if (!solrUrl.endsWith("/")) solrUrl += "/"; + + File configsetsDir = new File(cli.getOptionValue("configsetsDir")); + if (!configsetsDir.isDirectory()) + throw new FileNotFoundException(configsetsDir.getAbsolutePath() + " not found!"); + + String configSet = cli.getOptionValue("confdir", SolrCLI.DEFAULT_CONFIG_SET); + File configSetDir = new File(configsetsDir, configSet); + if (!configSetDir.isDirectory()) { + // we allow them to pass a directory instead of a configset name + File possibleConfigDir = new File(configSet); + if (possibleConfigDir.isDirectory()) { + configSetDir = possibleConfigDir; + } else { + throw new FileNotFoundException( + "Specified config directory " + + configSet + + " not found in " + + configsetsDir.getAbsolutePath()); + } + } + + String coreName = cli.getOptionValue(NAME); + + String systemInfoUrl = solrUrl + "admin/info/system"; + String coreRootDirectory; // usually same as solr home, but not always + try (CloseableHttpClient httpClient = SolrCLI.getHttpClient()) { + Map systemInfo = SolrCLI.getJson(httpClient, systemInfoUrl, 2, true); + if ("solrcloud".equals(systemInfo.get("mode"))) { + throw new IllegalStateException( + "Solr at " + + solrUrl + + " is running in SolrCloud mode, please use create_collection command instead."); + } + + // convert raw JSON into user-friendly output + coreRootDirectory = (String) systemInfo.get("core_root"); + + // Fall back to solr_home, in case we are running against older server that does not return + // the property + if (coreRootDirectory == null) coreRootDirectory = (String) systemInfo.get("solr_home"); + if (coreRootDirectory == null) + coreRootDirectory = configsetsDir.getParentFile().getAbsolutePath(); + } + + String coreStatusUrl = solrUrl + "admin/cores?action=STATUS&core=" + coreName; + if (SolrCLI.safeCheckCoreExists(coreStatusUrl, coreName)) { + throw new IllegalArgumentException( + "\nCore '" + + coreName + + "' already exists!\nChecked core existence using Core API command:\n" + + coreStatusUrl); + } + + File coreInstanceDir = new File(coreRootDirectory, coreName); + File confDir = new File(configSetDir, "conf"); + if (!coreInstanceDir.isDirectory()) { + boolean result = coreInstanceDir.mkdirs(); + if (!result || !coreInstanceDir.isDirectory()) + throw new IOException( + "Failed to create new core instance directory: " + coreInstanceDir.getAbsolutePath()); + + if (confDir.isDirectory()) { + FileUtils.copyDirectoryToDirectory(confDir, coreInstanceDir); + } else { + // hmmm ... the configset we're cloning doesn't have a conf sub-directory, + // we'll just assume it is OK if it has solrconfig.xml + if ((new File(configSetDir, "solrconfig.xml")).isFile()) { + FileUtils.copyDirectory(configSetDir, new File(coreInstanceDir, "conf")); + } else { + throw new IllegalArgumentException( + "\n" + + configSetDir.getAbsolutePath() + + " doesn't contain a conf subdirectory or solrconfig.xml\n"); + } + } + echoIfVerbose( + "\nCopying configuration to new core instance directory:\n" + + coreInstanceDir.getAbsolutePath(), + cli); + } + + String createCoreUrl = + String.format( + Locale.ROOT, + "%sadmin/cores?action=CREATE&name=%s&instanceDir=%s", + solrUrl, + coreName, + coreName); + + echoIfVerbose( + "\nCreating new core '" + coreName + "' using command:\n" + createCoreUrl + "\n", cli); + + try { + Map json = SolrCLI.getJson(createCoreUrl); + if (cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt())) { + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(json); + echo(arr.toString()); + echo("\n"); + } else { + echo(String.format(Locale.ROOT, "\nCreated new core '%s'", coreName)); + } + } catch (Exception e) { + /* create-core failed, cleanup the copied configset before propagating the error. */ + PathUtils.deleteDirectory(coreInstanceDir.toPath()); + throw e; + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/util/cli/CreateTool.java new file mode 100644 index 000000000000..df74dd21e775 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/CreateTool.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.util.Map; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; + +public class CreateTool extends ToolBase { + + public CreateTool() { + this(CLIO.getOutStream()); + } + + public CreateTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "create"; + } + + @Override + public Option[] getOptions() { + return SolrCLI.CREATE_COLLECTION_OPTIONS; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String solrUrl = cli.getOptionValue("solrUrl", SolrCLI.DEFAULT_SOLR_URL); + if (!solrUrl.endsWith("/")) solrUrl += "/"; + + String systemInfoUrl = solrUrl + "admin/info/system"; + + ToolBase tool; + try (CloseableHttpClient httpClient = SolrCLI.getHttpClient()) { + Map systemInfo = SolrCLI.getJson(httpClient, systemInfoUrl, 2, true); + if ("solrcloud".equals(systemInfo.get("mode"))) { + tool = new CreateCollectionTool(stdout); + } else { + tool = new CreateCoreTool(stdout); + } + tool.runImpl(cli); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/DeleteTool.java b/solr/core/src/java/org/apache/solr/util/cli/DeleteTool.java new file mode 100644 index 000000000000..59d46d5199f0 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/DeleteTool.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import static org.apache.solr.common.params.CommonParams.NAME; + +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.noggit.CharArr; +import org.noggit.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DeleteTool extends ToolBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public DeleteTool() { + this(CLIO.getOutStream()); + } + + public DeleteTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "delete"; + } + + @Override + public Option[] getOptions() { + return new Option[] { + SolrCLI.OPTION_SOLRURL, + Option.builder(NAME) + .argName("NAME") + .hasArg() + .required(true) + .desc("Name of the core / collection to delete.") + .build(), + Option.builder("deleteConfig") + .argName("true|false") + .hasArg() + .required(false) + .desc( + "Flag to indicate if the underlying configuration directory for a collection should also be deleted; default is true.") + .build(), + Option.builder("forceDeleteConfig") + .required(false) + .desc( + "Skip safety checks when deleting the configuration directory used by a collection.") + .build(), + SolrCLI.OPTION_ZKHOST, + SolrCLI.OPTION_VERBOSE + }; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String solrUrl = cli.getOptionValue("solrUrl", SolrCLI.DEFAULT_SOLR_URL); + if (!solrUrl.endsWith("/")) solrUrl += "/"; + + String systemInfoUrl = solrUrl + "admin/info/system"; + try (CloseableHttpClient httpClient = SolrCLI.getHttpClient()) { + Map systemInfo = SolrCLI.getJson(httpClient, systemInfoUrl, 2, true); + if ("solrcloud".equals(systemInfo.get("mode"))) { + deleteCollection(cli); + } else { + deleteCore(cli, solrUrl); + } + } + } + + protected void deleteCollection(CommandLine cli) throws Exception { + String zkHost = SolrCLI.getZkHost(cli); + try (CloudSolrClient cloudSolrClient = + new CloudLegacySolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) + .withSocketTimeout(30000, TimeUnit.MILLISECONDS) + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) + .build()) { + echoIfVerbose("Connecting to ZooKeeper at " + zkHost, cli); + cloudSolrClient.connect(); + deleteCollection(cloudSolrClient, cli); + } + } + + protected void deleteCollection(CloudSolrClient cloudSolrClient, CommandLine cli) + throws Exception { + Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); + if (liveNodes.isEmpty()) + throw new IllegalStateException( + "No live nodes found! Cannot delete a collection until " + + "there is at least 1 live node in the cluster."); + + String firstLiveNode = liveNodes.iterator().next(); + ZkStateReader zkStateReader = ZkStateReader.from(cloudSolrClient); + String baseUrl = zkStateReader.getBaseUrlForNodeName(firstLiveNode); + String collectionName = cli.getOptionValue(NAME); + if (!zkStateReader.getClusterState().hasCollection(collectionName)) { + throw new IllegalArgumentException("Collection " + collectionName + " not found!"); + } + + String configName = + zkStateReader.getClusterState().getCollection(collectionName).getConfigName(); + boolean deleteConfig = "true".equals(cli.getOptionValue("deleteConfig", "true")); + if (deleteConfig && configName != null) { + if (cli.hasOption("forceDeleteConfig")) { + log.warn( + "Skipping safety checks, configuration directory {} will be deleted with impunity.", + configName); + } else { + // need to scan all Collections to see if any are using the config + Set collections = zkStateReader.getClusterState().getCollectionsMap().keySet(); + + // give a little note to the user if there are many collections in case it takes a while + if (collections.size() > 50) + if (log.isInfoEnabled()) { + log.info( + "Scanning {} to ensure no other collections are using config {}", + collections.size(), + configName); + } + + Optional inUse = + collections.stream() + .filter(name -> !name.equals(collectionName)) // ignore this collection + .filter( + name -> + configName.equals( + zkStateReader.getClusterState().getCollection(name).getConfigName())) + .findFirst(); + if (inUse.isPresent()) { + deleteConfig = false; + log.warn( + "Configuration directory {} is also being used by {}{}", + configName, + inUse.get(), + "; configuration will not be deleted from ZooKeeper. You can pass the -forceDeleteConfig flag to force delete."); + } + } + } + + String deleteCollectionUrl = + String.format( + Locale.ROOT, "%s/admin/collections?action=DELETE&name=%s", baseUrl, collectionName); + + echoIfVerbose( + "\nDeleting collection '" + + collectionName + + "' using command:\n" + + deleteCollectionUrl + + "\n", + cli); + + Map json; + try { + json = SolrCLI.getJson(deleteCollectionUrl); + } catch (SolrServerException sse) { + throw new Exception( + "Failed to delete collection '" + collectionName + "' due to: " + sse.getMessage()); + } + + if (deleteConfig) { + String configZnode = "/configs/" + configName; + try { + zkStateReader.getZkClient().clean(configZnode); + } catch (Exception exc) { + echo( + "\nWARNING: Failed to delete configuration directory " + + configZnode + + " in ZooKeeper due to: " + + exc.getMessage() + + "\nYou'll need to manually delete this znode using the zkcli script."); + } + } + + if (json != null) { + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(json); + echo(arr.toString()); + echo("\n"); + } + + echo("Deleted collection '" + collectionName + "' using command:\n" + deleteCollectionUrl); + } + + protected void deleteCore(CommandLine cli, String solrUrl) throws Exception { + String coreName = cli.getOptionValue(NAME); + String deleteCoreUrl = + String.format( + Locale.ROOT, + "%sadmin/cores?action=UNLOAD&core=%s&deleteIndex=true&deleteDataDir=true&deleteInstanceDir=true", + solrUrl, + coreName); + + echo("\nDeleting core '" + coreName + "' using command:\n" + deleteCoreUrl + "\n"); + + Map json; + try { + json = SolrCLI.getJson(deleteCoreUrl); + } catch (SolrServerException sse) { + throw new Exception("Failed to delete core '" + coreName + "' due to: " + sse.getMessage()); + } + + if (json != null) { + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(json); + echoIfVerbose(arr.toString(), cli); + echoIfVerbose("\n", cli); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/HealthcheckTool.java b/solr/core/src/java/org/apache/solr/util/cli/HealthcheckTool.java new file mode 100644 index 000000000000..660048cfb521 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/HealthcheckTool.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import static org.apache.solr.common.params.CommonParams.DISTRIB; + +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.cli.CommandLine; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkCoreNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.noggit.CharArr; +import org.noggit.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Requests health information about a specific collection in SolrCloud. */ +public class HealthcheckTool extends SolrCloudTool { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public HealthcheckTool() { + this(CLIO.getOutStream()); + } + + public HealthcheckTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "healthcheck"; + } + + @Override + protected void runCloudTool(CloudLegacySolrClient cloudSolrClient, CommandLine cli) + throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String collection = cli.getOptionValue("collection"); + if (collection == null) + throw new IllegalArgumentException("Must provide a collection to run a healthcheck against!"); + + log.debug("Running healthcheck for {}", collection); + + ZkStateReader zkStateReader = ZkStateReader.from(cloudSolrClient); + + ClusterState clusterState = zkStateReader.getClusterState(); + Set liveNodes = clusterState.getLiveNodes(); + final DocCollection docCollection = clusterState.getCollectionOrNull(collection); + if (docCollection == null || docCollection.getSlices() == null) + throw new IllegalArgumentException("Collection " + collection + " not found!"); + + Collection slices = docCollection.getSlices(); + // Test http code using a HEAD request first, fail fast if authentication failure + String urlForColl = + zkStateReader.getLeaderUrl(collection, slices.stream().findFirst().get().getName(), 1000); + SolrCLI.attemptHttpHead(urlForColl, cloudSolrClient.getHttpClient()); + + SolrQuery q = new SolrQuery("*:*"); + q.setRows(0); + QueryResponse qr = cloudSolrClient.query(collection, q); + String collErr = null; + long docCount = -1; + try { + docCount = qr.getResults().getNumFound(); + } catch (Exception exc) { + collErr = String.valueOf(exc); + } + + List shardList = new ArrayList<>(); + boolean collectionIsHealthy = (docCount != -1); + + for (Slice slice : slices) { + String shardName = slice.getName(); + // since we're reporting health of this shard, there's no guarantee of a leader + String leaderUrl = null; + try { + leaderUrl = zkStateReader.getLeaderUrl(collection, shardName, 1000); + } catch (Exception exc) { + log.warn("Failed to get leader for shard {} due to: {}", shardName, exc); + } + + List replicaList = new ArrayList<>(); + for (Replica r : slice.getReplicas()) { + + String uptime = null; + String memory = null; + String replicaStatus; + long numDocs = -1L; + + ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r); + String coreUrl = replicaCoreProps.getCoreUrl(); + boolean isLeader = coreUrl.equals(leaderUrl); + + // if replica's node is not live, its status is DOWN + String nodeName = replicaCoreProps.getNodeName(); + if (nodeName == null || !liveNodes.contains(nodeName)) { + replicaStatus = Replica.State.DOWN.toString(); + } else { + // query this replica directly to get doc count and assess health + q = new SolrQuery("*:*"); + q.setRows(0); + q.set(DISTRIB, "false"); + try (HttpSolrClient solr = new HttpSolrClient.Builder(coreUrl).build()) { + + String solrUrl = solr.getBaseURL(); + + qr = solr.query(q); + numDocs = qr.getResults().getNumFound(); + + int lastSlash = solrUrl.lastIndexOf('/'); + String systemInfoUrl = solrUrl.substring(0, lastSlash) + "/admin/info/system"; + Map info = + SolrCLI.getJson(solr.getHttpClient(), systemInfoUrl, 2, true); + uptime = SolrCLI.uptime(SolrCLI.asLong("/jvm/jmx/upTimeMS", info)); + String usedMemory = SolrCLI.asString("/jvm/memory/used", info); + String totalMemory = SolrCLI.asString("/jvm/memory/total", info); + memory = usedMemory + " of " + totalMemory; + + // if we get here, we can trust the state + replicaStatus = replicaCoreProps.getState(); + } catch (Exception exc) { + log.error("ERROR: {} when trying to reach: {}", exc, coreUrl); + + if (SolrCLI.checkCommunicationError(exc)) { + replicaStatus = Replica.State.DOWN.toString(); + } else { + replicaStatus = "error: " + exc; + } + } + } + + replicaList.add( + new SolrCLI.ReplicaHealth( + shardName, r.getName(), coreUrl, replicaStatus, numDocs, isLeader, uptime, memory)); + } + + SolrCLI.ShardHealth shardHealth = new SolrCLI.ShardHealth(shardName, replicaList); + if (SolrCLI.ShardState.healthy != shardHealth.getShardState()) + collectionIsHealthy = false; // at least one shard is un-healthy + + shardList.add(shardHealth.asMap()); + } + + Map report = new LinkedHashMap<>(); + report.put("collection", collection); + report.put("status", collectionIsHealthy ? "healthy" : "degraded"); + if (collErr != null) { + report.put("error", collErr); + } + report.put("numDocs", docCount); + report.put("numShards", slices.size()); + report.put("shards", shardList); + + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(report); + echo(arr.toString()); + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/util/cli/RunExampleTool.java new file mode 100644 index 000000000000..8a510b6a7637 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/RunExampleTool.java @@ -0,0 +1,1046 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.net.Socket; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Scanner; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.exec.DefaultExecuteResultHandler; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.Executor; +import org.apache.commons.exec.OS; +import org.apache.commons.exec.environment.EnvironmentUtils; +import org.apache.commons.io.FileUtils; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.common.SolrException; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SimplePostTool; +import org.apache.solr.util.SolrCLI; +import org.noggit.CharArr; +import org.noggit.JSONWriter; + +/** Supports an interactive session with the user to launch (or relaunch the -e cloud example) */ +public class RunExampleTool extends ToolBase { + + private static final String PROMPT_FOR_NUMBER = "Please enter %s [%d]: "; + private static final String PROMPT_FOR_NUMBER_IN_RANGE = + "Please enter %s between %d and %d [%d]: "; + private static final String PROMPT_NUMBER_TOO_SMALL = + "%d is too small! " + PROMPT_FOR_NUMBER_IN_RANGE; + private static final String PROMPT_NUMBER_TOO_LARGE = + "%d is too large! " + PROMPT_FOR_NUMBER_IN_RANGE; + + protected InputStream userInput; + protected Executor executor; + protected String script; + protected File serverDir; + protected File exampleDir; + protected String urlScheme; + + /** Default constructor used by the framework when running as a command-line application. */ + public RunExampleTool() { + this(null, System.in, CLIO.getOutStream()); + } + + public RunExampleTool(Executor executor, InputStream userInput, PrintStream stdout) { + super(stdout); + this.executor = (executor != null) ? executor : new DefaultExecutor(); + this.userInput = userInput; + } + + @Override + public String getName() { + return "run_example"; + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("noprompt") + .required(false) + .desc( + "Don't prompt for input; accept all defaults when running examples that accept user input.") + .build(), + Option.builder("e") + .argName("NAME") + .hasArg() + .required(true) + .desc("Name of the example to launch, one of: cloud, techproducts, schemaless, films.") + .longOpt("example") + .build(), + Option.builder("script") + .argName("PATH") + .hasArg() + .required(false) + .desc("Path to the bin/solr script.") + .build(), + Option.builder("d") + .argName("DIR") + .hasArg() + .required(true) + .desc("Path to the Solr server directory.") + .longOpt("serverDir") + .build(), + Option.builder("force") + .argName("FORCE") + .desc("Force option in case Solr is run as root.") + .build(), + Option.builder("exampleDir") + .argName("DIR") + .hasArg() + .required(false) + .desc( + "Path to the Solr example directory; if not provided, ${serverDir}/../example is expected to exist.") + .build(), + Option.builder("urlScheme") + .argName("SCHEME") + .hasArg() + .required(false) + .desc("Solr URL scheme: http or https, defaults to http if not specified.") + .build(), + Option.builder("p") + .argName("PORT") + .hasArg() + .required(false) + .desc("Specify the port to start the Solr HTTP listener on; default is 8983.") + .longOpt("port") + .build(), + Option.builder("h") + .argName("HOSTNAME") + .hasArg() + .required(false) + .desc("Specify the hostname for this Solr instance.") + .longOpt("host") + .build(), + Option.builder("z") + .argName("ZKHOST") + .hasArg() + .required(false) + .desc("ZooKeeper connection string; only used when running in SolrCloud mode using -c.") + .longOpt("zkhost") + .build(), + Option.builder("c") + .required(false) + .desc( + "Start Solr in SolrCloud mode; if -z not supplied, an embedded ZooKeeper instance is started on Solr port+1000, such as 9983 if Solr is bound to 8983.") + .longOpt("cloud") + .build(), + Option.builder("m") + .argName("MEM") + .hasArg() + .required(false) + .desc( + "Sets the min (-Xms) and max (-Xmx) heap size for the JVM, such as: -m 4g results in: -Xms4g -Xmx4g; by default, this script sets the heap size to 512m.") + .longOpt("memory") + .build(), + Option.builder("a") + .argName("OPTS") + .hasArg() + .required(false) + .desc("Additional options to be passed to the JVM when starting example Solr server(s).") + .longOpt("addlopts") + .build() + }; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + this.urlScheme = cli.getOptionValue("urlScheme", "http"); + + serverDir = new File(cli.getOptionValue("serverDir")); + if (!serverDir.isDirectory()) + throw new IllegalArgumentException( + "Value of -serverDir option is invalid! " + + serverDir.getAbsolutePath() + + " is not a directory!"); + + script = cli.getOptionValue("script"); + if (script != null) { + if (!(new File(script)).isFile()) + throw new IllegalArgumentException( + "Value of -script option is invalid! " + script + " not found"); + } else { + File scriptFile = new File(serverDir.getParentFile(), "bin/solr"); + if (scriptFile.isFile()) { + script = scriptFile.getAbsolutePath(); + } else { + scriptFile = new File(serverDir.getParentFile(), "bin/solr.cmd"); + if (scriptFile.isFile()) { + script = scriptFile.getAbsolutePath(); + } else { + throw new IllegalArgumentException( + "Cannot locate the bin/solr script! Please pass -script to this application."); + } + } + } + + exampleDir = + (cli.hasOption("exampleDir")) + ? new File(cli.getOptionValue("exampleDir")) + : new File(serverDir.getParent(), "example"); + if (!exampleDir.isDirectory()) + throw new IllegalArgumentException( + "Value of -exampleDir option is invalid! " + + exampleDir.getAbsolutePath() + + " is not a directory!"); + + echoIfVerbose( + "Running with\nserverDir=" + + serverDir.getAbsolutePath() + + ",\nexampleDir=" + + exampleDir.getAbsolutePath() + + "\nscript=" + + script, + cli); + + String exampleType = cli.getOptionValue("example"); + if ("cloud".equals(exampleType)) { + runCloudExample(cli); + } else if ("techproducts".equals(exampleType) + || "schemaless".equals(exampleType) + || "films".equals(exampleType)) { + runExample(cli, exampleType); + } else { + throw new IllegalArgumentException( + "Unsupported example " + + exampleType + + "! Please choose one of: cloud, schemaless, techproducts, or films"); + } + } + + protected void runExample(CommandLine cli, String exampleName) throws Exception { + File exDir = setupExampleDir(serverDir, exampleDir, exampleName); + String collectionName = "schemaless".equals(exampleName) ? "gettingstarted" : exampleName; + String configSet = + "techproducts".equals(exampleName) ? "sample_techproducts_configs" : "_default"; + + boolean isCloudMode = cli.hasOption('c'); + String zkHost = cli.getOptionValue('z'); + int port = Integer.parseInt(cli.getOptionValue('p', "8983")); + Map nodeStatus = + startSolr(new File(exDir, "solr"), isCloudMode, cli, port, zkHost, 30); + + // invoke the CreateTool + File configsetsDir = new File(serverDir, "solr/configsets"); + + String solrUrl = (String) nodeStatus.get("baseUrl"); + + // safe check if core / collection already exists + boolean alreadyExists = false; + if (nodeStatus.get("cloud") != null) { + String collectionListUrl = solrUrl + "/admin/collections?action=list"; + if (SolrCLI.safeCheckCollectionExists(collectionListUrl, collectionName)) { + alreadyExists = true; + echo( + "\nWARNING: Collection '" + + collectionName + + "' already exists!\nChecked collection existence using Collections API command:\n" + + collectionListUrl + + "\n"); + } + } else { + String coreName = collectionName; + String coreStatusUrl = solrUrl + "/admin/cores?action=STATUS&core=" + coreName; + if (SolrCLI.safeCheckCoreExists(coreStatusUrl, coreName)) { + alreadyExists = true; + echo( + "\nWARNING: Core '" + + coreName + + "' already exists!\nChecked core existence using Core API command:\n" + + coreStatusUrl + + "\n"); + } + } + + if (!alreadyExists) { + String[] createArgs = + new String[] { + "-name", collectionName, + "-shards", "1", + "-replicationFactor", "1", + "-confname", collectionName, + "-confdir", configSet, + "-configsetsDir", configsetsDir.getAbsolutePath(), + "-solrUrl", solrUrl + }; + CreateTool createTool = new CreateTool(stdout); + int createCode = + createTool.runTool( + SolrCLI.processCommandLineArgs( + createTool.getName(), createTool.getOptions(), createArgs)); + if (createCode != 0) + throw new Exception( + "Failed to create " + collectionName + " using command: " + Arrays.asList(createArgs)); + } + + if ("techproducts".equals(exampleName) && !alreadyExists) { + + File exampledocsDir = new File(exampleDir, "exampledocs"); + if (!exampledocsDir.isDirectory()) { + File readOnlyExampleDir = new File(serverDir.getParentFile(), "example"); + if (readOnlyExampleDir.isDirectory()) { + exampledocsDir = new File(readOnlyExampleDir, "exampledocs"); + } + } + + if (exampledocsDir.isDirectory()) { + String updateUrl = String.format(Locale.ROOT, "%s/%s/update", solrUrl, collectionName); + echo("Indexing tech product example docs from " + exampledocsDir.getAbsolutePath()); + + String currentPropVal = System.getProperty("url"); + System.setProperty("url", updateUrl); + SimplePostTool.main(new String[] {exampledocsDir.getAbsolutePath() + "/*.xml"}); + if (currentPropVal != null) { + System.setProperty("url", currentPropVal); // reset + } else { + System.clearProperty("url"); + } + } else { + echo( + "exampledocs directory not found, skipping indexing step for the techproducts example"); + } + } else if ("films".equals(exampleName) && !alreadyExists) { + SolrClient solrClient = new HttpSolrClient.Builder(solrUrl).build(); + + echo("Adding dense vector field type to films schema \"_default\""); + try { + SolrCLI.postJsonToSolr( + solrClient, + "/" + collectionName + "/schema", + "{\n" + + " \"add-field-type\" : {\n" + + " \"name\":\"knn_vector_10\",\n" + + " \"class\":\"solr.DenseVectorField\",\n" + + " \"vectorDimension\":10,\n" + + " \"similarityFunction\":cosine\n" + + " \"knnAlgorithm\":hnsw\n" + + " }\n" + + " }"); + } catch (Exception ex) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, ex); + } + + echo( + "Adding name, initial_release_date, and film_vector fields to films schema \"_default\""); + try { + SolrCLI.postJsonToSolr( + solrClient, + "/" + collectionName + "/schema", + "{\n" + + " \"add-field\" : {\n" + + " \"name\":\"name\",\n" + + " \"type\":\"text_general\",\n" + + " \"multiValued\":false,\n" + + " \"stored\":true\n" + + " },\n" + + " \"add-field\" : {\n" + + " \"name\":\"initial_release_date\",\n" + + " \"type\":\"pdate\",\n" + + " \"stored\":true\n" + + " },\n" + + " \"add-field\" : {\n" + + " \"name\":\"film_vector\",\n" + + " \"type\":\"knn_vector_10\",\n" + + " \"indexed\":true\n" + + " \"stored\":true\n" + + " }\n" + + " }"); + } catch (Exception ex) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, ex); + } + + echo("Adding paramsets \"algo\" and \"algo_b\" to films configuration for relevancy tuning"); + try { + SolrCLI.postJsonToSolr( + solrClient, + "/" + collectionName + "/config/params", + "{\n" + + " \"set\": {\n" + + " \"algo_a\":{\n" + + " \"defType\":\"dismax\",\n" + + " \"qf\":\"name\"\n" + + " }\n" + + " },\n" + + " \"set\": {\n" + + " \"algo_b\":{\n" + + " \"defType\":\"dismax\",\n" + + " \"qf\":\"name\",\n" + + " \"mm\":\"100%\"\n" + + " }\n" + + " }\n" + + " }\n"); + } catch (Exception ex) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, ex); + } + + File filmsJsonFile = new File(exampleDir, "films/films.json"); + String updateUrl = String.format(Locale.ROOT, "%s/%s/update/json", solrUrl, collectionName); + echo("Indexing films example docs from " + filmsJsonFile.getAbsolutePath()); + String currentPropVal = System.getProperty("url"); + System.setProperty("url", updateUrl); + SimplePostTool.main(new String[] {filmsJsonFile.getAbsolutePath()}); + if (currentPropVal != null) { + System.setProperty("url", currentPropVal); // reset + } else { + System.clearProperty("url"); + } + } + + echo( + "\nSolr " + + exampleName + + " example launched successfully. Direct your Web browser to " + + solrUrl + + " to visit the Solr Admin UI"); + } + + protected void runCloudExample(CommandLine cli) throws Exception { + + boolean prompt = !cli.hasOption("noprompt"); + int numNodes = 2; + int[] cloudPorts = new int[] {8983, 7574, 8984, 7575}; + File cloudDir = new File(exampleDir, "cloud"); + if (!cloudDir.isDirectory()) { + cloudDir.mkdir(); + } + + echo("\nWelcome to the SolrCloud example!\n"); + + Scanner readInput = prompt ? new Scanner(userInput, StandardCharsets.UTF_8.name()) : null; + if (prompt) { + echo( + "This interactive session will help you launch a SolrCloud cluster on your local workstation."); + + // get the number of nodes to start + numNodes = + promptForInt( + readInput, + "To begin, how many Solr nodes would you like to run in your local cluster? (specify 1-4 nodes) [2]: ", + "a number", + numNodes, + 1, + 4); + + echo("Ok, let's start up " + numNodes + " Solr nodes for your example SolrCloud cluster."); + + // get the ports for each port + for (int n = 0; n < numNodes; n++) { + String promptMsg = + String.format( + Locale.ROOT, "Please enter the port for node%d [%d]: ", (n + 1), cloudPorts[n]); + int port = promptForPort(readInput, n + 1, promptMsg, cloudPorts[n]); + while (!isPortAvailable(port)) { + port = + promptForPort( + readInput, + n + 1, + "Oops! Looks like port " + + port + + " is already being used by another process. Please choose a different port.", + cloudPorts[n]); + } + + cloudPorts[n] = port; + echoIfVerbose("Using port " + port + " for node " + (n + 1), cli); + } + } else { + echo("Starting up " + numNodes + " Solr nodes for your example SolrCloud cluster.\n"); + } + + // setup a unique solr.solr.home directory for each node + File node1Dir = setupExampleDir(serverDir, cloudDir, "node1"); + for (int n = 2; n <= numNodes; n++) { + File nodeNDir = new File(cloudDir, "node" + n); + if (!nodeNDir.isDirectory()) { + echo("Cloning " + node1Dir.getAbsolutePath() + " into\n " + nodeNDir.getAbsolutePath()); + FileUtils.copyDirectory(node1Dir, nodeNDir); + } else { + echo(nodeNDir.getAbsolutePath() + " already exists."); + } + } + + // deal with extra args passed to the script to run the example + String zkHost = cli.getOptionValue('z'); + + // start the first node (most likely with embedded ZK) + Map nodeStatus = + startSolr(new File(node1Dir, "solr"), true, cli, cloudPorts[0], zkHost, 30); + + if (zkHost == null) { + @SuppressWarnings("unchecked") + Map cloudStatus = (Map) nodeStatus.get("cloud"); + if (cloudStatus != null) { + String zookeeper = (String) cloudStatus.get("ZooKeeper"); + if (zookeeper != null) zkHost = zookeeper; + } + if (zkHost == null) + throw new Exception("Could not get the ZooKeeper connection string for node1!"); + } + + if (numNodes > 1) { + // start the other nodes + for (int n = 1; n < numNodes; n++) + startSolr( + new File(cloudDir, "node" + (n + 1) + "/solr"), true, cli, cloudPorts[n], zkHost, 30); + } + + String solrUrl = (String) nodeStatus.get("baseUrl"); + if (solrUrl.endsWith("/")) solrUrl = solrUrl.substring(0, solrUrl.length() - 1); + + // wait until live nodes == numNodes + waitToSeeLiveNodes(10 /* max wait */, zkHost, numNodes); + + // create the collection + String collectionName = createCloudExampleCollection(numNodes, readInput, prompt, solrUrl); + + // update the config to enable soft auto-commit + echo("\nEnabling auto soft-commits with maxTime 3 secs using the Config API"); + setCollectionConfigProperty( + solrUrl, collectionName, "updateHandler.autoSoftCommit.maxTime", "3000"); + + echo("\n\nSolrCloud example running, please visit: " + solrUrl + " \n"); + } + + protected void setCollectionConfigProperty( + String solrUrl, String collectionName, String propName, String propValue) { + ConfigTool configTool = new ConfigTool(stdout); + String[] configArgs = + new String[] { + "-collection", + collectionName, + "-property", + propName, + "-value", + propValue, + "-solrUrl", + solrUrl + }; + + // let's not fail if we get this far ... just report error and finish up + try { + configTool.runTool( + SolrCLI.processCommandLineArgs( + configTool.getName(), configTool.getOptions(), configArgs)); + } catch (Exception exc) { + CLIO.err("Failed to update '" + propName + "' property due to: " + exc); + } + } + + protected void waitToSeeLiveNodes(int maxWaitSecs, String zkHost, int numNodes) { + try (CloudSolrClient cloudClient = + new CloudSolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()).build()) { + + cloudClient.connect(); + Set liveNodes = cloudClient.getClusterState().getLiveNodes(); + int numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0; + long timeout = + System.nanoTime() + TimeUnit.NANOSECONDS.convert(maxWaitSecs, TimeUnit.SECONDS); + while (System.nanoTime() < timeout && numLiveNodes < numNodes) { + echo( + "\nWaiting up to " + + maxWaitSecs + + " seconds to see " + + (numNodes - numLiveNodes) + + " more nodes join the SolrCloud cluster ..."); + try { + Thread.sleep(2000); + } catch (InterruptedException ie) { + Thread.interrupted(); + } + liveNodes = cloudClient.getClusterState().getLiveNodes(); + numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0; + } + if (numLiveNodes < numNodes) { + echo( + "\nWARNING: Only " + + numLiveNodes + + " of " + + numNodes + + " are active in the cluster after " + + maxWaitSecs + + " seconds! Please check the solr.log for each node to look for errors.\n"); + } + } catch (Exception exc) { + CLIO.err("Failed to see if " + numNodes + " joined the SolrCloud cluster due to: " + exc); + } + } + + protected Map startSolr( + File solrHomeDir, + boolean cloudMode, + CommandLine cli, + int port, + String zkHost, + int maxWaitSecs) + throws Exception { + + String extraArgs = readExtraArgs(cli.getArgs()); + + String host = cli.getOptionValue('h'); + String memory = cli.getOptionValue('m'); + + String hostArg = (host != null && !"localhost".equals(host)) ? " -h " + host : ""; + String zkHostArg = (zkHost != null) ? " -z " + zkHost : ""; + String memArg = (memory != null) ? " -m " + memory : ""; + String cloudModeArg = cloudMode ? "-cloud " : ""; + String forceArg = cli.hasOption("force") ? " -force" : ""; + + String addlOpts = cli.getOptionValue('a'); + String addlOptsArg = (addlOpts != null) ? " -a \"" + addlOpts + "\"" : ""; + + File cwd = new File(System.getProperty("user.dir")); + File binDir = (new File(script)).getParentFile(); + + boolean isWindows = (OS.isFamilyDOS() || OS.isFamilyWin9x() || OS.isFamilyWindows()); + String callScript = (!isWindows && cwd.equals(binDir.getParentFile())) ? "bin/solr" : script; + + String cwdPath = cwd.getAbsolutePath(); + String solrHome = solrHomeDir.getAbsolutePath(); + + // don't display a huge path for solr home if it is relative to the cwd + if (!isWindows && cwdPath.length() > 1 && solrHome.startsWith(cwdPath)) + solrHome = solrHome.substring(cwdPath.length() + 1); + + String startCmd = + String.format( + Locale.ROOT, + "\"%s\" start %s -p %d -s \"%s\" %s %s %s %s %s %s", + callScript, + cloudModeArg, + port, + solrHome, + hostArg, + zkHostArg, + memArg, + forceArg, + extraArgs, + addlOptsArg); + startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing + + echo("\nStarting up Solr on port " + port + " using command:"); + echo(startCmd + "\n"); + + String solrUrl = + String.format( + Locale.ROOT, "%s://%s:%d/solr", urlScheme, (host != null ? host : "localhost"), port); + + Map nodeStatus = checkPortConflict(solrUrl, solrHomeDir, port); + if (nodeStatus != null) + return nodeStatus; // the server they are trying to start is already running + + int code = 0; + if (isWindows) { + // On Windows, the execution doesn't return, so we have to execute async + // and when calling the script, it seems to be inheriting the environment that launched this + // app, so we have to prune out env vars that may cause issues + Map startEnv = new HashMap<>(); + Map procEnv = EnvironmentUtils.getProcEnvironment(); + if (procEnv != null) { + for (Map.Entry entry : procEnv.entrySet()) { + String envVar = entry.getKey(); + String envVarVal = entry.getValue(); + if (envVarVal != null && !"EXAMPLE".equals(envVar) && !envVar.startsWith("SOLR_")) { + startEnv.put(envVar, envVarVal); + } + } + } + DefaultExecuteResultHandler handler = new DefaultExecuteResultHandler(); + executor.execute(org.apache.commons.exec.CommandLine.parse(startCmd), startEnv, handler); + + // wait for execution. + try { + handler.waitFor(3000); + } catch (InterruptedException ie) { + // safe to ignore ... + Thread.interrupted(); + } + if (handler.hasResult() && handler.getExitValue() != 0) { + throw new Exception( + "Failed to start Solr using command: " + + startCmd + + " Exception : " + + handler.getException()); + } + } else { + try { + code = executor.execute(org.apache.commons.exec.CommandLine.parse(startCmd)); + } catch (ExecuteException e) { + throw new Exception( + "Failed to start Solr using command: " + startCmd + " Exception : " + e); + } + } + if (code != 0) throw new Exception("Failed to start Solr using command: " + startCmd); + + return getNodeStatus(solrUrl, maxWaitSecs); + } + + protected Map checkPortConflict(String solrUrl, File solrHomeDir, int port) { + // quickly check if the port is in use + if (isPortAvailable(port)) return null; // not in use ... try to start + + Map nodeStatus = null; + try { + nodeStatus = (new StatusTool()).getStatus(solrUrl); + } catch (Exception ignore) { + /* just trying to determine if this example is already running. */ + } + + if (nodeStatus != null) { + String solr_home = (String) nodeStatus.get("solr_home"); + if (solr_home != null) { + String solrHomePath = solrHomeDir.getAbsolutePath(); + if (!solrHomePath.endsWith("/")) solrHomePath += "/"; + if (!solr_home.endsWith("/")) solr_home += "/"; + + if (solrHomePath.equals(solr_home)) { + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(nodeStatus); + echo("Solr is already setup and running on port " + port + " with status:\n" + arr); + echo( + "\nIf this is not the example node you are trying to start, please choose a different port."); + nodeStatus.put("baseUrl", solrUrl); + return nodeStatus; + } + } + } + + throw new IllegalStateException("Port " + port + " is already being used by another process."); + } + + protected String readExtraArgs(String[] extraArgsArr) { + String extraArgs = ""; + if (extraArgsArr != null && extraArgsArr.length > 0) { + StringBuilder sb = new StringBuilder(); + int app = 0; + for (int e = 0; e < extraArgsArr.length; e++) { + String arg = extraArgsArr[e]; + if ("e".equals(arg) || "example".equals(arg)) { + e++; // skip over the example arg + continue; + } + + if (app > 0) sb.append(" "); + sb.append(arg); + ++app; + } + extraArgs = sb.toString().trim(); + } + return extraArgs; + } + + protected String createCloudExampleCollection( + int numNodes, Scanner readInput, boolean prompt, String solrUrl) throws Exception { + // yay! numNodes SolrCloud nodes running + int numShards = 2; + int replicationFactor = 2; + String cloudConfig = "_default"; + String collectionName = "gettingstarted"; + + File configsetsDir = new File(serverDir, "solr/configsets"); + String collectionListUrl = solrUrl + "/admin/collections?action=list"; + + if (prompt) { + echo( + "\nNow let's create a new collection for indexing documents in your " + + numNodes + + "-node cluster."); + + while (true) { + collectionName = + prompt( + readInput, + "Please provide a name for your new collection: [" + collectionName + "] ", + collectionName); + + // Test for existence and then prompt to either create another collection or skip the + // creation step + if (SolrCLI.safeCheckCollectionExists(collectionListUrl, collectionName)) { + echo("\nCollection '" + collectionName + "' already exists!"); + int oneOrTwo = + promptForInt( + readInput, + "Do you want to re-use the existing collection or create a new one? Enter 1 to reuse, 2 to create new [1]: ", + "a 1 or 2", + 1, + 1, + 2); + if (oneOrTwo == 1) { + return collectionName; + } else { + continue; + } + } else { + break; // user selected a collection that doesn't exist ... proceed on + } + } + + numShards = + promptForInt( + readInput, + "How many shards would you like to split " + collectionName + " into? [2]", + "a shard count", + 2, + 1, + 4); + + replicationFactor = + promptForInt( + readInput, + "How many replicas per shard would you like to create? [2] ", + "a replication factor", + 2, + 1, + 4); + + echo( + "Please choose a configuration for the " + + collectionName + + " collection, available options are:"); + String validConfigs = "_default or sample_techproducts_configs [" + cloudConfig + "] "; + cloudConfig = prompt(readInput, validConfigs, cloudConfig); + + // validate the cloudConfig name + while (!isValidConfig(configsetsDir, cloudConfig)) { + echo( + cloudConfig + + " is not a valid configuration directory! Please choose a configuration for the " + + collectionName + + " collection, available options are:"); + cloudConfig = prompt(readInput, validConfigs, cloudConfig); + } + } else { + // must verify if default collection exists + if (SolrCLI.safeCheckCollectionExists(collectionListUrl, collectionName)) { + echo( + "\nCollection '" + + collectionName + + "' already exists! Skipping collection creation step."); + return collectionName; + } + } + + // invoke the CreateCollectionTool + String[] createArgs = + new String[] { + "-name", collectionName, + "-shards", String.valueOf(numShards), + "-replicationFactor", String.valueOf(replicationFactor), + "-confname", collectionName, + "-confdir", cloudConfig, + "-configsetsDir", configsetsDir.getAbsolutePath(), + "-solrUrl", solrUrl + }; + + CreateCollectionTool createCollectionTool = new CreateCollectionTool(stdout); + int createCode = + createCollectionTool.runTool( + SolrCLI.processCommandLineArgs( + createCollectionTool.getName(), createCollectionTool.getOptions(), createArgs)); + + if (createCode != 0) + throw new Exception( + "Failed to create collection using command: " + Arrays.asList(createArgs)); + + return collectionName; + } + + protected boolean isValidConfig(File configsetsDir, String config) { + File configDir = new File(configsetsDir, config); + if (configDir.isDirectory()) return true; + + // not a built-in configset ... maybe it's a custom directory? + configDir = new File(config); + return configDir.isDirectory(); + } + + protected Map getNodeStatus(String solrUrl, int maxWaitSecs) throws Exception { + StatusTool statusTool = new StatusTool(); + if (verbose) echo("\nChecking status of Solr at " + solrUrl + " ..."); + + URL solrURL = new URL(solrUrl); + Map nodeStatus = + statusTool.waitToSeeSolrUp(solrUrl, maxWaitSecs, TimeUnit.SECONDS); + nodeStatus.put("baseUrl", solrUrl); + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(nodeStatus); + String mode = (nodeStatus.get("cloud") != null) ? "cloud" : "standalone"; + if (verbose) + echo( + "\nSolr is running on " + + solrURL.getPort() + + " in " + + mode + + " mode with status:\n" + + arr); + + return nodeStatus; + } + + protected File setupExampleDir(File serverDir, File exampleParentDir, String dirName) + throws IOException { + File solrXml = new File(serverDir, "solr/solr.xml"); + if (!solrXml.isFile()) + throw new IllegalArgumentException( + "Value of -serverDir option is invalid! " + solrXml.getAbsolutePath() + " not found!"); + + File zooCfg = new File(serverDir, "solr/zoo.cfg"); + if (!zooCfg.isFile()) + throw new IllegalArgumentException( + "Value of -serverDir option is invalid! " + zooCfg.getAbsolutePath() + " not found!"); + + File solrHomeDir = new File(exampleParentDir, dirName + "/solr"); + if (!solrHomeDir.isDirectory()) { + echo("Creating Solr home directory " + solrHomeDir); + solrHomeDir.mkdirs(); + } else { + echo("Solr home directory " + solrHomeDir.getAbsolutePath() + " already exists."); + } + + copyIfNeeded(solrXml, new File(solrHomeDir, "solr.xml")); + copyIfNeeded(zooCfg, new File(solrHomeDir, "zoo.cfg")); + + return solrHomeDir.getParentFile(); + } + + protected void copyIfNeeded(File src, File dest) throws IOException { + if (!dest.isFile()) FileUtils.copyFile(src, dest); + + if (!dest.isFile()) + throw new IllegalStateException("Required file " + dest.getAbsolutePath() + " not found!"); + } + + protected boolean isPortAvailable(int port) { + Socket s = null; + try { + s = new Socket("localhost", port); + return false; + } catch (IOException e) { + return true; + } finally { + if (s != null) { + try { + s.close(); + } catch (IOException ignore) { + } + } + } + } + + protected Integer promptForPort(Scanner s, int node, String prompt, Integer defVal) { + return promptForInt(s, prompt, "a port for node " + node, defVal, null, null); + } + + protected Integer promptForInt( + Scanner s, String prompt, String label, Integer defVal, Integer min, Integer max) { + Integer inputAsInt = null; + + String value = prompt(s, prompt, null /* default is null since we handle that here */); + if (value != null) { + int attempts = 3; + while (value != null && --attempts > 0) { + try { + inputAsInt = Integer.valueOf(value); + + if (min != null) { + if (inputAsInt < min) { + value = + prompt( + s, + String.format( + Locale.ROOT, + PROMPT_NUMBER_TOO_SMALL, + inputAsInt, + label, + min, + max, + defVal)); + inputAsInt = null; + continue; + } + } + + if (max != null) { + if (inputAsInt > max) { + value = + prompt( + s, + String.format( + Locale.ROOT, + PROMPT_NUMBER_TOO_LARGE, + inputAsInt, + label, + min, + max, + defVal)); + inputAsInt = null; + continue; + } + } + + } catch (NumberFormatException nfe) { + if (verbose) echo(value + " is not a number!"); + + if (min != null && max != null) { + value = + prompt( + s, + String.format( + Locale.ROOT, PROMPT_FOR_NUMBER_IN_RANGE, label, min, max, defVal)); + } else { + value = prompt(s, String.format(Locale.ROOT, PROMPT_FOR_NUMBER, label, defVal)); + } + } + } + if (attempts == 0 && value != null && inputAsInt == null) + echo("Too many failed attempts! Going with default value " + defVal); + } + + return (inputAsInt != null) ? inputAsInt : defVal; + } + + protected String prompt(Scanner s, String prompt) { + return prompt(s, prompt, null); + } + + protected String prompt(Scanner s, String prompt, String defaultValue) { + echo(prompt); + String nextInput = s.nextLine(); + if (nextInput != null) { + nextInput = nextInput.trim(); + if (nextInput.isEmpty()) nextInput = null; + } + return (nextInput != null) ? nextInput : defaultValue; + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/SolrCloudTool.java b/solr/core/src/java/org/apache/solr/util/cli/SolrCloudTool.java new file mode 100644 index 000000000000..fa17fb47e89d --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/SolrCloudTool.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.util.Collections; +import java.util.Optional; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; +import org.apache.solr.util.SolrCLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helps build SolrCloud aware tools by initializing a CloudSolrClient instance before running the + * tool. + */ +public abstract class SolrCloudTool extends ToolBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + protected SolrCloudTool(PrintStream stdout) { + super(stdout); + } + + @Override + public Option[] getOptions() { + return SolrCLI.CLOUD_OPTIONS; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String zkHost = cli.getOptionValue(SolrCLI.OPTION_ZKHOST.getOpt(), SolrCLI.DEFAULT_ZK_HOST); + + log.debug("Connecting to Solr cluster: {}", zkHost); + try (var cloudSolrClient = + new CloudLegacySolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) + .build()) { + + cloudSolrClient.connect(); + runCloudTool(cloudSolrClient, cli); + } + } + + /** Runs a SolrCloud tool with CloudSolrClient initialized */ + protected abstract void runCloudTool(CloudLegacySolrClient cloudSolrClient, CommandLine cli) + throws Exception; +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/StatusTool.java b/solr/core/src/java/org/apache/solr/util/cli/StatusTool.java new file mode 100644 index 000000000000..66bf6b46d55f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/StatusTool.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.net.URL; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.net.ssl.SSLPeerUnverifiedException; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.http.client.HttpClient; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.noggit.CharArr; +import org.noggit.JSONWriter; + +/** Get the status of a Solr server. */ +public class StatusTool extends ToolBase { + + public StatusTool() { + this(CLIO.getOutStream()); + } + + public StatusTool(PrintStream stdout) { + super(stdout); + } + + @Override + public String getName() { + return "status"; + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("solr") + .argName("URL") + .hasArg() + .required(false) + .desc( + "Address of the Solr Web application, defaults to: " + SolrCLI.DEFAULT_SOLR_URL + '.') + .build(), + Option.builder("maxWaitSecs") + .argName("SECS") + .hasArg() + .required(false) + .desc("Wait up to the specified number of seconds to see Solr running.") + .build() + }; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + int maxWaitSecs = Integer.parseInt(cli.getOptionValue("maxWaitSecs", "0")); + String solrUrl = cli.getOptionValue("solr", SolrCLI.DEFAULT_SOLR_URL); + if (maxWaitSecs > 0) { + int solrPort = (new URL(solrUrl)).getPort(); + echo("Waiting up to " + maxWaitSecs + " seconds to see Solr running on port " + solrPort); + try { + waitToSeeSolrUp(solrUrl, maxWaitSecs, TimeUnit.SECONDS); + echo("Started Solr server on port " + solrPort + ". Happy searching!"); + } catch (TimeoutException timeout) { + throw new Exception( + "Solr at " + solrUrl + " did not come online within " + maxWaitSecs + " seconds!"); + } + } else { + try { + CharArr arr = new CharArr(); + new JSONWriter(arr, 2).write(getStatus(solrUrl)); + echo(arr.toString()); + } catch (Exception exc) { + if (SolrCLI.exceptionIsAuthRelated(exc)) { + throw exc; + } + if (SolrCLI.checkCommunicationError(exc)) { + // this is not actually an error from the tool as it's ok if Solr is not online. + CLIO.err("Solr at " + solrUrl + " not online."); + } else { + throw new Exception( + "Failed to get system information from " + solrUrl + " due to: " + exc); + } + } + } + } + + public Map waitToSeeSolrUp(String solrUrl, long maxWait, TimeUnit unit) + throws Exception { + long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(maxWait, unit); + while (System.nanoTime() < timeout) { + try { + return getStatus(solrUrl); + } catch (SSLPeerUnverifiedException exc) { + throw exc; + } catch (Exception exc) { + if (SolrCLI.exceptionIsAuthRelated(exc)) { + throw exc; + } + try { + Thread.sleep(2000L); + } catch (InterruptedException interrupted) { + timeout = 0; // stop looping + } + } + } + throw new TimeoutException( + "Did not see Solr at " + + solrUrl + + " come online within " + + TimeUnit.SECONDS.convert(maxWait, unit) + + " seconds!"); + } + + public Map getStatus(String solrUrl) throws Exception { + Map status; + + if (!solrUrl.endsWith("/")) solrUrl += "/"; + + String systemInfoUrl = solrUrl + "admin/info/system"; + try (CloseableHttpClient httpClient = SolrCLI.getHttpClient()) { + // hit Solr to get system info + Map systemInfo = SolrCLI.getJson(httpClient, systemInfoUrl, 2, true); + // convert raw JSON into user-friendly output + status = reportStatus(solrUrl, systemInfo, httpClient); + } + return status; + } + + public Map reportStatus( + String solrUrl, Map info, HttpClient httpClient) throws Exception { + Map status = new LinkedHashMap<>(); + + String solrHome = (String) info.get("solr_home"); + status.put("solr_home", solrHome != null ? solrHome : "?"); + status.put("version", SolrCLI.asString("/lucene/solr-impl-version", info)); + status.put("startTime", SolrCLI.asString("/jvm/jmx/startTime", info)); + status.put("uptime", SolrCLI.uptime(SolrCLI.asLong("/jvm/jmx/upTimeMS", info))); + + String usedMemory = SolrCLI.asString("/jvm/memory/used", info); + String totalMemory = SolrCLI.asString("/jvm/memory/total", info); + status.put("memory", usedMemory + " of " + totalMemory); + + // if this is a Solr in solrcloud mode, gather some basic cluster info + if ("solrcloud".equals(info.get("mode"))) { + String zkHost = (String) info.get("zkHost"); + status.put("cloud", getCloudStatus(httpClient, solrUrl, zkHost)); + } + + return status; + } + + /** + * Calls the CLUSTERSTATUS endpoint in Solr to get basic status information about the SolrCloud + * cluster. + */ + protected Map getCloudStatus(HttpClient httpClient, String solrUrl, String zkHost) + throws Exception { + Map cloudStatus = new LinkedHashMap<>(); + cloudStatus.put("ZooKeeper", (zkHost != null) ? zkHost : "?"); + + String clusterStatusUrl = solrUrl + "admin/collections?action=CLUSTERSTATUS"; + Map json = SolrCLI.getJson(httpClient, clusterStatusUrl, 2, true); + + List liveNodes = SolrCLI.asList("/cluster/live_nodes", json); + cloudStatus.put("liveNodes", String.valueOf(liveNodes.size())); + + Map collections = SolrCLI.asMap("/cluster/collections", json); + cloudStatus.put("collections", String.valueOf(collections.size())); + + return cloudStatus; + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/Tool.java b/solr/core/src/java/org/apache/solr/util/cli/Tool.java new file mode 100644 index 000000000000..90d877e6b59e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/Tool.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; + +/** Defines the interface to a Solr tool that can be run from the command-line app. */ +public interface Tool { + String getName(); + + Option[] getOptions(); + + int runTool(CommandLine cli) throws Exception; +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/ToolBase.java b/solr/core/src/java/org/apache/solr/util/cli/ToolBase.java new file mode 100644 index 000000000000..bc42c120b397 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/ToolBase.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import org.apache.commons.cli.CommandLine; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; + +public abstract class ToolBase implements Tool { + protected PrintStream stdout; + protected boolean verbose = false; + + protected ToolBase() { + this(CLIO.getOutStream()); + } + + protected ToolBase(PrintStream stdout) { + this.stdout = stdout; + } + + protected void echoIfVerbose(final String msg, CommandLine cli) { + if (cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt())) { + echo(msg); + } + } + + protected void echo(final String msg) { + stdout.println(msg); + } + + @Override + public int runTool(CommandLine cli) throws Exception { + verbose = cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt()); + + int toolExitStatus = 0; + try { + runImpl(cli); + } catch (Exception exc) { + // since this is a CLI, spare the user the stacktrace + String excMsg = exc.getMessage(); + if (excMsg != null) { + CLIO.err("\nERROR: " + excMsg + "\n"); + if (verbose) { + exc.printStackTrace(CLIO.getErrStream()); + } + toolExitStatus = 1; + } else { + throw exc; + } + } + return toolExitStatus; + } + + public abstract void runImpl(CommandLine cli) throws Exception; +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/ZkCpTool.java b/solr/core/src/java/org/apache/solr/util/cli/ZkCpTool.java new file mode 100644 index 000000000000..6c0fb78b9d05 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/ZkCpTool.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZkCpTool extends ToolBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public ZkCpTool() { + this(CLIO.getOutStream()); + } + + public ZkCpTool(PrintStream stdout) { + super(stdout); + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("src") + .argName("src") + .hasArg() + .required(true) + .desc("Source file or directory, may be local or a Znode.") + .build(), + Option.builder("dst") + .argName("dst") + .hasArg() + .required(true) + .desc("Destination of copy, may be local or a Znode.") + .build(), + SolrCLI.OPTION_RECURSE, + SolrCLI.OPTION_ZKHOST, + SolrCLI.OPTION_VERBOSE + }; + } + + @Override + public String getName() { + return "cp"; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String zkHost = SolrCLI.getZkHost(cli); + if (zkHost == null) { + throw new IllegalStateException( + "Solr at " + + cli.getOptionValue("solrUrl") + + " is running in standalone server mode, cp can only be used when running in SolrCloud mode.\n"); + } + + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); + String src = cli.getOptionValue("src"); + String dst = cli.getOptionValue("dst"); + Boolean recurse = Boolean.parseBoolean(cli.getOptionValue("recurse")); + echo("Copying from '" + src + "' to '" + dst + "'. ZooKeeper at " + zkHost); + + boolean srcIsZk = src.toLowerCase(Locale.ROOT).startsWith("zk:"); + boolean dstIsZk = dst.toLowerCase(Locale.ROOT).startsWith("zk:"); + + String srcName = src; + if (srcIsZk) { + srcName = src.substring(3); + } else if (srcName.toLowerCase(Locale.ROOT).startsWith("file:")) { + srcName = srcName.substring(5); + } + + String dstName = dst; + if (dstIsZk) { + dstName = dst.substring(3); + } else { + if (dstName.toLowerCase(Locale.ROOT).startsWith("file:")) { + dstName = dstName.substring(5); + } + } + zkClient.zkTransfer(srcName, srcIsZk, dstName, dstIsZk, recurse); + } catch (Exception e) { + log.error("Could not complete the zk operation for reason: ", e); + throw (e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/ZkLsTool.java b/solr/core/src/java/org/apache/solr/util/cli/ZkLsTool.java new file mode 100644 index 000000000000..47a9871f4378 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/ZkLsTool.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZkLsTool extends ToolBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public ZkLsTool() { + this(CLIO.getOutStream()); + } + + public ZkLsTool(PrintStream stdout) { + super(stdout); + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("path").argName("path").hasArg().required(true).desc("Path to list.").build(), + SolrCLI.OPTION_RECURSE, + SolrCLI.OPTION_ZKHOST, + SolrCLI.OPTION_VERBOSE + }; + } + + @Override + public String getName() { + return "ls"; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String zkHost = SolrCLI.getZkHost(cli); + + if (zkHost == null) { + throw new IllegalStateException( + "Solr at " + + cli.getOptionValue("zkHost") + + " is running in standalone server mode, 'zk ls' can only be used when running in SolrCloud mode.\n"); + } + + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); + + String znode = cli.getOptionValue("path"); + Boolean recurse = Boolean.parseBoolean(cli.getOptionValue("recurse")); + echoIfVerbose( + "Getting listing for ZooKeeper node " + + znode + + " from ZooKeeper at " + + zkHost + + " recurse: " + + recurse, + cli); + stdout.print(zkClient.listZnode(znode, recurse)); + } catch (Exception e) { + log.error("Could not complete ls operation for reason: ", e); + throw (e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/ZkMkrootTool.java b/solr/core/src/java/org/apache/solr/util/cli/ZkMkrootTool.java new file mode 100644 index 000000000000..9fcbfb23870c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/ZkMkrootTool.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZkMkrootTool extends ToolBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public ZkMkrootTool() { + this(CLIO.getOutStream()); + } + + public ZkMkrootTool(PrintStream stdout) { + super(stdout); + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("path") + .argName("path") + .hasArg() + .required(true) + .desc("Path to create.") + .build(), + SolrCLI.OPTION_ZKHOST, + SolrCLI.OPTION_VERBOSE + }; + } + + @Override + public String getName() { + return "mkroot"; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String zkHost = SolrCLI.getZkHost(cli); + + if (zkHost == null) { + throw new IllegalStateException( + "Solr at " + + cli.getOptionValue("zkHost") + + " is running in standalone server mode, 'zk mkroot' can only be used when running in SolrCloud mode.\n"); + } + + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); + + String znode = cli.getOptionValue("path"); + echo("Creating ZooKeeper path " + znode + " on ZooKeeper at " + zkHost); + zkClient.makePath(znode, true); + } catch (Exception e) { + log.error("Could not complete mkroot operation for reason: ", e); + throw (e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/ZkMvTool.java b/solr/core/src/java/org/apache/solr/util/cli/ZkMvTool.java new file mode 100644 index 000000000000..65b1c0a76808 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/ZkMvTool.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZkMvTool extends ToolBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public ZkMvTool() { + this(CLIO.getOutStream()); + } + + public ZkMvTool(PrintStream stdout) { + super(stdout); + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("src") + .argName("src") + .hasArg() + .required(true) + .desc("Source Znode to move from.") + .build(), + Option.builder("dst") + .argName("dst") + .hasArg() + .required(true) + .desc("Destination Znode to move to.") + .build(), + SolrCLI.OPTION_ZKHOST, + SolrCLI.OPTION_VERBOSE + }; + } + + @Override + public String getName() { + return "mv"; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String zkHost = SolrCLI.getZkHost(cli); + if (zkHost == null) { + throw new IllegalStateException( + "Solr at " + + cli.getOptionValue("solrUrl") + + " is running in standalone server mode, 'zk rm' can only be used when running in SolrCloud mode.\n"); + } + + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); + String src = cli.getOptionValue("src"); + String dst = cli.getOptionValue("dst"); + + if (src.toLowerCase(Locale.ROOT).startsWith("file:") + || dst.toLowerCase(Locale.ROOT).startsWith("file:")) { + throw new SolrServerException( + "mv command operates on znodes and 'file:' has been specified."); + } + String source = src; + if (src.toLowerCase(Locale.ROOT).startsWith("zk")) { + source = src.substring(3); + } + + String dest = dst; + if (dst.toLowerCase(Locale.ROOT).startsWith("zk")) { + dest = dst.substring(3); + } + + echo("Moving Znode " + source + " to " + dest + " on ZooKeeper at " + zkHost); + zkClient.moveZnode(source, dest); + } catch (Exception e) { + log.error("Could not complete mv operation for reason: ", e); + throw (e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/ZkRmTool.java b/solr/core/src/java/org/apache/solr/util/cli/ZkRmTool.java new file mode 100644 index 000000000000..11a2140d0f0d --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/ZkRmTool.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.cli; + +import java.io.PrintStream; +import java.lang.invoke.MethodHandles; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.util.CLIO; +import org.apache.solr.util.SolrCLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZkRmTool extends ToolBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public ZkRmTool() { + this(CLIO.getOutStream()); + } + + public ZkRmTool(PrintStream stdout) { + super(stdout); + } + + @Override + public Option[] getOptions() { + return new Option[] { + Option.builder("path") + .argName("path") + .hasArg() + .required(true) + .desc("Path to remove.") + .build(), + SolrCLI.OPTION_RECURSE, + SolrCLI.OPTION_ZKHOST, + SolrCLI.OPTION_VERBOSE + }; + } + + @Override + public String getName() { + return "rm"; + } + + @Override + public void runImpl(CommandLine cli) throws Exception { + SolrCLI.raiseLogLevelUnlessVerbose(cli); + String zkHost = SolrCLI.getZkHost(cli); + + if (zkHost == null) { + throw new IllegalStateException( + "Solr at " + + cli.getOptionValue("zkHost") + + " is running in standalone server mode, 'zk rm' can only be used when running in SolrCloud mode.\n"); + } + String target = cli.getOptionValue("path"); + boolean recurse = Boolean.parseBoolean(cli.getOptionValue("recurse")); + + String znode = target; + if (target.toLowerCase(Locale.ROOT).startsWith("zk:")) { + znode = target.substring(3); + } + if (znode.equals("/")) { + throw new SolrServerException("You may not remove the root ZK node ('/')!"); + } + echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + if (!recurse && zkClient.getChildren(znode, null, true).size() != 0) { + throw new SolrServerException( + "ZooKeeper node " + znode + " has children and recurse has NOT been specified."); + } + echo( + "Removing ZooKeeper node " + + znode + + " from ZooKeeper at " + + zkHost + + " recurse: " + + recurse); + zkClient.clean(znode); + } catch (Exception e) { + log.error("Could not complete rm operation for reason: ", e); + throw (e); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/util/cli/package-info.java b/solr/core/src/java/org/apache/solr/util/cli/package-info.java new file mode 100644 index 000000000000..4cd525bc843e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/cli/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Solr Command Line Interface classes */ +package org.apache.solr.util.cli; diff --git a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java index 96f1d78649df..891ea2720e0d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java +++ b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java @@ -215,10 +215,7 @@ void assertPackageVersion( } private void run(PackageTool tool, String[] args) throws Exception { - int res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + int res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); assertEquals("Non-zero status returned for: " + Arrays.toString(args), 0, res); } diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java index 0ee5d2894565..573711565c8e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SolrCLIZkUtilsTest.java @@ -34,6 +34,12 @@ import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.util.SolrCLI; +import org.apache.solr.util.cli.ConfigSetDownloadTool; +import org.apache.solr.util.cli.ConfigSetUploadTool; +import org.apache.solr.util.cli.ZkCpTool; +import org.apache.solr.util.cli.ZkLsTool; +import org.apache.solr.util.cli.ZkMvTool; +import org.apache.solr.util.cli.ZkRmTool; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; import org.junit.AfterClass; @@ -93,12 +99,9 @@ public void testUpconfig() throws Exception { configSet.toAbsolutePath().toString(), }; - SolrCLI.ConfigSetUploadTool tool = new SolrCLI.ConfigSetUploadTool(); + ConfigSetUploadTool tool = new ConfigSetUploadTool(); - int res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + int res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); assertEquals("tool should have returned 0 for success ", 0, res); // Now do we have that config up on ZK? verifyZkLocalPathsMatch(srcPathCheck, "/configs/upconfig2"); @@ -116,10 +119,7 @@ public void testUpconfig() throws Exception { configSet.toAbsolutePath().toString(), }; - res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); assertTrue("tool should have returned non-zero for failure ", 0 != res); String content = @@ -153,11 +153,10 @@ public void testDownconfig() throws Exception { zkAddr, }; - SolrCLI.ConfigSetDownloadTool downTool = new SolrCLI.ConfigSetDownloadTool(); + ConfigSetDownloadTool downTool = new ConfigSetDownloadTool(); int res = downTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(downTool.getOptions()), args)); + SolrCLI.processCommandLineArgs(downTool.getName(), downTool.getOptions(), args)); assertEquals("Download should have succeeded.", 0, res); verifyZkLocalPathsMatch( Paths.get(tmp.toAbsolutePath().toString(), "conf"), "/configs/downconfig1"); @@ -170,7 +169,7 @@ public void testDownconfig() throws Exception { // Now copy it up and back and insure it's still a file in the new place AbstractDistribZkTestBase.copyConfigUp(tmp.getParent(), "myconfset", "downconfig2", zkAddr); Path tmp2 = createTempDir("downConfigNewPlace2"); - downTool = new SolrCLI.ConfigSetDownloadTool(); + downTool = new ConfigSetDownloadTool(); args = new String[] { "-confname", @@ -183,8 +182,7 @@ public void testDownconfig() throws Exception { res = downTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(downTool.getOptions()), args)); + SolrCLI.processCommandLineArgs(downTool.getName(), downTool.getOptions(), args)); assertEquals("Download should have succeeded.", 0, res); verifyZkLocalPathsMatch( Paths.get(tmp.toAbsolutePath().toString(), "conf"), "/configs/downconfig2"); @@ -211,12 +209,10 @@ public void testCp() throws Exception { "-zkHost", zkAddr, }; - SolrCLI.ZkCpTool cpTool = new SolrCLI.ZkCpTool(); + ZkCpTool cpTool = new ZkCpTool(); int res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy from zk -> zk should have succeeded.", 0, res); verifyZnodesMatch("/configs/cp1", "/cp2"); @@ -235,9 +231,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); verifyZkLocalPathsMatch(tmp, "/configs/cp1"); @@ -256,9 +250,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); verifyZkLocalPathsMatch(tmp, "/configs/cp1"); @@ -276,9 +268,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); verifyZkLocalPathsMatch(srcPathCheck, "/cp3"); @@ -296,9 +286,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); verifyZkLocalPathsMatch(srcPathCheck, "/cp4"); @@ -309,9 +297,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertTrue("Copy should NOT have succeeded, recurse not specified.", 0 != res); // try with recurse = false @@ -328,9 +314,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertTrue("Copy should NOT have succeeded, recurse set to false.", 0 != res); // NOTE: really can't test copying to '.' because the test framework doesn't allow altering the @@ -345,9 +329,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should nave created intermediate directory locally.", 0, res); assertTrue( "File should have been copied to a directory successfully", @@ -368,9 +350,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy up to intermediate file should have succeeded.", 0, res); assertTrue( "Should have created an intermediate node on ZK", @@ -391,9 +371,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy up to named file should have succeeded.", 0, res); assertTrue( "Should NOT have created an intermediate node on ZK", @@ -417,9 +395,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy to local named file should have succeeded.", 0, res); Path locPath = Paths.get(localNamed); assertTrue("Should have found file: " + localNamed, Files.exists(locPath)); @@ -444,9 +420,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy from somewhere in ZK to ZK root should have succeeded.", 0, res); assertTrue( "Should have found znode /solrconfig.xml: ", zkClient.exists("/solrconfig.xml", true)); @@ -466,9 +440,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); verifyZkLocalPathsMatch(srcPathCheck, "/cp7/" + srcPathCheck.getFileName().toString()); @@ -492,9 +464,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); String content = @@ -503,9 +473,7 @@ public void testCp() throws Exception { assertTrue("There should be content in the node! ", content.contains("{Some Arbitrary Data}")); res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); tmp = createTempDir("cp8"); @@ -521,9 +489,7 @@ public void testCp() throws Exception { zkAddr, }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); // Next, copy cp7 down and verify that zknode.data exists for cp7 @@ -544,9 +510,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); content = @@ -571,9 +535,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); Path tmp2 = createTempDir("cp9"); @@ -590,9 +552,7 @@ public void testCp() throws Exception { zkAddr, }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); assertTrue("Empty files should NOT be copied down as directories", emptyDest.toFile().isFile()); @@ -612,9 +572,7 @@ public void testCp() throws Exception { }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); // Now copy it all back and make sure empty file is still a file when recursively copying. @@ -631,9 +589,7 @@ public void testCp() throws Exception { zkAddr, }; res = - cpTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(cpTool.getOptions()), args)); + cpTool.runTool(SolrCLI.processCommandLineArgs(cpTool.getName(), cpTool.getOptions(), args)); assertEquals("Copy should have succeeded.", 0, res); Path locEmpty = Paths.get(tmp2.toAbsolutePath().toString(), "stopwords", "emptyfile"); @@ -658,12 +614,10 @@ public void testMv() throws Exception { "-zkHost", zkAddr, }; - SolrCLI.ZkMvTool mvTool = new SolrCLI.ZkMvTool(); + ZkMvTool mvTool = new ZkMvTool(); int res = - mvTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(mvTool.getOptions()), args)); + mvTool.runTool(SolrCLI.processCommandLineArgs(mvTool.getName(), mvTool.getOptions(), args)); assertEquals("Move should have succeeded.", 0, res); // Now does the moved directory match the original on disk? @@ -680,9 +634,7 @@ public void testMv() throws Exception { // Still in mv2 res = - mvTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(mvTool.getOptions()), args)); + mvTool.runTool(SolrCLI.processCommandLineArgs(mvTool.getName(), mvTool.getOptions(), args)); assertTrue("Move should NOT have succeeded with file: specified.", 0 != res); // Let's move it to yet another place with no zk: prefix. @@ -694,9 +646,7 @@ public void testMv() throws Exception { }; res = - mvTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(mvTool.getOptions()), args)); + mvTool.runTool(SolrCLI.processCommandLineArgs(mvTool.getName(), mvTool.getOptions(), args)); assertEquals("Move should have succeeded.", 0, res); assertFalse("Znode /mv3 really should be gone", zkClient.exists("/mv3", true)); @@ -712,9 +662,7 @@ public void testMv() throws Exception { }; res = - mvTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(mvTool.getOptions()), args)); + mvTool.runTool(SolrCLI.processCommandLineArgs(mvTool.getName(), mvTool.getOptions(), args)); assertEquals("Move should have succeeded.", 0, res); assertTrue( "Should be able to move a single file", @@ -731,9 +679,7 @@ public void testMv() throws Exception { }; res = - mvTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(mvTool.getOptions()), args)); + mvTool.runTool(SolrCLI.processCommandLineArgs(mvTool.getName(), mvTool.getOptions(), args)); assertEquals("Move should have succeeded.", 0, res); assertTrue( "Should be able to move a single file to a parent znode", @@ -760,12 +706,9 @@ public void testLs() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos, false, StandardCharsets.UTF_8.name()); - SolrCLI.ZkLsTool tool = new SolrCLI.ZkLsTool(ps); + ZkLsTool tool = new ZkLsTool(ps); - int res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + int res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); String content = new String(baos.toByteArray(), StandardCharsets.UTF_8); assertEquals("List should have succeeded", res, 0); @@ -780,10 +723,7 @@ public void testLs() throws Exception { "-zkHost", zkAddr, }; - res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); content = new String(baos.toByteArray(), StandardCharsets.UTF_8); assertEquals("List should have succeeded", res, 0); @@ -798,10 +738,7 @@ public void testLs() throws Exception { "-zkHost", zkAddr, }; - res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); content = new String(baos.toByteArray(), StandardCharsets.UTF_8); assertEquals("List should have succeeded", res, 0); @@ -816,10 +753,7 @@ public void testLs() throws Exception { "-zkHost", zkAddr, }; - res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); content = new String(baos.toByteArray(), StandardCharsets.UTF_8); assertEquals("List should have succeeded", res, 0); @@ -831,10 +765,7 @@ public void testLs() throws Exception { "-path", "/", "-zkHost", zkAddr, }; - res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); content = new String(baos.toByteArray(), StandardCharsets.UTF_8); assertEquals("List should have succeeded", res, 0); assertFalse("Return should not contain /zookeeper", content.contains("/zookeeper")); @@ -847,10 +778,7 @@ public void testLs() throws Exception { "-zkHost", zkAddr, }; - res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); content = new String(baos.toByteArray(), StandardCharsets.UTF_8); assertEquals("List should have succeeded", res, 0); @@ -873,12 +801,9 @@ public void testRm() throws Exception { "-path", "/configs/rm1", "-zkHost", zkAddr, }; - SolrCLI.ZkRmTool tool = new SolrCLI.ZkRmTool(); + ZkRmTool tool = new ZkRmTool(); - int res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + int res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); assertTrue( "Should have failed to remove node with children unless -recurse is set to true", res != 0); @@ -893,10 +818,7 @@ public void testRm() throws Exception { "-zkHost", zkAddr, }; - res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); assertTrue( "Should have failed to remove node with children if -recurse is set to false", res != 0); @@ -908,10 +830,7 @@ public void testRm() throws Exception { "-zkHost", zkAddr, }; - res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); assertEquals("Should have removed node /configs/rm1", res, 0); assertFalse( "Znode /configs/toremove really should be gone", zkClient.exists("/configs/rm1", true)); @@ -924,10 +843,7 @@ public void testRm() throws Exception { "-zkHost", zkAddr, }; - res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); assertEquals("Should have removed node /configs/rm2", res, 0); assertFalse( "Znode /configs/toremove2 really should be gone", zkClient.exists("/configs/rm2", true)); @@ -941,10 +857,7 @@ public void testRm() throws Exception { }; AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "rm3", zkAddr); - res = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); assertNotEquals("Should fail when trying to remove /.", 0, res); } diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java index 0a688acabbf2..106332bea1cd 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java @@ -51,6 +51,10 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.util.ExternalPaths; import org.apache.solr.util.SolrCLI; +import org.apache.solr.util.cli.ConfigTool; +import org.apache.solr.util.cli.CreateCollectionTool; +import org.apache.solr.util.cli.DeleteTool; +import org.apache.solr.util.cli.HealthcheckTool; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,7 +74,6 @@ public SolrCloudExampleTest() { } @Test - // 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018 public void testLoadDocsIntoGettingStartedCollection() throws Exception { waitForThingsToLevelOut(30, TimeUnit.SECONDS); @@ -109,9 +112,8 @@ public void testLoadDocsIntoGettingStartedCollection() throws Exception { // NOTE: not calling SolrCLI.main as the script does because it calls System.exit which is a // no-no in a JUnit test - SolrCLI.CreateCollectionTool tool = new SolrCLI.CreateCollectionTool(); - CommandLine cli = - SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args); + CreateCollectionTool tool = new CreateCollectionTool(); + CommandLine cli = SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args); log.info("Creating the '{}' collection using SolrCLI with: {}", testCollectionName, solrUrl); tool.runTool(cli); assertTrue( @@ -132,6 +134,7 @@ public void testLoadDocsIntoGettingStartedCollection() throws Exception { // now index docs like bin/post would, but we can't use SimplePostTool because it uses // System.exit when it encounters an error, which JUnit doesn't like ... log.info("Created collection, now posting example docs!"); + assert ExternalPaths.SOURCE_HOME != null; Path exampleDocsDir = Path.of(ExternalPaths.SOURCE_HOME, "example", "exampledocs"); assertTrue(exampleDocsDir.toAbsolutePath() + " not found!", Files.isDirectory(exampleDocsDir)); @@ -197,9 +200,8 @@ protected void doTestHealthcheck(String testCollectionName, String zkHost) throw "-collection", testCollectionName, "-zkHost", zkHost }; - SolrCLI.HealthcheckTool tool = new SolrCLI.HealthcheckTool(); - CommandLine cli = - SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args); + HealthcheckTool tool = new HealthcheckTool(); + CommandLine cli = SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args); assertEquals("Healthcheck action failed!", 0, tool.runTool(cli)); } @@ -209,9 +211,8 @@ protected void doTestDeleteAction(String testCollectionName, String solrUrl) thr "-name", testCollectionName, "-solrUrl", solrUrl }; - SolrCLI.DeleteTool tool = new SolrCLI.DeleteTool(); - CommandLine cli = - SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args); + DeleteTool tool = new DeleteTool(); + CommandLine cli = SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args); assertEquals("Delete action failed!", 0, tool.runTool(cli)); assertFalse( SolrCLI.safeCheckCollectionExists( @@ -244,9 +245,8 @@ protected void doTestConfigUpdate(String testCollectionName, String solrUrl) thr Map startTimes = getSoftAutocommitInterval(testCollectionName); - SolrCLI.ConfigTool tool = new SolrCLI.ConfigTool(); - CommandLine cli = - SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args); + ConfigTool tool = new ConfigTool(); + CommandLine cli = SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args); log.info("Sending set-property '{}'={} to SolrCLI.ConfigTool.", prop, maxTime); assertEquals("Set config property failed!", 0, tool.runTool(cli)); @@ -274,19 +274,20 @@ protected void doTestConfigUpdate(String testCollectionName, String solrUrl) thr // Since it takes some time for this command to complete we need to make sure all the reloads // for all the cores have been done. boolean allGood = false; - Map curSoftCommitInterval = null; - for (int idx = 0; idx < 600 && allGood == false; ++idx) { + Map curSoftCommitInterval; + for (int idx = 0; idx < 600 && !allGood; ++idx) { curSoftCommitInterval = getSoftAutocommitInterval(testCollectionName); // no point in even trying if they're not the same size! if (curSoftCommitInterval.size() > 0 && curSoftCommitInterval.size() == startTimes.size()) { allGood = true; for (Map.Entry currEntry : curSoftCommitInterval.entrySet()) { - if (currEntry.getValue().equals(maxTime) == false) { + if (!currEntry.getValue().equals(maxTime)) { allGood = false; + break; } } } - if (allGood == false) { + if (!allGood) { Thread.sleep(100); } } diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java index 427d598b2320..a0beac81e8df 100644 --- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java @@ -66,6 +66,7 @@ import org.apache.solr.util.LogLevel; import org.apache.solr.util.SolrCLI; import org.apache.solr.util.TimeOut; +import org.apache.solr.util.cli.StatusTool; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -302,12 +303,10 @@ public void testBasicAuth() throws Exception { String[] toolArgs = new String[] {"status", "-solr", baseUrl}; ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream stdoutSim = new PrintStream(baos, true, StandardCharsets.UTF_8.name()); - SolrCLI.StatusTool tool = new SolrCLI.StatusTool(stdoutSim); + StatusTool tool = new StatusTool(stdoutSim); try { System.setProperty("basicauth", "harry:HarryIsUberCool"); - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs)); + tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), toolArgs)); Map obj = (Map) Utils.fromJSON(new ByteArrayInputStream(baos.toByteArray())); assertTrue(obj.containsKey("version")); assertTrue(obj.containsKey("startTime")); diff --git a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java index 855e85e0c54d..8db5f047be52 100644 --- a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java +++ b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java @@ -46,6 +46,8 @@ import org.apache.solr.common.SolrInputDocument; import org.apache.solr.embedded.JettyConfig; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.cli.DeleteTool; +import org.apache.solr.util.cli.RunExampleTool; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -363,12 +365,11 @@ protected void testExample(String exampleName) throws Exception { RunExampleExecutor executor = new RunExampleExecutor(stdoutSim); closeables.add(executor); - SolrCLI.RunExampleTool tool = new SolrCLI.RunExampleTool(executor, System.in, stdoutSim); + RunExampleTool tool = new RunExampleTool(executor, System.in, stdoutSim); try { int status = tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs)); + SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), toolArgs)); if (status == -1) { // maybe it's the port, try again @@ -378,8 +379,7 @@ protected void testExample(String exampleName) throws Exception { Thread.sleep(100); status = tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs)); + SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), toolArgs)); } assertEquals("it should be ok " + tool + " " + Arrays.toString(toolArgs), 0, status); @@ -486,11 +486,9 @@ public void testInteractiveSolrCloudExample() throws Exception { RunExampleExecutor executor = new RunExampleExecutor(stdoutSim); closeables.add(executor); - SolrCLI.RunExampleTool tool = new SolrCLI.RunExampleTool(executor, userInputSim, stdoutSim); + RunExampleTool tool = new RunExampleTool(executor, userInputSim, stdoutSim); try { - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs)); + tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), toolArgs)); } catch (Exception e) { System.err.println( "RunExampleTool failed due to: " @@ -560,14 +558,13 @@ public void testInteractiveSolrCloudExample() throws Exception { } // delete the collection - SolrCLI.DeleteTool deleteTool = new SolrCLI.DeleteTool(stdoutSim); + DeleteTool deleteTool = new DeleteTool(stdoutSim); String[] deleteArgs = new String[] {"-name", collectionName, "-solrUrl", solrUrl}; deleteTool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(deleteTool.getOptions()), deleteArgs)); + SolrCLI.processCommandLineArgs(deleteTool.getName(), deleteTool.getOptions(), deleteArgs)); // dump all the output written by the SolrCLI commands to stdout - // System.out.println(toolOutput); + System.out.println(toolOutput); // stop the test instance executor.execute(org.apache.commons.exec.CommandLine.parse("bin/solr stop -p " + bindPort)); @@ -609,11 +606,9 @@ public void testFailExecuteScript() throws Exception { DefaultExecutor executor = new DefaultExecutor(); - SolrCLI.RunExampleTool tool = new SolrCLI.RunExampleTool(executor, System.in, stdoutSim); + RunExampleTool tool = new RunExampleTool(executor, System.in, stdoutSim); int code = - tool.runTool( - SolrCLI.processCommandLineArgs( - SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs)); + tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), toolArgs)); assertEquals("Execution should have failed with return code 1", 1, code); } } diff --git a/solr/core/src/test/org/apache/solr/util/AuthToolTest.java b/solr/core/src/test/org/apache/solr/util/cli/AuthToolTest.java similarity index 95% rename from solr/core/src/test/org/apache/solr/util/AuthToolTest.java rename to solr/core/src/test/org/apache/solr/util/cli/AuthToolTest.java index 718b4fc7f992..9e652092b7fb 100644 --- a/solr/core/src/test/org/apache/solr/util/AuthToolTest.java +++ b/solr/core/src/test/org/apache/solr/util/cli/AuthToolTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.solr.util; +package org.apache.solr.util.cli; import static org.apache.solr.util.SolrCLI.findTool; import static org.apache.solr.util.SolrCLI.parseCmdLine; @@ -80,8 +80,8 @@ public void testEnableAuth() throws Exception { } private int runTool(String[] args) throws Exception { - SolrCLI.Tool tool = findTool(args); - assertTrue(tool instanceof SolrCLI.AuthTool); + Tool tool = findTool(args); + assertTrue(tool instanceof AuthTool); CommandLine cli = parseCmdLine(tool.getName(), args, tool.getOptions()); return tool.runTool(cli); } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java index 9f904f11f8ca..93769d2362ec 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java @@ -41,6 +41,7 @@ import org.apache.solr.embedded.JettySolrRunner; import org.apache.solr.util.SolrCLI; import org.apache.solr.util.TimeOut; +import org.apache.solr.util.cli.ConfigSetUploadTool; import org.apache.zookeeper.KeeperException; import org.junit.BeforeClass; import org.slf4j.Logger; @@ -417,12 +418,9 @@ protected static void copyConfigUp( "-configsetsDir", configSetDir.toString(), }; - SolrCLI.ConfigSetUploadTool tool = new SolrCLI.ConfigSetUploadTool(); + ConfigSetUploadTool tool = new ConfigSetUploadTool(); - int res = - tool.runTool( - SolrCLI.processCommandLineArgs( - tool.getName(), SolrCLI.joinCommonAndToolOptions(tool.getOptions()), args)); + int res = tool.runTool(SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args)); assertEquals("Tool should have returned 0 for success, returned: " + res, res, 0); } }