From aadae0c0e03bff019eac819919430b7c6eb7aabd Mon Sep 17 00:00:00 2001 From: Gus Heck Date: Thu, 11 Jan 2018 15:09:25 -0500 Subject: [PATCH 1/3] SOLR-11722 patch #3 including docs, test SSL fix and precommit fixes --- .../src/java/org/apache/solr/api/ApiBag.java | 7 +- .../org/apache/solr/cloud/CreateAliasCmd.java | 265 ++++++++++++- .../OverseerCollectionMessageHandler.java | 1 + .../cloud/RoutedAliasCreateCollectionCmd.java | 40 +- .../handler/admin/BaseHandlerApiSupport.java | 110 +++--- .../handler/admin/CollectionsHandler.java | 172 +++++++-- .../TimeRoutedAliasUpdateProcessor.java | 13 +- .../ConcurrentCreateRoutedAliasTest.java | 291 +++++++++++++++ .../solr/cloud/CreateRoutedAliasTest.java | 351 ++++++++++++++++++ solr/solr-ref-guide/src/collections-api.adoc | 155 ++++++++ solr/solr-ref-guide/src/v2-api.adoc | 2 +- .../solrj/request/CollectionAdminRequest.java | 98 +++++ .../solrj/request/CollectionApiMapping.java | 12 + .../org/apache/solr/common/cloud/Aliases.java | 29 +- .../solr/common/cloud/ZkStateReader.java | 4 +- .../solr/common/params/CollectionParams.java | 1 + .../solr/common/util/JsonSchemaValidator.java | 21 +- .../apispec/collections.Commands.json | 64 +++- 18 files changed, 1501 insertions(+), 135 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/cloud/ConcurrentCreateRoutedAliasTest.java create mode 100644 solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java diff --git a/solr/core/src/java/org/apache/solr/api/ApiBag.java b/solr/core/src/java/org/apache/solr/api/ApiBag.java index 2caaeb93c4af..0c0d54b9a66b 100644 --- a/solr/core/src/java/org/apache/solr/api/ApiBag.java +++ b/solr/core/src/java/org/apache/solr/api/ApiBag.java @@ -305,7 +305,12 @@ public static List getCommandOperations(ContentStream stream, continue; } else { List errs = validator.validateJson(cmd.getCommandData()); - if (errs != null) for (String err : errs) cmd.addError(err); + if (errs != null){ + // otherwise swallowed in solrj tests, and just get "Error in command payload" in test log + // which is quite unhelpful. + log.error("Command errors for {}:{}", cmd.name, errs ); + for (String err : errs) cmd.addError(err); + } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java index e10d53e7c1aa..d9a04b92f2f4 100644 --- a/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java @@ -17,10 +17,24 @@ */ package org.apache.solr.cloud; +import java.lang.invoke.MethodHandles; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAccessor; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; +import java.util.TimeZone; import java.util.stream.Collectors; import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; @@ -30,13 +44,101 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; +import org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor; +import org.apache.solr.util.DateMathParser; +import org.apache.solr.util.TimeZoneUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import static java.time.format.DateTimeFormatter.ISO_INSTANT; +import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; +import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.common.params.CommonParams.TZ; +import static org.apache.solr.handler.admin.CollectionsHandler.ROUTED_ALIAS_COLLECTION_PROP_PFX; +import static org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER; public class CreateAliasCmd implements Cmd { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String ROUTING_TYPE = "router.name"; + public static final String ROUTING_FIELD = "router.field"; + public static final String ROUTING_INCREMENT = "router.interval"; + public static final String ROUTING_MAX_FUTURE = "router.max-future-ms"; + public static final String START = "start"; + // Collection constants should all reflect names in the v2 structured input for this command, not v1 + // names used for CREATE + public static final String CREATE_COLLECTION_CONFIG = "create-collection.config"; + public static final String CREATE_COLLECTION_ROUTER_NAME = "create-collection.router.name"; + public static final String CREATE_COLLECTION_ROUTER_FIELD = "create-collection.router.field"; + public static final String CREATE_COLLECTION_NUM_SHARDS = "create-collection.numShards"; + public static final String CREATE_COLLECTION_SHARDS = "create-collection.shards"; + public static final String CREATE_COLLECTION_REPLICATION_FACTOR = "create-collection.replicationFactor"; + public static final String CREATE_COLLECTION_NRT_REPLICAS = "create-collection.nrtReplicas"; + public static final String CREATE_COLLECTION_TLOG_REPLICAS = "create-collection.tlogReplicas"; + public static final String CREATE_COLLECTION_PULL_REPLICAS = "create-collection.pullReplicas"; + public static final String CREATE_COLLECTION_NODE_SET = "create-collection.nodeSet"; + public static final String CREATE_COLLECTION_SHUFFLE_NODES = "create-collection.shuffleNodes"; + public static final String CREATE_COLLECTION_MAX_SHARDS_PER_NODE = "create-collection.maxShardsPerNode"; + public static final String CREATE_COLLECTION_AUTO_ADD_REPLICAS = "create-collection.autoAddReplicas"; + public static final String CREATE_COLLECTION_RULE = "create-collection.rule"; + public static final String CREATE_COLLECTION_SNITCH = "create-collection.snitch"; + public static final String CREATE_COLLECTION_POLICY = "create-collection.policy"; + public static final String CREATE_COLLECTION_PROPERTIES = "create-collection.properties"; + + private final OverseerCollectionMessageHandler ocmh; + + /** + * Parameters required for creating a routed alias + */ + public static final List REQUIRED_ROUTING_PARAMS = Collections.unmodifiableList(Arrays.asList( + NAME, + START, + ROUTING_FIELD, + ROUTING_TYPE, + ROUTING_INCREMENT)); + + /** + * Optional parameters for creating a routed alias excluding parameters for collection creation. + */ + public static final List NONREQUIRED_ROUTING_PARAMS = Collections.unmodifiableList(Arrays.asList( + ROUTING_MAX_FUTURE, + TZ)); + /** + * Parameters used by routed Aliases to create collections. + */ + public static final List COLLECTION_ROUTING_PARAMS = Collections.unmodifiableList(Arrays.asList( + CREATE_COLLECTION_CONFIG, + CREATE_COLLECTION_ROUTER_FIELD, + CREATE_COLLECTION_ROUTER_NAME, + CREATE_COLLECTION_NUM_SHARDS, + CREATE_COLLECTION_SHARDS, + CREATE_COLLECTION_REPLICATION_FACTOR, + CREATE_COLLECTION_NRT_REPLICAS, + CREATE_COLLECTION_TLOG_REPLICAS, + CREATE_COLLECTION_PULL_REPLICAS, + CREATE_COLLECTION_NODE_SET, + CREATE_COLLECTION_SHUFFLE_NODES, + CREATE_COLLECTION_MAX_SHARDS_PER_NODE, + CREATE_COLLECTION_AUTO_ADD_REPLICAS, + CREATE_COLLECTION_RULE, + CREATE_COLLECTION_SNITCH, + CREATE_COLLECTION_POLICY, + CREATE_COLLECTION_PROPERTIES)); + + public static final List CREATE_ROUTED_ALIAS_PARAMS; + static { + List params = new ArrayList<>(); + params.addAll(REQUIRED_ROUTING_PARAMS); + params.addAll(NONREQUIRED_ROUTING_PARAMS); + params.addAll(COLLECTION_ROUTING_PARAMS); + CREATE_ROUTED_ALIAS_PARAMS = Collections.unmodifiableList(params); + } + public CreateAliasCmd(OverseerCollectionMessageHandler ocmh) { this.ocmh = ocmh; } @@ -45,13 +147,92 @@ public CreateAliasCmd(OverseerCollectionMessageHandler ocmh) { public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { final String aliasName = message.getStr(NAME); - final List canonicalCollectionList = parseCollectionsParameter(message.get("collections")); - final String canonicalCollectionsString = StrUtils.join(canonicalCollectionList, ','); - ZkStateReader zkStateReader = ocmh.zkStateReader; - validateAllCollectionsExistAndNoDups(canonicalCollectionList, zkStateReader); + ZkStateReader.AliasesManager holder = zkStateReader.aliasesHolder; + if (!anyRoutingParams(message)) { + final List canonicalCollectionList = parseCollectionsParameter(message.get("collections")); + final String canonicalCollectionsString = StrUtils.join(canonicalCollectionList, ','); + validateAllCollectionsExistAndNoDups(canonicalCollectionList, zkStateReader); + holder.applyModificationAndExportToZk(aliases -> aliases.cloneWithCollectionAlias(aliasName, canonicalCollectionsString)); + } else { + final String routedField = message.getStr(ROUTING_FIELD); + final String routingType = message.getStr(ROUTING_TYPE); + final String tz = message.getStr(TZ); + final String start = message.getStr(START); + final String increment = message.getStr(ROUTING_INCREMENT); + final String maxFutureMs = message.getStr(ROUTING_MAX_FUTURE); + + try { + if (0 > Long.valueOf(maxFutureMs)) { + throw new NumberFormatException("Negative value not allowed here"); + } + } catch (NumberFormatException e) { + throw new SolrException(BAD_REQUEST, ROUTING_MAX_FUTURE + " must be a valid long integer representing a number " + + "of milliseconds greater than or equal to zero"); + } - zkStateReader.aliasesHolder.applyModificationAndExportToZk(aliases -> aliases.cloneWithCollectionAlias(aliasName, canonicalCollectionsString)); + // Validate we got everything we need + if (routedField == null || routingType == null || start == null || increment == null) { + SolrException solrException = new SolrException(BAD_REQUEST, "If any of " + CREATE_ROUTED_ALIAS_PARAMS + + " are supplied, then all of " + REQUIRED_ROUTING_PARAMS + " must be present."); + log.error("Could not create routed alias",solrException); + throw solrException; + } + + if (!"time".equals(routingType)) { + SolrException solrException = new SolrException(BAD_REQUEST, "Only time based routing is supported at this time"); + log.error("Could not create routed alias",solrException); + throw solrException; + } + // Check for invalid timezone + if(tz != null && !TimeZoneUtils.KNOWN_TIMEZONE_IDS.contains(tz)) { + SolrException solrException = new SolrException(BAD_REQUEST, "Invalid timezone:" + tz); + log.error("Could not create routed alias",solrException); + throw solrException; + + } + TimeZone zone; + if (tz != null) { + zone = TimeZoneUtils.getTimeZone(tz); + } else { + zone = TimeZoneUtils.getTimeZone("UTC"); + } + DateTimeFormatter fmt = DATE_TIME_FORMATTER.withZone(zone.toZoneId()); + + // check that the increment is valid date math + String checkIncrement = ISO_INSTANT.format(Instant.now()) + increment; + DateMathParser.parseMath(new Date(), checkIncrement); // exception if invalid increment + + Instant startTime = validateStart(zone, fmt, start); + + // check field + String config = String.valueOf(message.getProperties().get(ROUTED_ALIAS_COLLECTION_PROP_PFX + COLL_CONF)); + if (!zkStateReader.getConfigManager().configExists(config)) { + SolrException solrException = new SolrException(BAD_REQUEST, "Could not find config '" + config + "'"); + log.error("Could not create routed alias",solrException); + throw solrException; + } + + // It's too much work to check the routed field against the schema, there seems to be no good way to get + // a copy of the schema aside from loading it directly from zookeeper based on the config name, but that + // also requires I load solrconfig.xml to check what the value for managedSchemaResourceName is too, (or + // discover that managed schema is not turned on and read schema.xml instead... and check for dynamic + // field patterns too. As much as it would be nice to validate all inputs it's not worth the effort. + + String initialCollectionName = TimeRoutedAliasUpdateProcessor + .formatCollectionNameFromInstant(aliasName, startTime, fmt ); + + NamedList createResults = new NamedList(); + ZkNodeProps collectionProps = selectByPrefix(ROUTED_ALIAS_COLLECTION_PROP_PFX, message) + .plus(NAME, initialCollectionName); + Map metadata = buildAliasMap(routedField, routingType, tz, increment, maxFutureMs, collectionProps); + RoutedAliasCreateCollectionCmd.createCollectionAndWait(state,createResults,aliasName,metadata,initialCollectionName,ocmh); + List collectionList = Collections.singletonList(initialCollectionName); + validateAllCollectionsExistAndNoDups(collectionList, zkStateReader); + holder.applyModificationAndExportToZk(aliases -> aliases + .cloneWithCollectionAlias(aliasName, initialCollectionName) + .cloneWithCollectionAliasMetadata(aliasName, metadata)); + } // Sleep a bit to allow ZooKeeper state propagation. // @@ -68,34 +249,100 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) Thread.sleep(100); } + private Map buildAliasMap(String routedField, String routingType, String tz, String increment, String maxFutureMs, ZkNodeProps collectionProps) { + Map properties = collectionProps.getProperties(); + Map cleanMap = properties.entrySet().stream() + .filter(stringObjectEntry -> + !"fromApi".equals(stringObjectEntry.getKey()) + && !"stateFormat".equals(stringObjectEntry.getKey()) + && !"name".equals(stringObjectEntry.getKey())) + .collect(Collectors.toMap((e) -> "collection-create." + e.getKey(), e -> String.valueOf(e.getValue()))); + cleanMap.put(ROUTING_FIELD, routedField); + cleanMap.put(ROUTING_TYPE, routingType); + cleanMap.put(ROUTING_INCREMENT, increment); + cleanMap.put(ROUTING_MAX_FUTURE, maxFutureMs); + cleanMap.put(TZ, tz); + return cleanMap; + } + + private Instant validateStart(TimeZone zone, DateTimeFormatter fmt, String start) { + // This is the normal/easy case, if we can get away with this great! + TemporalAccessor startTime = attemptTimeStampParsing(start, zone.toZoneId()); + if (startTime == null) { + // No luck, they gave us either date math, or garbage, so we have to do more work to figure out which and + // to make sure it's valid date math and that it doesn't encode any millisecond precision. + ZonedDateTime now = ZonedDateTime.now(zone.toZoneId()).truncatedTo(ChronoUnit.MILLIS); + try { + Date date = DateMathParser.parseMath(Date.from(now.toInstant()), start); + String reformatted = fmt.format(date.toInstant().truncatedTo(ChronoUnit.MILLIS)); + Date reparse = Date.from(Instant.from(DATE_TIME_FORMATTER.parse(reformatted))); + if (!reparse.equals(date)) { + throw new SolrException(BAD_REQUEST, + "Formatted time did not have the same milliseconds as original: " + date.getTime() + " vs. " + + reparse.getTime() + " This indicates that you used date math that includes milliseconds. " + + "(Hint: 'NOW' used without rounding always has this problem)" ); + } + return date.toInstant(); + } catch (SolrException e) { + throw new SolrException(BAD_REQUEST, + "Start Time for the first collection must be a timestamp of the format yyyy-MM-dd_HH_mm_ss, " + + "or a valid date math expression not containing specific milliseconds", e); + } + } + return Instant.from(startTime); + } + + private TemporalAccessor attemptTimeStampParsing(String start, ZoneId zone) { + try { + DATE_TIME_FORMATTER.withZone(zone); + return DATE_TIME_FORMATTER.parse(start); + } catch (DateTimeParseException e) { + return null; + } + } + + private boolean anyRoutingParams(ZkNodeProps message) { + + return message.containsKey(ROUTING_FIELD) || message.containsKey(ROUTING_TYPE) || message.containsKey(START) + || message.containsKey(ROUTING_INCREMENT) || message.containsKey(TZ); + } + private void validateAllCollectionsExistAndNoDups(List collectionList, ZkStateReader zkStateReader) { final String collectionStr = StrUtils.join(collectionList, ','); if (new HashSet<>(collectionList).size() != collectionList.size()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + throw new SolrException(BAD_REQUEST, String.format(Locale.ROOT, "Can't create collection alias for collections='%s', since it contains duplicates", collectionStr)); } ClusterState clusterState = zkStateReader.getClusterState(); Set aliasNames = zkStateReader.getAliases().getCollectionAliasListMap().keySet(); for (String collection : collectionList) { if (clusterState.getCollectionOrNull(collection) == null && !aliasNames.contains(collection)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + throw new SolrException(BAD_REQUEST, String.format(Locale.ROOT, "Can't create collection alias for collections='%s', '%s' is not an existing collection or alias", collectionStr, collection)); } } } - + /** * The v2 API directs that the 'collections' parameter be provided as a JSON array (e.g. ["a", "b"]). We also * maintain support for the legacy format, a comma-separated list (e.g. a,b). */ @SuppressWarnings("unchecked") private List parseCollectionsParameter(Object colls) { - if (colls == null) throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing collections param"); + if (colls == null) throw new SolrException(BAD_REQUEST, "missing collections param"); if (colls instanceof List) return (List) colls; return StrUtils.splitSmart(colls.toString(), ",", true).stream() .map(String::trim) .collect(Collectors.toList()); } + private ZkNodeProps selectByPrefix(String prefix, ZkNodeProps source) { + final ZkNodeProps[] subSet = {new ZkNodeProps()}; + source.getProperties().entrySet().stream() + .filter(entry -> entry.getKey().startsWith(prefix)) + .forEach(e -> subSet[0] = subSet[0].plus(e.getKey().substring(prefix.length()),e.getValue())); + return subSet[0]; + } + } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java index 426c87960743..bac2ef3626c7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java @@ -218,6 +218,7 @@ public OverseerCollectionMessageHandler(ZkStateReader zkStateReader, String myId .put(RELOAD, this::reloadCollection) .put(DELETE, new DeleteCollectionCmd(this)) .put(CREATEALIAS, new CreateAliasCmd(this)) + .put(CREATEROUTEDALIAS, new CreateAliasCmd(this)) .put(DELETEALIAS, new DeleteAliasCmd(this)) .put(ROUTEDALIAS_CREATECOLL, new RoutedAliasCreateCollectionCmd(this)) .put(OVERSEERSTATUS, new OverseerStatusCmd(this)) diff --git a/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java index 607588c4fded..0c5278b1a993 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java @@ -128,9 +128,30 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList resul //---- COMPUTE NEXT COLLECTION NAME final Instant nextCollTimestamp = TimeRoutedAliasUpdateProcessor.computeNextCollTimestamp(mostRecentCollTimestamp, intervalDateMath, intervalTimeZone); assert nextCollTimestamp.isAfter(mostRecentCollTimestamp); - final String createCollName = TimeRoutedAliasUpdateProcessor.formatCollectionNameFromInstant(aliasName, nextCollTimestamp); + final String createCollName = TimeRoutedAliasUpdateProcessor.formatCollectionNameFromInstant(aliasName, nextCollTimestamp, TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER); //---- CREATE THE COLLECTION + createCollectionAndWait(clusterState, results, aliasName, aliasMetadata, createCollName, ocmh); + + //TODO delete some of the oldest collection(s) ? + + //---- UPDATE THE ALIAS + aliasesHolder.applyModificationAndExportToZk(curAliases -> { + final List curTargetCollections = curAliases.getCollectionAliasListMap().get(aliasName); + if (curTargetCollections.contains(createCollName)) { + return curAliases; + } else { + List newTargetCollections = new ArrayList<>(curTargetCollections.size() + 1); + // prepend it on purpose (thus reverse sorted). Solr alias resolution defaults to the first collection in a list + newTargetCollections.add(createCollName); + newTargetCollections.addAll(curTargetCollections); + return curAliases.cloneWithCollectionAlias(aliasName, StrUtils.join(newTargetCollections, ',')); + } + }); + + } + + static void createCollectionAndWait(ClusterState clusterState, NamedList results, String aliasName, Map aliasMetadata, String createCollName, OverseerCollectionMessageHandler ocmh) throws Exception { // Map alias metadata starting with a prefix to a create-collection API request final ModifiableSolrParams createReqParams = new ModifiableSolrParams(); for (Map.Entry e : aliasMetadata.entrySet()) { @@ -155,23 +176,6 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList resul ocmh.commandMap.get(CollectionParams.CollectionAction.CREATE).call(clusterState, new ZkNodeProps(createMsgMap), results); CollectionsHandler.waitForActiveCollection(createCollName, null, ocmh.overseer.getCoreContainer(), new OverseerSolrResponse(results)); - - //TODO delete some of the oldest collection(s) ? - - //---- UPDATE THE ALIAS - aliasesHolder.applyModificationAndExportToZk(curAliases -> { - final List curTargetCollections = curAliases.getCollectionAliasListMap().get(aliasName); - if (curTargetCollections.contains(createCollName)) { - return curAliases; - } else { - List newTargetCollections = new ArrayList<>(curTargetCollections.size() + 1); - // prepend it on purpose (thus reverse sorted). Solr alias resolution defaults to the first collection in a list - newTargetCollections.add(createCollName); - newTargetCollections.addAll(curTargetCollections); - return curAliases.cloneWithCollectionAlias(aliasName, StrUtils.join(newTargetCollections, ',')); - } - }); - } private SolrException newAliasMustExistException(String aliasName) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java b/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java index 087c6f1d0a7c..89b63fcc2808 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java @@ -130,55 +130,7 @@ public void call(SolrQueryRequest req, SolrQueryResponse rsp) { } private static void wrapParams(final SolrQueryRequest req, final CommandOperation co, final ApiCommand cmd, final boolean useRequestParams) { - final Map pathValues = req.getPathTemplateValues(); - final Map map = co == null || !(co.getCommandData() instanceof Map) ? - Collections.singletonMap("", co.getCommandData()) : co.getDataMap(); - final SolrParams origParams = req.getParams(); - - req.setParams( - new SolrParams() { - @Override - public String get(String param) { - Object vals = getParams0(param); - if (vals == null) return null; - if (vals instanceof String) return (String) vals; - if (vals instanceof Boolean || vals instanceof Number) return String.valueOf(vals); - if (vals instanceof String[] && ((String[]) vals).length > 0) return ((String[]) vals)[0]; - return null; - } - - private Object getParams0(String param) { - param = cmd.meta().getParamSubstitute(param); - Object o = param.indexOf('.') > 0 ? - Utils.getObjectByPath(map, true, splitSmart(param, '.')) : - map.get(param); - if (o == null) o = pathValues.get(param); - if (o == null && useRequestParams) o = origParams.getParams(param); - if (o instanceof List) { - List l = (List) o; - return l.toArray(new String[l.size()]); - } - - return o; - } - - @Override - public String[] getParams(String param) { - Object vals = getParams0(param); - return vals == null || vals instanceof String[] ? - (String[]) vals : - new String[]{vals.toString()}; - } - - @Override - public Iterator getParameterNamesIterator() { - return cmd.meta().getParamNames(co).iterator(); - - } - - - }); - + req.setParams(new V2ToV1SolrParams(cmd.meta(), req.getPathTemplateValues(), useRequestParams, req.getParams(), co)); } protected abstract Collection getCommands(); @@ -188,8 +140,66 @@ public Iterator getParameterNamesIterator() { public interface ApiCommand { CommandMeta meta(); - void invoke(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) throws Exception; } + /** + * Wrapper for SolrParams that converts V2 params into V1 params. Only meant for internal use within solr's + * admin request handling code, other usages may not be supported (hence the package-private access). + */ + static class V2ToV1SolrParams extends SolrParams { + private final CommandMeta meta; + private final Map map; + private final Map pathValues; + private final boolean useRequestParams; + private final SolrParams origParams; + private final CommandOperation co; + + V2ToV1SolrParams(CommandMeta meta, Map pathValues, boolean useRequestParams, SolrParams origParams, CommandOperation co) { + this.meta = meta; + this.pathValues = pathValues; + this.useRequestParams = useRequestParams; + this.origParams = origParams; + this.co = co; + this.map = co == null || !(co.getCommandData() instanceof Map) ? + Collections.singletonMap("", co.getCommandData()) : co.getDataMap();; + } + + @Override + public String get(String param) { + Object vals = getParams0(param); + if (vals == null) return null; + if (vals instanceof String) return (String) vals; + if (vals instanceof Boolean || vals instanceof Number) return String.valueOf(vals); + if (vals instanceof String[] && ((String[]) vals).length > 0) return ((String[]) vals)[0]; + return null; + } + + private Object getParams0(String param) { + param = meta.getParamSubstitute(param); + Object o = param.indexOf('.') > 0 ? + Utils.getObjectByPath(map, true, splitSmart(param, '.')) : + map.get(param); + if (o == null) o = pathValues.get(param); + if (o == null && useRequestParams) o = origParams.getParams(param); + if (o instanceof List) { + List l = (List) o; + return l.toArray(new String[l.size()]); + } + return o; + } + + @Override + public String[] getParams(String param) { + Object vals = getParams0(param); + return vals == null || vals instanceof String[] ? + (String[]) vals : + new String[]{vals.toString()}; + } + + @Override + public Iterator getParameterNamesIterator() { + return meta.getParamNames(co).iterator(); + } + } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 74d47647eef9..04a7055f8e8e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -30,6 +30,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import com.google.common.collect.ImmutableSet; import org.apache.commons.io.IOUtils; @@ -38,6 +39,7 @@ import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; +import org.apache.solr.client.solrj.request.CollectionApiMapping; import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestSyncShard; import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; @@ -74,6 +76,7 @@ import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.Utils; @@ -99,6 +102,8 @@ import static org.apache.solr.client.solrj.response.RequestStatusState.NOT_FOUND; import static org.apache.solr.client.solrj.response.RequestStatusState.RUNNING; import static org.apache.solr.client.solrj.response.RequestStatusState.SUBMITTED; +import static org.apache.solr.cloud.CreateAliasCmd.NONREQUIRED_ROUTING_PARAMS; +import static org.apache.solr.cloud.CreateAliasCmd.REQUIRED_ROUTING_PARAMS; import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_PROP_PREFIX; @@ -146,6 +151,7 @@ public class CollectionsHandler extends RequestHandlerBase implements PermissionNameProvider { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String ROUTED_ALIAS_COLLECTION_PROP_PFX = "__collection-"; protected final CoreContainer coreContainer; private final CollectionHandlerApi v2Handler ; @@ -335,6 +341,14 @@ private boolean overseerCollectionQueueContains(String asyncId) throws KeeperExc return collectionQueue.containsTaskWithRequestId(ASYNC, asyncId); } + /** + * Copy prefixed params into a map. + * + * @param params The source of params from which copies should be made + * @param props The map into which param names and values should be copied as keys and values respectively + * @param prefix The prefix to select. + * @return the map supplied in the props parameter, modified to contain the prefixed params. + */ private static Map copyPropertiesWithPrefix(SolrParams params, Map props, String prefix) { Iterator iter = params.getParameterNamesIterator(); while (iter.hasNext()) { @@ -404,47 +418,8 @@ public enum CollectionOperation implements CollectionOp { * as well as specific replicas= options */ CREATE_OP(CREATE, (req, rsp, h) -> { - Map props = req.getParams().required().getAll(null, NAME); - props.put("fromApi", "true"); - req.getParams().getAll(props, - REPLICATION_FACTOR, - COLL_CONF, - NUM_SLICES, - MAX_SHARDS_PER_NODE, - CREATE_NODE_SET, - CREATE_NODE_SET_SHUFFLE, - SHARDS_PROP, - STATE_FORMAT, - AUTO_ADD_REPLICAS, - RULE, - SNITCH, - PULL_REPLICAS, - TLOG_REPLICAS, - NRT_REPLICAS, - POLICY, - WAIT_FOR_FINAL_STATE); - - if (props.get(STATE_FORMAT) == null) { - props.put(STATE_FORMAT, "2"); - } - addMapObject(props, RULE); - addMapObject(props, SNITCH); - verifyRuleParams(h.coreContainer, props); - final String collectionName = SolrIdentifierValidator.validateCollectionName((String) props.get(NAME)); - final String shardsParam = (String) props.get(SHARDS_PROP); - if (StringUtils.isNotEmpty(shardsParam)) { - verifyShardsParam(shardsParam); - } - if (CollectionAdminParams.SYSTEM_COLL.equals(collectionName)) { - //We must always create a .system collection with only a single shard - props.put(NUM_SLICES, 1); - props.remove(SHARDS_PROP); - createSysConfigSet(h.coreContainer); - - } - copyPropertiesWithPrefix(req.getParams(), props, COLL_PROP_PREFIX); - return copyPropertiesWithPrefix(req.getParams(), props, "router."); - + req.getParams().required().getAll(null, NAME); + return parseColletionCreationProps(h, req.getParams(), null); }), DELETE_OP(DELETE, (req, rsp, h) -> req.getParams().required().getAll(null, NAME)), @@ -476,6 +451,31 @@ public enum CollectionOperation implements CollectionOp { SolrIdentifierValidator.validateAliasName(req.getParams().get(NAME)); return req.getParams().required().getAll(null, NAME, "collections"); }), + CREATEROUTEDALIAS_OP(CREATEROUTEDALIAS, (req, rsp, h) -> { + String alias = req.getParams().get(NAME); + SolrIdentifierValidator.validateAliasName(alias); + Map params = req.getParams().required() + .getAll(null, REQUIRED_ROUTING_PARAMS.toArray(new String[REQUIRED_ROUTING_PARAMS.size()])); + req.getParams().getAll(params, NONREQUIRED_ROUTING_PARAMS); + // subset the params to reuse the collection creation/parsing code + ModifiableSolrParams collectionParams = extractPrefixedParams("create-collection.", req.getParams()); + if (collectionParams.get(NAME) != null) { + SolrException solrException = new SolrException(BAD_REQUEST, "routed aliases calculate names for their " + + "dependent collections, you cannot specify the name."); + log.error("Could not create routed alias",solrException); + throw solrException; + } + SolrParams v1Params = convertToV1WhenRequired(req, collectionParams); + + // We need to add this temporary name just to pass validation. + collectionParams.add(NAME, "TMP_name_TMP_name_TMP"); + params.putAll(parseColletionCreationProps(h, v1Params, ROUTED_ALIAS_COLLECTION_PROP_PFX)); + + // We will be giving the collection a real name based on the partition scheme later. + params.remove(ROUTED_ALIAS_COLLECTION_PROP_PFX + NAME); + params.remove("create-collection"); // don't include a stringified version now that we've parsed things out. + return params; + }), DELETEALIAS_OP(DELETEALIAS, (req, rsp, h) -> req.getParams().required().getAll(null, NAME)), /** @@ -931,6 +931,26 @@ public Map execute(SolrQueryRequest req, SolrQueryResponse rsp, "shard"); }), DELETENODE_OP(DELETENODE, (req, rsp, h) -> req.getParams().required().getAll(null, "node")); + + /** + * Extract only the params that have a given prefix. Any params with a different prefix are removed, and + * the params with the prefix have the prefix removed in the result + * + * @param prefix the prefix to select + * @param params the source of parameters that should be searched + * @return a SolrParams object containing only the params that had the prefix, with the prefix removed. + */ + private static ModifiableSolrParams extractPrefixedParams(String prefix, SolrParams params) { + ModifiableSolrParams result = new ModifiableSolrParams(); + for (Iterator i = params.getParameterNamesIterator(); i.hasNext();) { + String name = i.next(); + if (name.startsWith(prefix)) { + result.add(name.substring(prefix.length()), params.get(name)); + } + } + return result; + } + public final CollectionOp fun; CollectionAction action; long timeOut; @@ -962,6 +982,76 @@ public Map execute(SolrQueryRequest req, SolrQueryResponse rsp, } } + private static SolrParams convertToV1WhenRequired(SolrQueryRequest req, ModifiableSolrParams params) { + SolrParams v1Params = params; // (maybe...) + + // in the v2 world we get a data map based on the json request, from the CommandOperation associated + // with the request, so locate that if we can.. if we find it we have to translate the v2 request + // properties to v1 params, otherwise we're already good to go. + List cmds = req.getCommands(true); + if (cmds.size() > 1) { + // todo: not sure if this is the right thing to do here, but also not sure what to do if there is more than one... + throw new SolrException(BAD_REQUEST, "Only one command is allowed when creating a routed alias"); + } + CommandOperation c = cmds.size() == 0 ? null : cmds.get(0); + if (c != null) { // v2 api, do conversion to v1 + v1Params = new BaseHandlerApiSupport.V2ToV1SolrParams(CollectionApiMapping.Meta.CREATE_COLLECTION, + req.getPathTemplateValues(), true, params, + new CommandOperation("create", c.getDataMap().get("create-collection"))); + } + return v1Params; + } + + private static Map parseColletionCreationProps(CollectionsHandler h, SolrParams params, String prefix) + throws KeeperException, InterruptedException { + Map props = params.getAll(null, + NAME, + REPLICATION_FACTOR, + COLL_CONF, + NUM_SLICES, + MAX_SHARDS_PER_NODE, + CREATE_NODE_SET, + CREATE_NODE_SET_SHUFFLE, + SHARDS_PROP, + STATE_FORMAT, + AUTO_ADD_REPLICAS, + RULE, + SNITCH, + PULL_REPLICAS, + TLOG_REPLICAS, + NRT_REPLICAS, + POLICY, + WAIT_FOR_FINAL_STATE); + props.put("fromApi", "true"); + if (props.get(STATE_FORMAT) == null) { + props.put(STATE_FORMAT, "2"); + } + addMapObject(props, RULE); + addMapObject(props, SNITCH); + verifyRuleParams(h.coreContainer, props); + final String collectionName = SolrIdentifierValidator.validateCollectionName((String) props.get(NAME)); + final String shardsParam = (String) props.get(SHARDS_PROP); + if (StringUtils.isNotEmpty(shardsParam)) { + verifyShardsParam(shardsParam); + } + if (CollectionAdminParams.SYSTEM_COLL.equals(collectionName)) { + //We must always create a .system collection with only a single shard + props.put(NUM_SLICES, 1); + props.remove(SHARDS_PROP); + createSysConfigSet(h.coreContainer); + } + copyPropertiesWithPrefix(params, props, COLL_PROP_PREFIX); + Map result = copyPropertiesWithPrefix(params, props, "router."); + if (StringUtils.isNotBlank(prefix)) { + result = addPrefix(prefix, result); + } + return result; + } + + private static Map addPrefix(String prefix, Map aMap) { + return aMap.entrySet().stream().collect(Collectors.toMap( entry -> prefix + entry.getKey(), Map.Entry::getValue)); + } + private static void forceLeaderElection(SolrQueryRequest req, CollectionsHandler handler) { ClusterState clusterState = handler.coreContainer.getZkController().getClusterState(); String collectionName = req.getParams().required().get(COLLECTION_PROP); diff --git a/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java index bc242ba7bfd5..7e0cae133419 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java @@ -79,7 +79,9 @@ * requests to create new collections on-demand. * * Depends on this core having a special core property that points to the alias name that this collection is a part of. - * And further requires certain metadata on the Alias. + * And further requires certain metadata on the Alias. Collections pointed to by the alias must be named for the alias + * plus underscored ('_') and a time stamp of ISO_DATE plus optionally _HH_mm_ss. These collections should not be + * created by the user, but are created automatically by the time partitioning system. * * @since 7.2.0 */ @@ -99,6 +101,8 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor { .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) + // Setting a timezone here is fine as a default, but generally need to clone with user's timezone, so that + // truncation of milliseconds is consistent. .toFormatter(Locale.ROOT).withZone(ZoneOffset.UTC); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -252,6 +256,7 @@ public void processAdd(AddUpdateCommand cmd) throws IOException { /** Computes the timestamp of the next collection given the timestamp of the one before. */ public static Instant computeNextCollTimestamp(Instant fromTimestamp, String intervalDateMath, TimeZone intervalTimeZone) { //TODO overload DateMathParser.parseMath to take tz and "now" + // GH: I don't think that's necessary, you can set the TZ on now when you pass it in? final DateMathParser dateMathParser = new DateMathParser(intervalTimeZone); dateMathParser.setNow(Date.from(fromTimestamp)); final Instant nextCollTimestamp; @@ -398,14 +403,14 @@ static Instant parseInstantFromCollectionName(String aliasName, String collectio return DATE_TIME_FORMATTER.parse(dateTimePart, Instant::from); } - public static String formatCollectionNameFromInstant(String aliasName, Instant timestamp) { - String nextCollName = TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER.format(timestamp); + public static String formatCollectionNameFromInstant(String aliasName, Instant timestamp, DateTimeFormatter fmt) { + String nextCollName = fmt.format(timestamp); for (int i = 0; i < 3; i++) { // chop off seconds, minutes, hours if (nextCollName.endsWith("_00")) { nextCollName = nextCollName.substring(0, nextCollName.length()-3); } } - assert TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER.parse(nextCollName, Instant::from).equals(timestamp); + assert fmt.parse(nextCollName, Instant::from).equals(timestamp); return aliasName + "_" + nextCollName; } diff --git a/solr/core/src/test/org/apache/solr/cloud/ConcurrentCreateRoutedAliasTest.java b/solr/core/src/test/org/apache/solr/cloud/ConcurrentCreateRoutedAliasTest.java new file mode 100644 index 000000000000..9b8744de60b6 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/ConcurrentCreateRoutedAliasTest.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.common.util.IOUtils; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_MAX_SHARDS_PER_NODE; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_NUM_SHARDS; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_PULL_REPLICAS; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_REPLICATION_FACTOR; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_ROUTER_FIELD; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_ROUTER_NAME; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_SHARDS; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_TLOG_REPLICAS; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.ROUTING_FIELD; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.ROUTING_INCREMENT; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.ROUTING_MAX_FUTURE; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.ROUTING_TYPE; + +public class ConcurrentCreateRoutedAliasTest extends SolrTestCaseJ4 { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private MiniSolrCloudCluster solrCluster; + + // to avoid having to delete stuff... + volatile int num = 0; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + solrCluster = new MiniSolrCloudCluster(4, createTempDir(), buildJettyConfig("/solr")); + } + + @Override + @After + public void tearDown() throws Exception { + solrCluster.shutdown(); + super.tearDown(); + } + + @Test + public void testConcurrentCreateRoutedAliasMinimal() throws IOException, KeeperException.NoNodeException { + // this is the test where be blow out a bunch of create commands all out at once. + // other tests are more functionality based, and just use a single thread. + + // Failure of this test very occasionally due to overseer overload would not be worrisome (just bothersome). + // Any use case creating large numbers of time routed aliases concurrently would be an EXTREMELY odd + // if not fundamentally broken use case. This test method is just here to guard against any race + // conditions in the code that could crop up rarely in lower volume usage. + + // That said any failures involving about NPE's or missing parameters or oddities other than overwhelming + // the overseer queue with retry races emanating from this test should be investigated. Also if it fails + // frequently that needs to be investigated of course. + + + final AtomicReference failure = new AtomicReference<>(); + final int timeToRunSec = 30; + + // Note: this number of threads seems to work regularly with the up-tweaked number of retries (50) in + // org.apache.solr.common.cloud.ZkStateReader.AliasesManager.applyModificationAndExportToZk() + // with the original 5 retries this wouldn't reliably pass with 10 threads, but with 50 retries it seems + // to handle 50 threads about a dozen times without any failure (on a 32 thread processor) + // it also passed 3/3 at 150 threads and 2/3 with 250 threads on both 1 node and 4 nodes... + // the failure mode seems to be overseer tasks that are not found. I suspect this happens when enough + // threads get into retry races and the spam overwhelms the overseer. (that this can happen might imply + // an issue over there, but I'm not sure, since there is an intentional hard limit on the overseer queue + // and I haven't tried to count the retries up and figure out if the requests are actually exceeding that + // limit or not, but the speed of retries might indicate an effectively hot loop, but again, a separate issue. + + // The hope is that the level of concurrency supported by create routed alias and the code it uses is such + // that this test wouldn't spuriously fail more than once a year. If that's true users should never see + // an issue in the wild unless they are doing something we probably don't want to support anyway + + final CreateRoutedAliasThread[] threads = new CreateRoutedAliasThread[50]; + int numStart = num; + for (; num < threads.length + numStart; num++) { + final String aliasName = "testAlias" + num; + uploadConfig(configset("_default"), aliasName); + final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); + final SolrClient solrClient = getHttpSolrClient(baseUrl); + + + int i = num - numStart; + Map routingParams = getMinimalRoutingCommands(); + Map collectionParams = getMinimalCollectionParams(); + threads[i] = new CreateRoutedAliasThread("create-delete-search-" + i, aliasName, "NOW/HOUR", + "UTC", routingParams, collectionParams, timeToRunSec, solrClient, failure, false); + } + + startAll(threads); + joinAll(threads); + + assertNull("concurrent alias creation failed " + failure.get(), failure.get()); + } + + + @Test + public void testConcurrentCreateRoutedAliasComplex() { + final AtomicReference failure = new AtomicReference<>(); + final int timeToRunSec = 30; + + final CreateRoutedAliasThread[] threads = new CreateRoutedAliasThread[1]; + int numStart = num; + System.out.println("NUM ==> " +num); + for (; num < threads.length + numStart; num++) { + final String aliasName = "testAliasCplx" + num; + uploadConfig(configset("_default"), aliasName); + final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); + final SolrClient solrClient = getHttpSolrClient(baseUrl); + + int i = num - numStart; + Map routingParams = getMinimalRoutingCommands(); + Map collectionParams = getComplicatedCollectionParams(); + threads[i] = new CreateRoutedAliasThread("create-routed-alias-cplx-" + i, + aliasName, "2017-12-25_23_24_25","EST", routingParams, collectionParams, + timeToRunSec, solrClient, failure, true); + } + + startAll(threads); + joinAll(threads); + + assertNull("concurrent alias creation failed " + failure.get(), failure.get()); + } + + public Map getComplicatedCollectionParams() { + Map result = getMinimalCollectionParams(); + result.put(CREATE_COLLECTION_ROUTER_NAME, "implicit"); + result.put(CREATE_COLLECTION_ROUTER_FIELD, "implicit_s"); + result.put(CREATE_COLLECTION_SHARDS, "foo,bar"); + result.remove(CREATE_COLLECTION_NUM_SHARDS); + result.remove(CREATE_COLLECTION_REPLICATION_FACTOR); + result.put(CREATE_COLLECTION_REPLICATION_FACTOR,"2" ); + result.put(CREATE_COLLECTION_NUM_SHARDS,"2"); + result.put(CREATE_COLLECTION_MAX_SHARDS_PER_NODE, "4"); + result.put(CREATE_COLLECTION_PULL_REPLICAS, "2"); + result.put(CREATE_COLLECTION_TLOG_REPLICAS, "2"); + return result; + } + + static Map getMinimalCollectionParams() { + // needs to be V1 param name below + + HashMap cparams = new HashMap<>(); + cparams.put("create-collection.collection.configName", "_default"); + cparams.put(CREATE_COLLECTION_NUM_SHARDS,"1" ); + cparams.put(CREATE_COLLECTION_REPLICATION_FACTOR,"1" ); + return cparams; + } + + static Map getMinimalRoutingCommands() { + HashMap rparams = new HashMap<>(); + rparams.put(ROUTING_TYPE, "time"); + rparams.put(ROUTING_FIELD, "routedFoo_dt"); + rparams.put(ROUTING_INCREMENT, "+12HOUR"); + rparams.put(ROUTING_MAX_FUTURE, String.valueOf(1000 * 60 * 60)); + return rparams; + } + + + private void uploadConfig(Path configDir, String configName) { + try { + solrCluster.uploadConfigSet(configDir, configName); + } catch (IOException | KeeperException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private void joinAll(final CreateRoutedAliasThread[] threads) { + for (CreateRoutedAliasThread t : threads) { + try { + t.joinAndClose(); + } catch (InterruptedException e) { + Thread.interrupted(); + throw new RuntimeException(e); + } + } + } + + private void startAll(final Thread[] threads) { + for (Thread t : threads) { + t.start(); + } + } + + private static class CreateRoutedAliasThread extends Thread { + protected final String aliasName; + protected final String start; + protected final String tz; + protected final Map routingParams; + protected final Map collectionParams; + protected final long timeToRunSec; + protected final SolrClient solrClient; + protected final AtomicReference failure; + private final boolean v2; + + public CreateRoutedAliasThread( + String name, String aliasName, String start, String tz, Map routingParams, Map collectionParams, long timeToRunSec, SolrClient solrClient, + AtomicReference failure, boolean v2) { + super(name); + this.aliasName = aliasName; + this.start = start; + this.tz = tz; + this.routingParams = routingParams; + this.collectionParams = collectionParams; + this.timeToRunSec = timeToRunSec; + this.solrClient = solrClient; + this.failure = failure; + this.v2 = v2; + } + + @Override + public void run() { + doWork(); + } + + protected void doWork() { + createAlias(); + } + + protected void addFailure(Exception e) { + log.error("Add Failure", e); + synchronized (failure) { + if (failure.get() != null) { + failure.get().addSuppressed(e); + } else { + failure.set(e); + } + } + } + + private void createAlias() { + try { + CollectionAdminRequest.CreateRoutedAlias rq = CollectionAdminRequest + .createRoutedAlias(aliasName, start, "UTC", getMinimalRoutingCommands(), getMinimalCollectionParams()); + + final CollectionAdminResponse response = rq.process(solrClient); + if (response.getStatus() != 0) { + addFailure(new RuntimeException("failed to create collection " + aliasName)); + } + } catch (Exception e) { + addFailure(e); + } + + } + + + public void joinAndClose() throws InterruptedException { + try { + super.join(60000); + } finally { + IOUtils.closeQuietly(solrClient); + } + } + } + + +} diff --git a/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java b/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java new file mode 100644 index 000000000000..cabf1449cc20 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.Date; +import java.util.Locale; +import java.util.Map; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.lucene.util.IOUtils; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.common.cloud.Aliases; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor; +import org.apache.solr.util.DateMathParser; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Direct http tests of the CreateRoutedAlias functionality. + */ +public class CreateRoutedAliasTest extends SolrCloudTestCase { + + private CloudSolrClient solrClient; + private CloseableHttpClient httpclient; + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(2).configure(); + } + + @After + public void finish() throws Exception { + IOUtils.close(solrClient, httpclient); + } + + @Before + public void doBefore() throws Exception { + solrClient = getCloudSolrClient(cluster); + httpclient = HttpClients.createDefault(); + // delete aliases first to avoid problems such as: https://issues.apache.org/jira/browse/SOLR-11839 + ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader(); + zkStateReader.aliasesHolder.applyModificationAndExportToZk(aliases -> { + Aliases a = zkStateReader.getAliases(); + for (String alias : a.getCollectionAliasMap().keySet()) { + a = a.cloneWithCollectionAlias(alias,null); // remove + } + return a; + }); + for (String col : CollectionAdminRequest.listCollections(solrClient)) { + CollectionAdminRequest.deleteCollection(col).process(solrClient); + } + } + + @Test + public void testV2() throws Exception { + final String aliasName = "testAlias"; + cluster.uploadConfigSet(configset("_default"), aliasName); + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + HttpPost post = new HttpPost(baseUrl + "/____v2/c"); + post.setHeader("Content-Type", ContentType.APPLICATION_JSON.getMimeType()); + HttpEntity httpEntity = new InputStreamEntity(org.apache.commons.io.IOUtils.toInputStream("{\n" + + " \"create-routed-alias\" : {\n" + + " \"name\": \"testaliasV2\",\n" + + " \"router\" : {\n" + + " \"name\": \"time\",\n" + + " \"field\": \"evt_dt\",\n" + + " \"interval\":\"+2HOUR\",\n" + + " \"max-future-ms\":\"14400000\"\n" + + " },\n" + + " \"start\":\"NOW/DAY\",\n" + // small window for test failure once a day. + " \"create-collection\" : {\n" + + " \"router\": {\n" + + " \"name\":\"implicit\",\n" + + " \"field\":\"foo_s\"\n" + + " },\n" + + " \"shards\":\"foo,bar,baz\",\n" + + " \"config\":\"_default\",\n" + + " \"numShards\": 2,\n" + + " \"tlogReplicas\":1,\n" + + " \"pullReplicas\":1,\n" + + " \"maxShardsPerNode\":3,\n" + + " \"properties\" : {\n" + + " \"foobar\":\"bazbam\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}", "UTF-8"), org.apache.http.entity.ContentType.APPLICATION_JSON); + post.setEntity(httpEntity); + try (CloseableHttpResponse response = httpclient.execute(post)) { + assertEquals(200, response.getStatusLine().getStatusCode()); + } + Date date = DateMathParser.parseMath(new Date(), "NOW/DAY"); + String initialCollectionName = TimeRoutedAliasUpdateProcessor + .formatCollectionNameFromInstant("testaliasV2", date.toInstant(), + TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER); + HttpGet get = new HttpGet(baseUrl + "/____v2/c/"+ initialCollectionName); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(200, response.getStatusLine().getStatusCode()); + } + + Aliases aliases = cluster.getSolrClient().getZkStateReader().getAliases(); + Map collectionAliasMap = aliases.getCollectionAliasMap(); + String alias = collectionAliasMap.get("testaliasV2"); + assertNotNull(alias); + Map meta = aliases.getCollectionAliasMetadata("testaliasV2"); + assertNotNull(meta); + assertEquals("evt_dt",meta.get("router.field")); + assertEquals("foo_s",meta.get("collection-create.router.field")); + assertEquals("bazbam",meta.get("collection-create.property.foobar")); + } + + @Test + public void testV1() throws Exception { + + final String aliasName = "testAlias"; + cluster.uploadConfigSet(configset("_default"), aliasName); + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis + String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); + HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + + "&wt=xml" + + "&name=testalias" + + "&router.field=evt_dt" + + "&router.name=time" + + "&start=" + timestamp + + "&router.interval=%2B30MINUTE" + + "&router.max-future-ms=60000" + + "&create-collection.collection.configName=_default" + + "&create-collection.numShards=2"); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(200, response.getStatusLine().getStatusCode()); + } + String initialCollectionName = TimeRoutedAliasUpdateProcessor + .formatCollectionNameFromInstant("testalias", instant, + TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER); + get = new HttpGet(baseUrl + "/____v2/c/"+ initialCollectionName); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(200, response.getStatusLine().getStatusCode()); + } + + Aliases aliases = cluster.getSolrClient().getZkStateReader().getAliases(); + Map collectionAliasMap = aliases.getCollectionAliasMap(); + String alias = collectionAliasMap.get("testalias"); + assertNotNull(alias); + Map meta = aliases.getCollectionAliasMetadata("testalias"); + assertNotNull(meta); + assertEquals("evt_dt",meta.get("router.field")); + assertEquals("_default",meta.get("collection-create.collection.configName")); + assertEquals("2",meta.get("collection-create.numShards")); + assertEquals(null ,meta.get("start")); + } + + @Test + public void testAliasNameMustBeValid() throws Exception { + + final String aliasName = "testAlias"; + cluster.uploadConfigSet(configset("_default"), aliasName); + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis + String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); + + HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + + "&wt=json" + + "&name=735741!45" + // ! not allowed + "&router.field=evt_dt" + + "&router.name=time" + + "&start=" + timestamp + + "&router.interval=%2B30MINUTE" + + "&router.max-future-ms=60000" + + "&create-collection.collection.configName=_default" + + "&create-collection.numShards=2"); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(400, response.getStatusLine().getStatusCode()); + assertErrorStartsWith(response,"Invalid alias"); + + } + } + @Test + public void testRandomRouterNameFails() throws Exception { + + final String aliasName = "testAlias"; + cluster.uploadConfigSet(configset("_default"), aliasName); + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis + String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); + HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + + "&wt=json" + + "&name=testalias" + + "&router.field=evt_dt" + + "&router.name=tiafasme" + + "&start=" + timestamp + + "&router.interval=%2B30MINUTE" + + "&router.max-future-ms=60000" + + "&create-collection.collection.configName=_default" + + "&create-collection.numShards=2"); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(400, response.getStatusLine().getStatusCode()); + assertErrorStartsWith(response,"Only time based routing is supported"); + } + } + + @Test + public void testTimeStampWithMsFails() throws Exception { + final String aliasName = "testAlias"; + cluster.uploadConfigSet(configset("_default"), aliasName); + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS).plus(123,ChronoUnit.MILLIS); // mostly make sure no millis + String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); + HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + + "&wt=json" + + "&name=testalias" + + "&router.field=evt_dt" + + "&router.name=time" + + "&start=" + timestamp + + "&router.interval=%2B30MINUTE" + + "&router.max-future-ms=60000" + + "&create-collection.collection.configName=_default" + + "&create-collection.numShards=2"); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(400, response.getStatusLine().getStatusCode()); + assertErrorStartsWith(response, "Start Time for the first collection must be a timestamp of the format yyyy-MM-dd_HH_mm_ss"); + } + } + + private void assertErrorStartsWith(CloseableHttpResponse response, String prefix) throws IOException { + String entity = getStringEntity(response); + System.out.println(entity); + ObjectMapper mapper = new ObjectMapper(); + Map map = mapper.readValue(entity, new TypeReference>() {}); + Map exception = (Map) map.get("exception"); + if (exception == null) { + exception = (Map) map.get("error"); + } + String msg = (String) exception.get("msg"); + assertTrue(msg.toLowerCase(Locale.ROOT).startsWith(prefix.toLowerCase(Locale.ROOT))); + } + + private String getStringEntity(CloseableHttpResponse response) throws IOException { + ByteArrayOutputStream outstream = new ByteArrayOutputStream(); + response.getEntity().writeTo(outstream); + return new String(outstream.toByteArray(), StandardCharsets.UTF_8); + } + + @Test + public void testBadDateMathIntervalFails() throws Exception { + + final String aliasName = "testAlias"; + cluster.uploadConfigSet(configset("_default"), aliasName); + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis + String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); + HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + + "&wt=json" + + "&name=testalias" + + "&router.field=evt_dt" + + "&router.name=time" + + "&start=" + timestamp + + "&router.interval=%2B30MINUTEx" + + "&router.max-future-ms=60000" + + "&create-collection.collection.configName=_default" + + "&create-collection.numShards=2"); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(400, response.getStatusLine().getStatusCode()); + assertErrorStartsWith(response,"Invalid Date Math"); + } + } + @Test + public void testNegativeFutureFails() throws Exception { + + final String aliasName = "testAlias"; + cluster.uploadConfigSet(configset("_default"), aliasName); + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis + String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); + HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + + "&wt=json" + + "&name=testalias" + + "&router.field=evt_dt" + + "&router.name=time" + + "&start=" + timestamp + + "&router.interval=%2B30MINUTE" + + "&router.max-future-ms=-60000" + + "&create-collection.collection.configName=_default" + + "&create-collection.numShards=2"); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(400, response.getStatusLine().getStatusCode()); + assertErrorStartsWith(response, "router.max-future-ms must be a valid long integer"); + } + } + @Test + public void testUnParseableFutureFails() throws Exception { + + final String aliasName = "testAlias"; + cluster.uploadConfigSet(configset("_default"), aliasName); + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis + String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); + HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + + "&wt=json" + + "&name=testalias" + + "&router.field=evt_dt" + + "&router.name=time" + + "&start=" + timestamp + + "&router.interval=%2B30MINUTE" + + "&router.max-future-ms=SixtyThousandMiliseconds" + + "&create-collection.collection.configName=_default" + + "&create-collection.numShards=2"); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(400, response.getStatusLine().getStatusCode()); + assertErrorStartsWith(response, "router.max-future-ms must be a valid long integer"); + } + } + + // not testing collection parameters, those should inherit error checking from the collection creation code. +} diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc index a0c8038951e4..e8c1eb7e7f90 100644 --- a/solr/solr-ref-guide/src/collections-api.adoc +++ b/solr/solr-ref-guide/src/collections-api.adoc @@ -518,6 +518,161 @@ http://localhost:8983/solr/admin/collections?action=CREATEALIAS&name=testalias&c ---- +[[createroutedalias]] +== CREATEROUTEDALIAS: Create an alias that partitions data + +The CREATEROUTEDALIAS will create a special type of alias that automates the partitioning of data across a series of +collections. This feature allows for indefinite indexing of data without degradation of performance otherwise +experienced due to the continuous growth of the index. As new data arrives, a field on the document is inspected and +the document is then assigned to a partition. Physically these partitions are modeled as separate collections, +which can be queried independently or via the alias created by this command. These collections are created +automatically on the fly as new data arrives based on the parameters supplied in this command. + +*NOTE* Presently only partitioning of time based data is available, though other schemes may become available in +the future. +[source,text] +---- +localhost:8983/solr/admin/collections?action=CREATEROUTEDALIAS&name=timedata&start=NOW/DAY&router.field=evt_dt&router.name=time&router.interval=%2B1DAY&router.max-future-ms=3600000&create-collection.collection.configName=myConfig&create-collection.numShards=2 +---- + +If run on Jan 15, 2018 The above will create an alias named timedata, that contains collections with names such as +`timedata` and an initial collection named `timedata_2018_01_15`. Updates sent to this alias with a (required) value +in `evt_dt` that is before or after 2018-01-15 will be rejected, until the last 60 minutes of 2018-01-15. After +2018-01-15T23:00:00 documents for either 2018-01-15 or 2018-01-16 will be accepted. As soon as the system receives a +document for an allowable time window for which there is no collection it will automatically create the next required +collection (and potentially any intervening collections if router.interval is smaller than router.max-future-ms). Both +the initial collection and any subsequent collections will be created using the specified configset. All Collection +creation parameters other than `name` are allowed, prefixed by `collection-create.` + +This means that one could (for example) partition their collections by day, and within each daily collection route +the data to shards based on customer id. Such shards can be of any type (NRT, PULL or TLOG), and rule based replica +placement strategies may also be used. The values supplied in this command for collection creation will be retained +in alias metadata, and can be verified by inspecting aliases.json in zookeeper. + +=== CREATEROUTEDALIAS Parameters + +`name`:: +The alias name to be created. This parameter is required, and also functions as a prefix for the names of the +dependent collections that will be created. It must therefore adhere to normal requirements for alias and collection +naming. + +`start`:: +The minimum allowable value for router.field. This field may contain date math expressions but MUST NOT resolve to a +time that includes milliseconds other than 0. Particularly, this means `NOW` will fail 999 times out of 1000, though +`NOW/SECOND`, `NOW/MINUTE`, etc. will work just fine. A pure absolute time can also be specified in the format used +for naming collections `yyyy-MM-dd[_HH[_mm[_ss]]]` It is also important to pay attention to URL encoding to avoid +loosing `+` symbols in your date math. This field is required. + +`TZ`:: +The timezone to be used when evaluating eligibility for collections. If GMT-4 is supplied for this value then a document +dated 2018-01-14T21:00:00:01.2345Z would be stored in the myAlias_2018-01-15_01 collection (assumming an interval of ++1HOUR). The default timezone is UTC. + +`router.field`:: +The field to inspect to determine which partition (e.g. collection) an incoming document should be routed to. This +field is required + +`router.name`:: +The type of routing to use, presently only `time` is valid. This field is required. + +`router.interval`:: +A fragment of a date math expression that will be appended to a timestamp to determine the next collection in the series. +Any date math expression that can be evaluated if appended to a timestamp of the form 2018-01-15T16:17:18 will +work here. This field is required. + +`router.max-future-ms`:: +The maximum distance beyond the current (latest) partition for which data may be sent to this alias without error. +This field is required. + +`create-collection.*`:: +The * can be replaced with any parameter from the <> command except `name`. All other fields +are identical in requirements and naming. + +`async`:: +Request ID to track this action which will be <>. + +=== CREATEROUTEDALIAS Response + +The output will simply be a responseHeader with details of the time it took to process the request. To confirm the +creation of the alias and the values of the associated metadata, you can look in the Solr Admin UI, under the Cloud +section and find the `aliases.json` file. The initial collection should also be visible in various parts +of the admin UI. + +=== Examples using CREATEROUTEDALIAS + +Create an alias named "myTimeData" for data begining on `2018-01-15` in the UTC time zone and partitioning daily +based on the `evt_dt` field in the incomming documents. Data more than an hour beyond the latest (most recent) +partiton is to be rejected and collections are created using a config set named myConfig and + + +*Input* + +[source,text] +---- +localhost:8983/solr/admin/collections?action=CREATEROUTEDALIAS&name=myTimeData&start=NOW/DAY&router.field=evt_dt&router.name=time&router.interval=%2B1DAY&router.max-future-ms=3600000&create-collection.collection.configName=myConfig&create-collection.numShards=2 +---- + +*Output* + +[source,xml] +---- + + + 0 + 1234 + + +---- + +A somewhat contrived example demonstrating the <> usage and additional collection creation options. +Notice that the collection creation fields follow the v2 api naming convention, not the v1 naming conventions. + +*Input* + +[source,json] +---- +POST /api/c +{ + "create-routed-alias" : { + "name": "somethingTemporalThisWayComes", + "router" : { + "name": "time", + "field": "evt_dt", + "interval":"+2HOUR", + "max-future-ms":"14400000" + }, + "start":"NOW/MINUTE", + "create-collection" : { + "config":"_default", + "router": { + "name":"implicit", + "field":"foo_s" + }, + "shards":"foo,bar,baz", + "numShards": 3, + "tlogReplicas":1, + "pullReplicas":1, + "maxShardsPerNode":2, + "properties" : { + "foobar":"bazbam" + } + } + } +} +---- + +*Output* + +[source,xml] +---- +{ + "responseHeader": { + "status": 0, + "QTime": 1234 + } +} +---- + [[listaliases]] == LISTALIASES: List of all aliases in the cluster diff --git a/solr/solr-ref-guide/src/v2-api.adoc b/solr/solr-ref-guide/src/v2-api.adoc index c7ecfee3f33e..e2da0bfaff77 100644 --- a/solr/solr-ref-guide/src/v2-api.adoc +++ b/solr/solr-ref-guide/src/v2-api.adoc @@ -16,7 +16,7 @@ // specific language governing permissions and limitations // under the License. - +[[top-v2-api]] The v2 API is a modernized self-documenting API interface covering most current Solr APIs. It is anticipated that once the v2 API reaches full coverage, and Solr-internal API usages like SolrJ and the Admin UI have been converted from the old API to the v2 API, the old API will eventually be retired. For now the two API styles will coexist, and all the old APIs will continue to work without any change. You can disable all v2 API endpoints by starting your servers with this system property: `-Ddisable.v2.api=true`. diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index edc5a8be1cef..9fdc6d3f7714 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -41,6 +41,7 @@ import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.common.params.CommonAdminParams; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ShardParams; @@ -1356,6 +1357,103 @@ public SolrParams getParams() { } + /** + * Returns a SolrRequest to create a routed alias. Only time based routing is supported presently, + * For time based routing, the start is a timestamp or date math. + * + * @param aliasName the name of the alias to create. + * @param start the the start of the routing. + * @param tz the timezone on which to base date math (for time based routing) + * @param routingParams all router.* parameters, including the 'router.' prefix + * @param collectionParams all create-collection.* parameters including the 'create-collection.' prefix + * + */ + public static CreateRoutedAlias createRoutedAlias(String aliasName, String start, String tz, + Map routingParams, + Map collectionParams) { + return new CreateRoutedAlias(aliasName,start,tz,routingParams,collectionParams); + } + + public static class CreateRoutedAlias extends AsyncCollectionAdminRequest { + // TODO: This and other commands in this file seem to need to share some sort of constants class with core + // to allow this stuff not to be duplicated. (this is pasted from CreateAliasCmd.java), however I think + // a comprehensive cleanup of this for all the requests in this class should be done as a separate ticket. + + + public static final String ROUTING_TYPE = "router.name"; + public static final String ROUTING_FIELD = "router.field"; + public static final String ROUTING_INCREMENT = "router.interval"; + public static final String ROUTING_MAX_FUTURE = "router.max-future-ms"; + public static final String START = "start"; + public static final String CREATE_COLLECTION_CONFIG = "create-collection.config"; + public static final String CREATE_COLLECTION_ROUTER_NAME = "create-collection.router.name"; + public static final String CREATE_COLLECTION_ROUTER_FIELD = "create-collection.router.field"; + public static final String CREATE_COLLECTION_NUM_SHARDS = "create-collection.numShards"; + public static final String CREATE_COLLECTION_SHARDS = "create-collection.shards"; + public static final String CREATE_COLLECTION_REPLICATION_FACTOR = "create-collection.replicationFactor"; + public static final String CREATE_COLLECTION_NRT_REPLICAS = "create-collection.nrtReplicas"; + public static final String CREATE_COLLECTION_TLOG_REPLICAS = "create-collection.tlogReplicas"; + public static final String CREATE_COLLECTION_PULL_REPLICAS = "create-collection.pullReplicas"; + public static final String CREATE_COLLECTION_NODE_SET = "create-collection.nodeSet"; + public static final String CREATE_COLLECTION_SHUFFLE_NODES = "create-collection.shuffleNodes"; + public static final String CREATE_COLLECTION_MAX_SHARDS_PER_NODE = "create-collection.maxShardsPerNode"; + public static final String CREATE_COLLECTION_AUTO_ADD_REPLICAS = "create-collection.autoAddReplicas"; + public static final String CREATE_COLLECTION_RULE = "create-collection.rule"; + public static final String CREATE_COLLECTION_SNITCH = "create-collection.snitch"; + public static final String CREATE_COLLECTION_POLICY = "create-collection.policy"; + public static final String CREATE_COLLECTION_PROPERTIES = "create-collection.properties"; + + private final String aliasName; + private final String start; + private final String tz; + private final Map routingParams; + private final Map collectionParams; + + + public CreateRoutedAlias(String aliasName, String start, String tz, + Map routingParams, + Map collectionParams) { + super(CollectionAction.CREATEROUTEDALIAS); + this.aliasName = aliasName; + this.start = start; + this.tz = tz; + this.routingParams = routingParams; + this.collectionParams = collectionParams; + } + + @Override + public SolrParams getParams() { + ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); + params.add(CommonParams.NAME, aliasName); + params.add(START, start); + params.add(CommonParams.TZ, tz ); + // these need to be V1 params, not V2 + params.add(ROUTING_FIELD, routingParams.get(ROUTING_FIELD)); + params.add(ROUTING_INCREMENT, routingParams.get(ROUTING_INCREMENT)); + params.add(ROUTING_MAX_FUTURE, routingParams.get(ROUTING_MAX_FUTURE)); + params.add(ROUTING_TYPE, routingParams.get(ROUTING_TYPE)); + params.add("create-collection.collection.configName", collectionParams.get("create-collection.collection.configName")); + params.add(CREATE_COLLECTION_ROUTER_NAME, collectionParams.get(CREATE_COLLECTION_ROUTER_NAME)); + params.add(CREATE_COLLECTION_ROUTER_FIELD, collectionParams.get(CREATE_COLLECTION_ROUTER_FIELD)); + params.add(CREATE_COLLECTION_AUTO_ADD_REPLICAS, collectionParams.get(CREATE_COLLECTION_AUTO_ADD_REPLICAS)); + params.add(CREATE_COLLECTION_MAX_SHARDS_PER_NODE, collectionParams.get(CREATE_COLLECTION_MAX_SHARDS_PER_NODE)); + params.add(CREATE_COLLECTION_NODE_SET, collectionParams.get(CREATE_COLLECTION_NODE_SET)); + params.add(CREATE_COLLECTION_NRT_REPLICAS, collectionParams.get(CREATE_COLLECTION_NRT_REPLICAS)); + params.add(CREATE_COLLECTION_NUM_SHARDS, collectionParams.get(CREATE_COLLECTION_NUM_SHARDS)); + params.add(CREATE_COLLECTION_POLICY, collectionParams.get(CREATE_COLLECTION_POLICY)); + params.add(CREATE_COLLECTION_PROPERTIES, collectionParams.get(CREATE_COLLECTION_PROPERTIES)); + params.add(CREATE_COLLECTION_PULL_REPLICAS, collectionParams.get(CREATE_COLLECTION_PULL_REPLICAS)); + params.add(CREATE_COLLECTION_REPLICATION_FACTOR, collectionParams.get(CREATE_COLLECTION_REPLICATION_FACTOR)); + params.add(CREATE_COLLECTION_RULE, collectionParams.get(CREATE_COLLECTION_RULE)); + params.add(CREATE_COLLECTION_SHARDS, collectionParams.get(CREATE_COLLECTION_SHARDS)); + params.add(CREATE_COLLECTION_SNITCH, collectionParams.get(CREATE_COLLECTION_SNITCH)); + params.add(CREATE_COLLECTION_SHUFFLE_NODES, collectionParams.get(CREATE_COLLECTION_SHUFFLE_NODES)); + params.add(CREATE_COLLECTION_TLOG_REPLICAS, collectionParams.get(CREATE_COLLECTION_TLOG_REPLICAS)); + return params; + } + + } + /** * Returns a SolrRequest to delete an alias */ diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java index 701a45b1326a..46ce712557f0 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java @@ -34,6 +34,9 @@ import static org.apache.solr.client.solrj.SolrRequest.METHOD.DELETE; import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET; import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_CONFIG; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_NODE_SET; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_SHUFFLE_NODES; import static org.apache.solr.client.solrj.request.CollectionApiMapping.ConfigSetEndPoint.CONFIG_COMMANDS; import static org.apache.solr.client.solrj.request.CollectionApiMapping.ConfigSetEndPoint.CONFIG_DEL; import static org.apache.solr.client.solrj.request.CollectionApiMapping.ConfigSetEndPoint.LIST_CONFIG; @@ -113,6 +116,15 @@ public enum Meta implements CommandMeta { CREATEALIAS, "create-alias", null), + CREATE_ROUTED_ALIAS(COLLECTIONS_COMMANDS, + POST, + CREATEROUTEDALIAS, + "create-routed-alias", + Utils.makeMap( + "create-collection.collection.configName", CREATE_COLLECTION_CONFIG, + "createNodeSet",CREATE_COLLECTION_NODE_SET, + "create-collection.createNodeSet.shuffle", CREATE_COLLECTION_SHUFFLE_NODES + )), DELETE_ALIAS(COLLECTIONS_COMMANDS, POST, diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java b/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java index d77b544c985e..0aa3c7ee3976 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java @@ -239,18 +239,35 @@ public Aliases cloneWithCollectionAlias(String alias, String collections) { * @return An immutable copy of the aliases with the new metadata. */ public Aliases cloneWithCollectionAliasMetadata(String alias, String metadataKey, String metadataValue){ + return cloneWithCollectionAliasMetadata(alias, Collections.singletonMap(metadataKey,metadataValue)); + } + + /** + * Set the values for some metadata keys on a collection alias. This is done by creating a new Aliases instance + * with the same data as the current one but with a modification based on the parameters. + *

+ * Note that the state in zookeeper is unaffected by this method and the change must still be persisted via + * {@link ZkStateReader.AliasesManager#applyModificationAndExportToZk(UnaryOperator)} + * + * @param alias the alias to update + * @param metadata the metadata to add/replace, null values in the map will remove the key. + * @return An immutable copy of the aliases with the new metadata. + */ + public Aliases cloneWithCollectionAliasMetadata(String alias, Map metadata){ if (!collectionAliases.containsKey(alias)) { throw new IllegalArgumentException(alias + " is not a valid alias"); } - if (metadataKey == null) { - throw new IllegalArgumentException("Null is not a valid metadata key"); + if (metadata == null) { + throw new IllegalArgumentException("Null is not a valid metadata map"); } Map> newColMetadata = new LinkedHashMap<>(this.collectionAliasMetadata);//clone to modify Map newMetaMap = new LinkedHashMap<>(newColMetadata.getOrDefault(alias, Collections.emptyMap())); - if (metadataValue != null) { - newMetaMap.put(metadataKey, metadataValue); - } else { - newMetaMap.remove(metadataKey); + for (Map.Entry metaEntry : metadata.entrySet()) { + if (metaEntry.getValue() != null) { + newMetaMap.put(metaEntry.getKey(), metaEntry.getValue()); + } else { + newMetaMap.remove(metaEntry.getKey()); + } } newColMetadata.put(alias, Collections.unmodifiableMap(newMetaMap)); return new Aliases(collectionAliases, newColMetadata, zNodeVersion); diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index 8ab7ecbca5b0..f11339ce533d 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -1458,7 +1458,9 @@ public Aliases getAliases() { */ public void applyModificationAndExportToZk(UnaryOperator op) { final long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); - int triesLeft = 5; + // up-tweaked to better handle ConcurrentCreateRoutedAliasTest.testConcurrentCreateRoutedAliasMinimal() + // This may be too aggressive but the impact on that test should be validated before changing this. + int triesLeft = 50; while (triesLeft > 0) { triesLeft--; // we could synchronize on "this" but there doesn't seem to be a point; we have a retry loop. diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java index 9d5fc36bb671..9f6a2544c988 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java @@ -76,6 +76,7 @@ enum CollectionAction { RELOAD(true, LockLevel.COLLECTION), SYNCSHARD(true, LockLevel.SHARD), CREATEALIAS(true, LockLevel.COLLECTION), + CREATEROUTEDALIAS(true, LockLevel.COLLECTION), DELETEALIAS(true, LockLevel.COLLECTION), LISTALIASES(false, LockLevel.NONE), ROUTEDALIAS_CREATECOLL(true, LockLevel.COLLECTION), diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java b/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java index e5a1d44770b5..9f9231417345 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java @@ -18,6 +18,7 @@ package org.apache.solr.common.util; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -226,12 +227,26 @@ class RequiredValidator extends Validator> { @Override boolean validate(Object o, List errs) { + return validate(o,errs,requiredProps); + } + + boolean validate( Object o, List errs, Set requiredProps) { if (o instanceof Map) { Set fnames = ((Map) o).keySet(); for (String requiredProp : requiredProps) { - if (!fnames.contains(requiredProp)) { - errs.add("Missing required attribute '" + requiredProp + "' in object " + Utils.toJSONString(o)); - return false; + if (requiredProp.contains(".")) { + if (requiredProp.endsWith(".")) { + errs.add("Illegal required attribute name (ends with '.'" + requiredProp + ") This is a bug."); + } + String subprop = requiredProp.substring(requiredProp.indexOf(".") + 1); + if (!validate(((Map)o).get(requiredProp), errs, Collections.singleton(subprop))) { + return false; + } + } else { + if (!fnames.contains(requiredProp)) { + errs.add("Missing required attribute '" + requiredProp + "' in object " + Utils.toJSONString(o)); + return false; + } } } return true; diff --git a/solr/solrj/src/resources/apispec/collections.Commands.json b/solr/solrj/src/resources/apispec/collections.Commands.json index 294b1633bbed..e017807801ec 100644 --- a/solr/solrj/src/resources/apispec/collections.Commands.json +++ b/solr/solrj/src/resources/apispec/collections.Commands.json @@ -1,3 +1,4 @@ + { "documentation": "https://lucene.apache.org/solr/guide/collections-api.html#create", "description": "Create collections and collection aliases, backup or restore collections, and delete collections and aliases.", @@ -126,6 +127,7 @@ "name" ] }, + "create-alias": { "documentation": "https://lucene.apache.org/solr/guide/collections-api.html#createalias", "description": "Allows one or more collections to be known by another name. If this command is used on an existing alias, the existing alias will be replaced with the new collection details.", @@ -152,6 +154,64 @@ "collections" ] }, + "create-routed-alias": { + "type": "object", + "documentation": "https://lucene.apache.org/solr/guide/collections-api.html#createalias", + "description": "Allows one or more collections to be known by another name. If this command is used on an existing alias, the existing alias will be replaced with the new collection details.", + "properties": { + "name": { + "type": "string", + "description": "The alias name to be created." + }, + "router" : { + "type":"object", + "documentation": "https://lucene.apache.org/solr/guide/collections-api.html#createalias", + "description":"routing specific attributes", + "properties" : { + "name" : { + "type" : "string", + "description": "The type of routing to perform. Currently only 'time' is supported. This property is mutually exclusive with 'collections', and required with any route-* property. Collections created will be named for the alias and a time stamp representing the first value allowed for the partition represented by each collection." + }, + "field" : { + "type": "string", + "description": "The field in incoming documents that is consulted to decide which collection the document should be routed to." + }, + "interval" : { + "type": "string", + "description": "A specification of the width of the interval for each partition collection. For time based routing this should be a date math expression fragment starting with the + character. When appended to a standard iso date the result should be a valid date math expression, with a resolution no finer than 1 second." + }, + "max-future-ms": { + "type": "integer", + "description":"How far into the future to accept documents. Documents with a value in router.field that is greater than now() + max-future-ms will be rejected" + } + } + }, + "start": { + "type": "string", + "description": "The earliest/lowest value for routeField that may be indexed into this alias. Documents with values less than this will return an error. For time based routing this may be a date math expression. Both the timestamp and the date math must not specify millisecond precision. Single seconds are the minimum supported time unit" + }, + "TZ": { + "type": "string", + "description": "Optional timezone for time based routing. Overrides the TimeZone used when rounding Dates in DateMath expressions If omitted, UTC timezone will be used" + }, + "create-collection": { + "type": "object", + "documentation": "https://lucene.apache.org/solr/guide/collections-api.html#create", + "description": "The settings to use to create a collection for each new time partition. Most options from the collection create command are available with several exceptions. Exceptions include 'name' because collection names for routed aliases are controlled by the routing mechanisms. 'async' and 'waitForFinalState' are also unavailable as they are determined by the top level command.", + "additionalProperties": true + }, + "async": { + "type": "string", + "description": "Defines a request ID that can be used to track this action after it's submitted. The action will be processed asynchronously." + } + }, + "required": [ + "name", + "router", + "start", + "create-collection" + ] + }, "delete-alias": { "documentation": "https://lucene.apache.org/solr/guide/collections-api.html#deletealias", "description": "Deletes a collection alias", @@ -166,7 +226,9 @@ "description": "Defines a request ID that can be used to track this action after it's submitted. The action will be processed asynchronously." } }, - "required":["name"] + "required": [ + "name" + ] }, "backup-collection": { "documentation": "https://lucene.apache.org/solr/guide/collections-api.html#backup", From 677f3967af03a45c2ca2f9792926253ccbf0f023 Mon Sep 17 00:00:00 2001 From: Gus Heck Date: Fri, 12 Jan 2018 02:29:28 -0500 Subject: [PATCH 2/3] SOLR-11722 update PR with fixes per comments in review. Docs still need to be re-adjusted and some further date wrangling may be needed. --- .../org/apache/solr/cloud/CreateAliasCmd.java | 110 ++++++++---------- .../handler/admin/CollectionsHandler.java | 10 +- .../TimeRoutedAliasUpdateProcessor.java | 3 - .../solr/cloud/CreateRoutedAliasTest.java | 93 ++++++++++++--- .../TimeRoutedAliasUpdateProcessorTest.java | 10 ++ 5 files changed, 140 insertions(+), 86 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java index d9a04b92f2f4..e191071cb069 100644 --- a/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java @@ -18,10 +18,9 @@ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; +import java.text.ParseException; import java.time.Instant; import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalAccessor; @@ -42,6 +41,7 @@ import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; import org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor; @@ -50,10 +50,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static java.time.format.DateTimeFormatter.ISO_INSTANT; import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; -import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CommonParams.TZ; import static org.apache.solr.handler.admin.CollectionsHandler.ROUTED_ALIAS_COLLECTION_PROP_PFX; import static org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER; @@ -87,6 +85,14 @@ public class CreateAliasCmd implements Cmd { public static final String CREATE_COLLECTION_SNITCH = "create-collection.snitch"; public static final String CREATE_COLLECTION_POLICY = "create-collection.policy"; public static final String CREATE_COLLECTION_PROPERTIES = "create-collection.properties"; + public static final String FROM_API = "fromApi"; + public static final String STATE_FORMAT = "stateFormat"; + public static final String NAME = "name"; + + // These are parameters that get added by the collection creation command parsing that we + // will want to not store in alias metadata. + public static final Set COLLECTION_CREATION_CRUFT = // wanted: Set.of() from jdk9 + Collections.unmodifiableSet(new HashSet<>(Arrays.asList(FROM_API, STATE_FORMAT, NAME))); private final OverseerCollectionMessageHandler ocmh; @@ -96,7 +102,7 @@ public class CreateAliasCmd implements Cmd { * Parameters required for creating a routed alias */ public static final List REQUIRED_ROUTING_PARAMS = Collections.unmodifiableList(Arrays.asList( - NAME, + CommonParams.NAME, START, ROUTING_FIELD, ROUTING_TYPE, @@ -146,7 +152,7 @@ public CreateAliasCmd(OverseerCollectionMessageHandler ocmh) { @Override public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception { - final String aliasName = message.getStr(NAME); + final String aliasName = message.getStr(CommonParams.NAME); ZkStateReader zkStateReader = ocmh.zkStateReader; ZkStateReader.AliasesManager holder = zkStateReader.aliasesHolder; if (!anyRoutingParams(message)) { @@ -163,7 +169,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) final String maxFutureMs = message.getStr(ROUTING_MAX_FUTURE); try { - if (0 > Long.valueOf(maxFutureMs)) { + if (0 > Long.parseLong(maxFutureMs)) { throw new NumberFormatException("Negative value not allowed here"); } } catch (NumberFormatException e) { @@ -173,44 +179,35 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) // Validate we got everything we need if (routedField == null || routingType == null || start == null || increment == null) { - SolrException solrException = new SolrException(BAD_REQUEST, "If any of " + CREATE_ROUTED_ALIAS_PARAMS + + throw new SolrException(BAD_REQUEST, "If any of " + CREATE_ROUTED_ALIAS_PARAMS + " are supplied, then all of " + REQUIRED_ROUTING_PARAMS + " must be present."); - log.error("Could not create routed alias",solrException); - throw solrException; } if (!"time".equals(routingType)) { - SolrException solrException = new SolrException(BAD_REQUEST, "Only time based routing is supported at this time"); - log.error("Could not create routed alias",solrException); - throw solrException; + throw new SolrException(BAD_REQUEST, "Only time based routing is supported at this time"); } // Check for invalid timezone - if(tz != null && !TimeZoneUtils.KNOWN_TIMEZONE_IDS.contains(tz)) { - SolrException solrException = new SolrException(BAD_REQUEST, "Invalid timezone:" + tz); - log.error("Could not create routed alias",solrException); - throw solrException; - - } + TimeZoneUtils.parseTimezone(tz); TimeZone zone; if (tz != null) { zone = TimeZoneUtils.getTimeZone(tz); } else { zone = TimeZoneUtils.getTimeZone("UTC"); } - DateTimeFormatter fmt = DATE_TIME_FORMATTER.withZone(zone.toZoneId()); // check that the increment is valid date math - String checkIncrement = ISO_INSTANT.format(Instant.now()) + increment; - DateMathParser.parseMath(new Date(), checkIncrement); // exception if invalid increment + try { + new DateMathParser().parseMath(increment); + } catch (ParseException e) { + throw new SolrException(BAD_REQUEST,e.getMessage(),e); + } - Instant startTime = validateStart(zone, fmt, start); + Instant startTime = validateStart(zone, start); - // check field + // check config String config = String.valueOf(message.getProperties().get(ROUTED_ALIAS_COLLECTION_PROP_PFX + COLL_CONF)); if (!zkStateReader.getConfigManager().configExists(config)) { - SolrException solrException = new SolrException(BAD_REQUEST, "Could not find config '" + config + "'"); - log.error("Could not create routed alias",solrException); - throw solrException; + throw new SolrException(BAD_REQUEST, "Could not find config '" + config + "'"); } // It's too much work to check the routed field against the schema, there seems to be no good way to get @@ -220,13 +217,13 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) // field patterns too. As much as it would be nice to validate all inputs it's not worth the effort. String initialCollectionName = TimeRoutedAliasUpdateProcessor - .formatCollectionNameFromInstant(aliasName, startTime, fmt ); + .formatCollectionNameFromInstant(aliasName, startTime, DATE_TIME_FORMATTER); NamedList createResults = new NamedList(); ZkNodeProps collectionProps = selectByPrefix(ROUTED_ALIAS_COLLECTION_PROP_PFX, message) - .plus(NAME, initialCollectionName); + .plus(CommonParams.NAME, initialCollectionName); Map metadata = buildAliasMap(routedField, routingType, tz, increment, maxFutureMs, collectionProps); - RoutedAliasCreateCollectionCmd.createCollectionAndWait(state,createResults,aliasName,metadata,initialCollectionName,ocmh); + RoutedAliasCreateCollectionCmd.createCollectionAndWait(state, createResults, aliasName, metadata, initialCollectionName, ocmh); List collectionList = Collections.singletonList(initialCollectionName); validateAllCollectionsExistAndNoDups(collectionList, zkStateReader); holder.applyModificationAndExportToZk(aliases -> aliases @@ -252,10 +249,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) private Map buildAliasMap(String routedField, String routingType, String tz, String increment, String maxFutureMs, ZkNodeProps collectionProps) { Map properties = collectionProps.getProperties(); Map cleanMap = properties.entrySet().stream() - .filter(stringObjectEntry -> - !"fromApi".equals(stringObjectEntry.getKey()) - && !"stateFormat".equals(stringObjectEntry.getKey()) - && !"name".equals(stringObjectEntry.getKey())) + .filter(stringObjectEntry -> !COLLECTION_CREATION_CRUFT.contains(stringObjectEntry.getKey())) .collect(Collectors.toMap((e) -> "collection-create." + e.getKey(), e -> String.valueOf(e.getValue()))); cleanMap.put(ROUTING_FIELD, routedField); cleanMap.put(ROUTING_TYPE, routingType); @@ -265,31 +259,25 @@ private Map buildAliasMap(String routedField, String routingType return cleanMap; } - private Instant validateStart(TimeZone zone, DateTimeFormatter fmt, String start) { + private Instant validateStart(TimeZone zone, String start) { // This is the normal/easy case, if we can get away with this great! TemporalAccessor startTime = attemptTimeStampParsing(start, zone.toZoneId()); if (startTime == null) { - // No luck, they gave us either date math, or garbage, so we have to do more work to figure out which and - // to make sure it's valid date math and that it doesn't encode any millisecond precision. - ZonedDateTime now = ZonedDateTime.now(zone.toZoneId()).truncatedTo(ChronoUnit.MILLIS); - try { - Date date = DateMathParser.parseMath(Date.from(now.toInstant()), start); - String reformatted = fmt.format(date.toInstant().truncatedTo(ChronoUnit.MILLIS)); - Date reparse = Date.from(Instant.from(DATE_TIME_FORMATTER.parse(reformatted))); - if (!reparse.equals(date)) { - throw new SolrException(BAD_REQUEST, - "Formatted time did not have the same milliseconds as original: " + date.getTime() + " vs. " + - reparse.getTime() + " This indicates that you used date math that includes milliseconds. " + - "(Hint: 'NOW' used without rounding always has this problem)" ); - } - return date.toInstant(); - } catch (SolrException e) { - throw new SolrException(BAD_REQUEST, - "Start Time for the first collection must be a timestamp of the format yyyy-MM-dd_HH_mm_ss, " + - "or a valid date math expression not containing specific milliseconds", e); - } + Date date = DateMathParser.parseMath(new Date(), start); + checkMilis(date); + return date.toInstant(); + } + Instant startInstant = Instant.from(startTime); + checkMilis(Date.from(startInstant)); + return startInstant; + } + + private void checkMilis(Date date) { + if (!date.toInstant().truncatedTo(ChronoUnit.SECONDS).equals(date.toInstant())){ + throw new SolrException(BAD_REQUEST, + "Date or date math for start time includes milliseconds, which is not supported. " + + "(Hint: 'NOW' used without rounding always has this problem)"); } - return Instant.from(startTime); } private TemporalAccessor attemptTimeStampParsing(String start, ZoneId zone) { @@ -338,11 +326,13 @@ private List parseCollectionsParameter(Object colls) { } private ZkNodeProps selectByPrefix(String prefix, ZkNodeProps source) { - final ZkNodeProps[] subSet = {new ZkNodeProps()}; - source.getProperties().entrySet().stream() - .filter(entry -> entry.getKey().startsWith(prefix)) - .forEach(e -> subSet[0] = subSet[0].plus(e.getKey().substring(prefix.length()),e.getValue())); - return subSet[0]; + ZkNodeProps subSet = new ZkNodeProps(); + for (Map.Entry entry : source.getProperties().entrySet()) { + if (entry.getKey().startsWith(prefix)) { + subSet = subSet.plus(entry.getKey().substring(prefix.length()), entry.getValue()); + } + } + return subSet; } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 04a7055f8e8e..8e891b3bab06 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -419,7 +419,7 @@ public enum CollectionOperation implements CollectionOp { */ CREATE_OP(CREATE, (req, rsp, h) -> { req.getParams().required().getAll(null, NAME); - return parseColletionCreationProps(h, req.getParams(), null); + return parseCollectionCreationProps(h, req.getParams(), null); }), DELETE_OP(DELETE, (req, rsp, h) -> req.getParams().required().getAll(null, NAME)), @@ -460,16 +460,14 @@ public enum CollectionOperation implements CollectionOp { // subset the params to reuse the collection creation/parsing code ModifiableSolrParams collectionParams = extractPrefixedParams("create-collection.", req.getParams()); if (collectionParams.get(NAME) != null) { - SolrException solrException = new SolrException(BAD_REQUEST, "routed aliases calculate names for their " + + throw new SolrException(BAD_REQUEST, "routed aliases calculate names for their " + "dependent collections, you cannot specify the name."); - log.error("Could not create routed alias",solrException); - throw solrException; } SolrParams v1Params = convertToV1WhenRequired(req, collectionParams); // We need to add this temporary name just to pass validation. collectionParams.add(NAME, "TMP_name_TMP_name_TMP"); - params.putAll(parseColletionCreationProps(h, v1Params, ROUTED_ALIAS_COLLECTION_PROP_PFX)); + params.putAll(parseCollectionCreationProps(h, v1Params, ROUTED_ALIAS_COLLECTION_PROP_PFX)); // We will be giving the collection a real name based on the partition scheme later. params.remove(ROUTED_ALIAS_COLLECTION_PROP_PFX + NAME); @@ -1002,7 +1000,7 @@ private static SolrParams convertToV1WhenRequired(SolrQueryRequest req, Modifiab return v1Params; } - private static Map parseColletionCreationProps(CollectionsHandler h, SolrParams params, String prefix) + private static Map parseCollectionCreationProps(CollectionsHandler h, SolrParams params, String prefix) throws KeeperException, InterruptedException { Map props = params.getAll(null, NAME, diff --git a/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java index 7e0cae133419..ff2043d21775 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java @@ -101,8 +101,6 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor { .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) - // Setting a timezone here is fine as a default, but generally need to clone with user's timezone, so that - // truncation of milliseconds is consistent. .toFormatter(Locale.ROOT).withZone(ZoneOffset.UTC); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -256,7 +254,6 @@ public void processAdd(AddUpdateCommand cmd) throws IOException { /** Computes the timestamp of the next collection given the timestamp of the one before. */ public static Instant computeNextCollTimestamp(Instant fromTimestamp, String intervalDateMath, TimeZone intervalTimeZone) { //TODO overload DateMathParser.parseMath to take tz and "now" - // GH: I don't think that's necessary, you can set the TZ on now when you pass it in? final DateMathParser dateMathParser = new DateMathParser(intervalTimeZone); dateMathParser.setNow(Date.from(fromTimestamp)); final Instant nextCollTimestamp; diff --git a/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java b/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java index cabf1449cc20..3bfb6c481fef 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java @@ -24,6 +24,7 @@ import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.Date; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -46,7 +47,6 @@ import org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor; import org.apache.solr.util.DateMathParser; import org.junit.After; -import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -54,6 +54,7 @@ /** * Direct http tests of the CreateRoutedAlias functionality. */ +@SolrTestCaseJ4.SuppressSSL public class CreateRoutedAliasTest extends SolrCloudTestCase { private CloudSolrClient solrClient; @@ -92,8 +93,8 @@ public void testV2() throws Exception { final String aliasName = "testAlias"; cluster.uploadConfigSet(configset("_default"), aliasName); final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - HttpPost post = new HttpPost(baseUrl + "/____v2/c"); - post.setHeader("Content-Type", ContentType.APPLICATION_JSON.getMimeType()); + HttpPost post = new HttpPost(baseUrl + "/____v2/c"); + post.setHeader("Content-Type", ContentType.APPLICATION_JSON.getMimeType()); HttpEntity httpEntity = new InputStreamEntity(org.apache.commons.io.IOUtils.toInputStream("{\n" + " \"create-routed-alias\" : {\n" + " \"name\": \"testaliasV2\",\n" + @@ -186,6 +187,42 @@ public void testV1() throws Exception { assertEquals(null ,meta.get("start")); } + + @Test + public void testTimezoneAbsoluteDate() throws Exception { + + final String aliasName = "testAlias"; + cluster.uploadConfigSet(configset("_default"), aliasName); + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis + String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); + HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + + "&wt=xml" + + "&name=testalias" + + "&router.field=evt_dt" + + "&router.name=time" + + "&start=2018-01-15T00:00:00Z" + + "&TZ=GMT-10" + + "&router.interval=%2B30MINUTE" + + "&router.max-future-ms=60000" + + "&create-collection.collection.configName=_default" + + "&create-collection.numShards=2"); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(200, response.getStatusLine().getStatusCode()); + } + //TODO: name like this because interval is in hours? + //assertInitialCollectionNameExists("testalias_2018-01-15_00_00"); + assertInitialCollectionNameExists("testalias_2018-01-15"); + //get = new HttpGet(baseUrl + "/____v2/c/testalias_2018-01-15_00_00"); + get = new HttpGet(baseUrl + "/____v2/c/testalias_2018-01-15"); + try (CloseableHttpResponse response = httpclient.execute(get)) { + System.out.println(responseToMap(response)); + System.out.println(getCollectionList()); + assertEquals(200, response.getStatusLine().getStatusCode()); + } + + } + @Test public void testAliasNameMustBeValid() throws Exception { @@ -254,22 +291,11 @@ public void testTimeStampWithMsFails() throws Exception { "&create-collection.numShards=2"); try (CloseableHttpResponse response = httpclient.execute(get)) { assertEquals(400, response.getStatusLine().getStatusCode()); - assertErrorStartsWith(response, "Start Time for the first collection must be a timestamp of the format yyyy-MM-dd_HH_mm_ss"); + assertErrorStartsWith(response, "Date or date math for start time includes milliseconds"); } } - private void assertErrorStartsWith(CloseableHttpResponse response, String prefix) throws IOException { - String entity = getStringEntity(response); - System.out.println(entity); - ObjectMapper mapper = new ObjectMapper(); - Map map = mapper.readValue(entity, new TypeReference>() {}); - Map exception = (Map) map.get("exception"); - if (exception == null) { - exception = (Map) map.get("error"); - } - String msg = (String) exception.get("msg"); - assertTrue(msg.toLowerCase(Locale.ROOT).startsWith(prefix.toLowerCase(Locale.ROOT))); - } + private String getStringEntity(CloseableHttpResponse response) throws IOException { ByteArrayOutputStream outstream = new ByteArrayOutputStream(); @@ -297,7 +323,7 @@ public void testBadDateMathIntervalFails() throws Exception { "&create-collection.numShards=2"); try (CloseableHttpResponse response = httpclient.execute(get)) { assertEquals(400, response.getStatusLine().getStatusCode()); - assertErrorStartsWith(response,"Invalid Date Math"); + assertErrorStartsWith(response,"Unit not recognized"); } } @Test @@ -347,5 +373,38 @@ public void testUnParseableFutureFails() throws Exception { } } + private void assertErrorStartsWith(CloseableHttpResponse response, String prefix) throws IOException { + Map map = responseToMap(response); + Map exception = (Map) map.get("exception"); + if (exception == null) { + exception = (Map) map.get("error"); + } + String msg = (String) exception.get("msg"); + assertTrue(msg.toLowerCase(Locale.ROOT).startsWith(prefix.toLowerCase(Locale.ROOT))); + } + + private Map responseToMap(CloseableHttpResponse response) throws IOException { + String entity = getStringEntity(response); + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(entity, new TypeReference>() {}); + } + + private void assertInitialCollectionNameExists(String name) throws IOException { + List collections; + collections = getCollectionList(); + assertTrue(name + " not found among existing collections:" + collections,collections.contains(name)); + } + + private List getCollectionList() throws IOException { + List collections; + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + HttpGet get = new HttpGet(baseUrl + "/____v2/c"); + try (CloseableHttpResponse response = httpclient.execute(get)) { + assertEquals(200, response.getStatusLine().getStatusCode()); + Map map = responseToMap(response); + collections = (List) map.get("collections"); + } + return collections; + } // not testing collection parameters, those should inherit error checking from the collection creation code. } diff --git a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java index db4b877657ca..2ba380eeea83 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java @@ -43,6 +43,7 @@ import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.Aliases; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; @@ -80,6 +81,15 @@ public static void finish() throws Exception { //TODO this is necessary when -Dtests.iters but why? Some other tests aren't affected @Before public void doBefore() throws Exception { + // delete aliases first to avoid problems such as: https://issues.apache.org/jira/browse/SOLR-11839 + ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader(); + zkStateReader.aliasesHolder.applyModificationAndExportToZk(aliases -> { + Aliases a = zkStateReader.getAliases(); + for (String alias : a.getCollectionAliasMap().keySet()) { + a = a.cloneWithCollectionAlias(alias,null); // remove + } + return a; + }); for (String col : CollectionAdminRequest.listCollections(solrClient)) { CollectionAdminRequest.deleteCollection(col).process(solrClient); } From d2bce19e7e86030f2babee10a6402263112ce486 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Tue, 23 Jan 2018 00:50:52 -0500 Subject: [PATCH 3/3] SOLR-11722 various improvements. Fixed v2->v1 mapping bug. Avoided redundant constants. Adjusted CollectionAdminRequest API. Simplified tests. --- .../org/apache/solr/cloud/CreateAliasCmd.java | 186 +++------ .../solr/cloud/CreateCollectionCmd.java | 6 +- .../cloud/RoutedAliasCreateCollectionCmd.java | 11 +- .../handler/admin/BaseHandlerApiSupport.java | 4 +- .../handler/admin/CollectionsHandler.java | 197 +++++---- .../apache/solr/request/SolrRequestInfo.java | 2 +- .../org/apache/solr/util/DateMathParser.java | 17 +- .../org/apache/solr/util/TimeZoneUtils.java | 2 +- solr/core/src/test-files/log4j.properties | 2 + .../ConcurrentCreateRoutedAliasTest.java | 108 +---- .../solr/cloud/CreateRoutedAliasTest.java | 378 ++++++++---------- .../TimeRoutedAliasUpdateProcessorTest.java | 17 - .../solrj/impl/HttpClusterStateProvider.java | 1 + .../solrj/request/CollectionAdminRequest.java | 119 +++--- .../solrj/request/CollectionApiMapping.java | 66 +-- .../solr/common/cloud/ZkStateReader.java | 2 +- .../solr/common/util/JsonSchemaValidator.java | 3 +- 17 files changed, 458 insertions(+), 663 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java index e191071cb069..a5313c00cd75 100644 --- a/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java @@ -20,20 +20,18 @@ import java.lang.invoke.MethodHandles; import java.text.ParseException; import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeParseException; import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalAccessor; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TimeZone; +import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd; @@ -50,10 +48,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; +import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CommonParams.TZ; -import static org.apache.solr.handler.admin.CollectionsHandler.ROUTED_ALIAS_COLLECTION_PROP_PFX; import static org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER; @@ -61,43 +58,16 @@ public class CreateAliasCmd implements Cmd { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String START = "start"; //TODO, router related public static final String ROUTING_TYPE = "router.name"; public static final String ROUTING_FIELD = "router.field"; public static final String ROUTING_INCREMENT = "router.interval"; public static final String ROUTING_MAX_FUTURE = "router.max-future-ms"; - public static final String START = "start"; - // Collection constants should all reflect names in the v2 structured input for this command, not v1 - // names used for CREATE - public static final String CREATE_COLLECTION_CONFIG = "create-collection.config"; - public static final String CREATE_COLLECTION_ROUTER_NAME = "create-collection.router.name"; - public static final String CREATE_COLLECTION_ROUTER_FIELD = "create-collection.router.field"; - public static final String CREATE_COLLECTION_NUM_SHARDS = "create-collection.numShards"; - public static final String CREATE_COLLECTION_SHARDS = "create-collection.shards"; - public static final String CREATE_COLLECTION_REPLICATION_FACTOR = "create-collection.replicationFactor"; - public static final String CREATE_COLLECTION_NRT_REPLICAS = "create-collection.nrtReplicas"; - public static final String CREATE_COLLECTION_TLOG_REPLICAS = "create-collection.tlogReplicas"; - public static final String CREATE_COLLECTION_PULL_REPLICAS = "create-collection.pullReplicas"; - public static final String CREATE_COLLECTION_NODE_SET = "create-collection.nodeSet"; - public static final String CREATE_COLLECTION_SHUFFLE_NODES = "create-collection.shuffleNodes"; - public static final String CREATE_COLLECTION_MAX_SHARDS_PER_NODE = "create-collection.maxShardsPerNode"; - public static final String CREATE_COLLECTION_AUTO_ADD_REPLICAS = "create-collection.autoAddReplicas"; - public static final String CREATE_COLLECTION_RULE = "create-collection.rule"; - public static final String CREATE_COLLECTION_SNITCH = "create-collection.snitch"; - public static final String CREATE_COLLECTION_POLICY = "create-collection.policy"; - public static final String CREATE_COLLECTION_PROPERTIES = "create-collection.properties"; - public static final String FROM_API = "fromApi"; - public static final String STATE_FORMAT = "stateFormat"; - public static final String NAME = "name"; - - // These are parameters that get added by the collection creation command parsing that we - // will want to not store in alias metadata. - public static final Set COLLECTION_CREATION_CRUFT = // wanted: Set.of() from jdk9 - Collections.unmodifiableSet(new HashSet<>(Arrays.asList(FROM_API, STATE_FORMAT, NAME))); + public static final String CREATE_COLLECTION_PREFIX = "create-collection."; private final OverseerCollectionMessageHandler ocmh; - /** * Parameters required for creating a routed alias */ @@ -114,35 +84,14 @@ public class CreateAliasCmd implements Cmd { public static final List NONREQUIRED_ROUTING_PARAMS = Collections.unmodifiableList(Arrays.asList( ROUTING_MAX_FUTURE, TZ)); - /** - * Parameters used by routed Aliases to create collections. - */ - public static final List COLLECTION_ROUTING_PARAMS = Collections.unmodifiableList(Arrays.asList( - CREATE_COLLECTION_CONFIG, - CREATE_COLLECTION_ROUTER_FIELD, - CREATE_COLLECTION_ROUTER_NAME, - CREATE_COLLECTION_NUM_SHARDS, - CREATE_COLLECTION_SHARDS, - CREATE_COLLECTION_REPLICATION_FACTOR, - CREATE_COLLECTION_NRT_REPLICAS, - CREATE_COLLECTION_TLOG_REPLICAS, - CREATE_COLLECTION_PULL_REPLICAS, - CREATE_COLLECTION_NODE_SET, - CREATE_COLLECTION_SHUFFLE_NODES, - CREATE_COLLECTION_MAX_SHARDS_PER_NODE, - CREATE_COLLECTION_AUTO_ADD_REPLICAS, - CREATE_COLLECTION_RULE, - CREATE_COLLECTION_SNITCH, - CREATE_COLLECTION_POLICY, - CREATE_COLLECTION_PROPERTIES)); - - public static final List CREATE_ROUTED_ALIAS_PARAMS; - static { - List params = new ArrayList<>(); - params.addAll(REQUIRED_ROUTING_PARAMS); - params.addAll(NONREQUIRED_ROUTING_PARAMS); - params.addAll(COLLECTION_ROUTING_PARAMS); - CREATE_ROUTED_ALIAS_PARAMS = Collections.unmodifiableList(params); + + private static Predicate PARAM_IS_METADATA = key -> !key.equals(NAME) && !key.equals(START) + && (REQUIRED_ROUTING_PARAMS.contains(key) || NONREQUIRED_ROUTING_PARAMS.contains(key) + || key.startsWith(CREATE_COLLECTION_PREFIX)); + + private static boolean anyRoutingParams(ZkNodeProps message) { + return message.containsKey(ROUTING_FIELD) || message.containsKey(ROUTING_TYPE) || message.containsKey(START) + || message.containsKey(ROUTING_INCREMENT) || message.containsKey(TZ); } public CreateAliasCmd(OverseerCollectionMessageHandler ocmh) { @@ -155,13 +104,31 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) final String aliasName = message.getStr(CommonParams.NAME); ZkStateReader zkStateReader = ocmh.zkStateReader; ZkStateReader.AliasesManager holder = zkStateReader.aliasesHolder; + + //TODO refactor callCreatePlainAlias if (!anyRoutingParams(message)) { + final List canonicalCollectionList = parseCollectionsParameter(message.get("collections")); final String canonicalCollectionsString = StrUtils.join(canonicalCollectionList, ','); validateAllCollectionsExistAndNoDups(canonicalCollectionList, zkStateReader); holder.applyModificationAndExportToZk(aliases -> aliases.cloneWithCollectionAlias(aliasName, canonicalCollectionsString)); - } else { - final String routedField = message.getStr(ROUTING_FIELD); + + } else { //TODO refactor callCreateRoutedAlias + + // Validate we got everything we need + if (!message.getProperties().keySet().containsAll(REQUIRED_ROUTING_PARAMS)) { + throw new SolrException(BAD_REQUEST, "A routed alias requires these params: " + REQUIRED_ROUTING_PARAMS + + " plus some create-collection prefixed ones."); + } + + Map aliasMetadata = new LinkedHashMap<>(); + message.getProperties().entrySet().stream() + .filter(entry -> PARAM_IS_METADATA.test(entry.getKey())) + .forEach(entry -> aliasMetadata.put(entry.getKey(), (String) entry.getValue())); + + //TODO read these from metadata where appropriate. This leads to consistent logic between initial routed alias + // collection creation, and subsequent collections to be created. + final String routingType = message.getStr(ROUTING_TYPE); final String tz = message.getStr(TZ); final String start = message.getStr(START); @@ -169,7 +136,7 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) final String maxFutureMs = message.getStr(ROUTING_MAX_FUTURE); try { - if (0 > Long.parseLong(maxFutureMs)) { + if (maxFutureMs != null && 0 > Long.parseLong(maxFutureMs)) { throw new NumberFormatException("Negative value not allowed here"); } } catch (NumberFormatException e) { @@ -177,38 +144,21 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) "of milliseconds greater than or equal to zero"); } - // Validate we got everything we need - if (routedField == null || routingType == null || start == null || increment == null) { - throw new SolrException(BAD_REQUEST, "If any of " + CREATE_ROUTED_ALIAS_PARAMS + - " are supplied, then all of " + REQUIRED_ROUTING_PARAMS + " must be present."); - } - if (!"time".equals(routingType)) { throw new SolrException(BAD_REQUEST, "Only time based routing is supported at this time"); } + // Check for invalid timezone - TimeZoneUtils.parseTimezone(tz); - TimeZone zone; - if (tz != null) { - zone = TimeZoneUtils.getTimeZone(tz); - } else { - zone = TimeZoneUtils.getTimeZone("UTC"); - } + TimeZone zone = TimeZoneUtils.parseTimezone(tz); // check that the increment is valid date math try { - new DateMathParser().parseMath(increment); + new DateMathParser(zone).parseMath(increment); } catch (ParseException e) { throw new SolrException(BAD_REQUEST,e.getMessage(),e); } - Instant startTime = validateStart(zone, start); - - // check config - String config = String.valueOf(message.getProperties().get(ROUTED_ALIAS_COLLECTION_PROP_PFX + COLL_CONF)); - if (!zkStateReader.getConfigManager().configExists(config)) { - throw new SolrException(BAD_REQUEST, "Could not find config '" + config + "'"); - } + Instant startTime = parseStart(start, zone); // It's too much work to check the routed field against the schema, there seems to be no good way to get // a copy of the schema aside from loading it directly from zookeeper based on the config name, but that @@ -219,16 +169,15 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) String initialCollectionName = TimeRoutedAliasUpdateProcessor .formatCollectionNameFromInstant(aliasName, startTime, DATE_TIME_FORMATTER); + // Create the collection NamedList createResults = new NamedList(); - ZkNodeProps collectionProps = selectByPrefix(ROUTED_ALIAS_COLLECTION_PROP_PFX, message) - .plus(CommonParams.NAME, initialCollectionName); - Map metadata = buildAliasMap(routedField, routingType, tz, increment, maxFutureMs, collectionProps); - RoutedAliasCreateCollectionCmd.createCollectionAndWait(state, createResults, aliasName, metadata, initialCollectionName, ocmh); - List collectionList = Collections.singletonList(initialCollectionName); - validateAllCollectionsExistAndNoDups(collectionList, zkStateReader); + RoutedAliasCreateCollectionCmd.createCollectionAndWait(state, createResults, aliasName, aliasMetadata, initialCollectionName, ocmh); + validateAllCollectionsExistAndNoDups(Collections.singletonList(initialCollectionName), zkStateReader); + + // Create/update the alias holder.applyModificationAndExportToZk(aliases -> aliases .cloneWithCollectionAlias(aliasName, initialCollectionName) - .cloneWithCollectionAliasMetadata(aliasName, metadata)); + .cloneWithCollectionAliasMetadata(aliasName, aliasMetadata)); } // Sleep a bit to allow ZooKeeper state propagation. @@ -246,55 +195,20 @@ public void call(ClusterState state, ZkNodeProps message, NamedList results) Thread.sleep(100); } - private Map buildAliasMap(String routedField, String routingType, String tz, String increment, String maxFutureMs, ZkNodeProps collectionProps) { - Map properties = collectionProps.getProperties(); - Map cleanMap = properties.entrySet().stream() - .filter(stringObjectEntry -> !COLLECTION_CREATION_CRUFT.contains(stringObjectEntry.getKey())) - .collect(Collectors.toMap((e) -> "collection-create." + e.getKey(), e -> String.valueOf(e.getValue()))); - cleanMap.put(ROUTING_FIELD, routedField); - cleanMap.put(ROUTING_TYPE, routingType); - cleanMap.put(ROUTING_INCREMENT, increment); - cleanMap.put(ROUTING_MAX_FUTURE, maxFutureMs); - cleanMap.put(TZ, tz); - return cleanMap; - } - - private Instant validateStart(TimeZone zone, String start) { - // This is the normal/easy case, if we can get away with this great! - TemporalAccessor startTime = attemptTimeStampParsing(start, zone.toZoneId()); - if (startTime == null) { - Date date = DateMathParser.parseMath(new Date(), start); - checkMilis(date); - return date.toInstant(); - } - Instant startInstant = Instant.from(startTime); - checkMilis(Date.from(startInstant)); - return startInstant; + private Instant parseStart(String str, TimeZone zone) { + Instant start = DateMathParser.parseMath(new Date(), str, zone).toInstant(); + checkMilis(start); + return start; } - private void checkMilis(Date date) { - if (!date.toInstant().truncatedTo(ChronoUnit.SECONDS).equals(date.toInstant())){ + private void checkMilis(Instant date) { + if (!date.truncatedTo(ChronoUnit.SECONDS).equals(date)) { throw new SolrException(BAD_REQUEST, "Date or date math for start time includes milliseconds, which is not supported. " + "(Hint: 'NOW' used without rounding always has this problem)"); } } - private TemporalAccessor attemptTimeStampParsing(String start, ZoneId zone) { - try { - DATE_TIME_FORMATTER.withZone(zone); - return DATE_TIME_FORMATTER.parse(start); - } catch (DateTimeParseException e) { - return null; - } - } - - private boolean anyRoutingParams(ZkNodeProps message) { - - return message.containsKey(ROUTING_FIELD) || message.containsKey(ROUTING_TYPE) || message.containsKey(START) - || message.containsKey(ROUTING_INCREMENT) || message.containsKey(TZ); - } - private void validateAllCollectionsExistAndNoDups(List collectionList, ZkStateReader zkStateReader) { final String collectionStr = StrUtils.join(collectionList, ','); diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java index 2171c605bf55..a6711878ee1d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java @@ -405,9 +405,6 @@ public static void createCollectionZkNode(DistribStateManager stateManager, Stri try { Map collectionProps = new HashMap<>(); - // TODO: if collection.configName isn't set, and there isn't already a conf in zk, just use that? - String defaultConfigName = System.getProperty(ZkController.COLLECTION_PARAM_PREFIX + ZkController.CONFIGNAME_PROP, collection); - if (params.size() > 0) { collectionProps.putAll(params); // if the config name wasn't passed in, use the default @@ -417,6 +414,8 @@ public static void createCollectionZkNode(DistribStateManager stateManager, Stri } } else if (System.getProperty("bootstrap_confdir") != null) { + String defaultConfigName = System.getProperty(ZkController.COLLECTION_PARAM_PREFIX + ZkController.CONFIGNAME_PROP, collection); + // if we are bootstrapping a collection, default the config for // a new collection to the collection we are bootstrapping log.info("Setting config for collection:" + collection + " to " + defaultConfigName); @@ -445,6 +444,7 @@ public static void createCollectionZkNode(DistribStateManager stateManager, Stri stateManager.makePath(collectionPath, Utils.toJSON(zkProps), CreateMode.PERSISTENT, false); } catch (KeeperException e) { + //TODO shouldn't the stateManager ensure this does not happen; should throw AlreadyExistsException // it's okay if the node already exists if (e.code() != KeeperException.Code.NODEEXISTS) { throw e; diff --git a/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java index 0c5278b1a993..89166082d630 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/RoutedAliasCreateCollectionCmd.java @@ -41,6 +41,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.solr.cloud.CreateAliasCmd.CREATE_COLLECTION_PREFIX; import static org.apache.solr.cloud.OverseerCollectionMessageHandler.COLL_CONF; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor.ROUTER_FIELD_METADATA; @@ -60,8 +61,6 @@ public class RoutedAliasCreateCollectionCmd implements OverseerCollectionMessage public static final String IF_MOST_RECENT_COLL_NAME = "ifMostRecentCollName"; - public static final String COLL_METAPREFIX = "collection-create."; - private final OverseerCollectionMessageHandler ocmh; public RoutedAliasCreateCollectionCmd(OverseerCollectionMessageHandler ocmh) { @@ -151,16 +150,16 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList resul } - static void createCollectionAndWait(ClusterState clusterState, NamedList results, String aliasName, Map aliasMetadata, String createCollName, OverseerCollectionMessageHandler ocmh) throws Exception { + static void createCollectionAndWait(ClusterState clusterState, NamedList results, String aliasName, Map aliasMetadata, String createCollName, OverseerCollectionMessageHandler ocmh) throws Exception { // Map alias metadata starting with a prefix to a create-collection API request final ModifiableSolrParams createReqParams = new ModifiableSolrParams(); for (Map.Entry e : aliasMetadata.entrySet()) { - if (e.getKey().startsWith(COLL_METAPREFIX)) { - createReqParams.set(e.getKey().substring(COLL_METAPREFIX.length()), e.getValue()); + if (e.getKey().startsWith(CREATE_COLLECTION_PREFIX)) { + createReqParams.set(e.getKey().substring(CREATE_COLLECTION_PREFIX.length()), e.getValue()); } } if (createReqParams.get(COLL_CONF) == null) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "We require an explicit " + COLL_CONF ); } createReqParams.set(NAME, createCollName); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java b/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java index 89b63fcc2808..e525269518e9 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java @@ -119,7 +119,7 @@ public void call(SolrQueryRequest req, SolrQueryResponse rsp) { } catch (SolrException e) { throw e; } catch (Exception e) { - throw new SolrException(BAD_REQUEST, e); + throw new SolrException(BAD_REQUEST, e); //TODO BAD_REQUEST is a wild guess; should we flip the default? fail here to investigate how this happens in tests } finally { req.setParams(params); } @@ -199,7 +199,7 @@ public String[] getParams(String param) { @Override public Iterator getParameterNamesIterator() { - return meta.getParamNames(co).iterator(); + return meta.getParamNamesIterator(co); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 8e891b3bab06..462f842353c3 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -30,7 +30,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; +import java.util.function.BiConsumer; import com.google.common.collect.ImmutableSet; import org.apache.commons.io.IOUtils; @@ -39,7 +39,6 @@ import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; -import org.apache.solr.client.solrj.request.CollectionApiMapping; import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestSyncShard; import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; @@ -76,7 +75,6 @@ import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.Utils; @@ -87,6 +85,7 @@ import org.apache.solr.core.snapshots.SolrSnapshotManager; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.component.ShardHandler; +import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.AuthorizationContext; @@ -102,6 +101,7 @@ import static org.apache.solr.client.solrj.response.RequestStatusState.NOT_FOUND; import static org.apache.solr.client.solrj.response.RequestStatusState.RUNNING; import static org.apache.solr.client.solrj.response.RequestStatusState.SUBMITTED; +import static org.apache.solr.cloud.CreateAliasCmd.CREATE_COLLECTION_PREFIX; import static org.apache.solr.cloud.CreateAliasCmd.NONREQUIRED_ROUTING_PARAMS; import static org.apache.solr.cloud.CreateAliasCmd.REQUIRED_ROUTING_PARAMS; import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; @@ -151,7 +151,6 @@ public class CollectionsHandler extends RequestHandlerBase implements PermissionNameProvider { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String ROUTED_ALIAS_COLLECTION_PROP_PFX = "__collection-"; protected final CoreContainer coreContainer; private final CollectionHandlerApi v2Handler ; @@ -419,8 +418,46 @@ public enum CollectionOperation implements CollectionOp { */ CREATE_OP(CREATE, (req, rsp, h) -> { req.getParams().required().getAll(null, NAME); - return parseCollectionCreationProps(h, req.getParams(), null); + SolrParams params = req.getParams(); + Map props = params.getAll(null, + NAME, + REPLICATION_FACTOR, + COLL_CONF, + NUM_SLICES, + MAX_SHARDS_PER_NODE, + CREATE_NODE_SET, + CREATE_NODE_SET_SHUFFLE, + SHARDS_PROP, + STATE_FORMAT, + AUTO_ADD_REPLICAS, + RULE, + SNITCH, + PULL_REPLICAS, + TLOG_REPLICAS, + NRT_REPLICAS, + POLICY, + WAIT_FOR_FINAL_STATE); + props.put("fromApi", "true"); + props.putIfAbsent(STATE_FORMAT, "2"); + addMapObject(props, RULE); + addMapObject(props, SNITCH); + verifyRuleParams(h.coreContainer, props); + final String collectionName = SolrIdentifierValidator.validateCollectionName((String) props.get(NAME)); + final String shardsParam = (String) props.get(SHARDS_PROP); + if (StringUtils.isNotEmpty(shardsParam)) { + verifyShardsParam(shardsParam); + } + if (CollectionAdminParams.SYSTEM_COLL.equals(collectionName)) { + //We must always create a .system collection with only a single shard + props.put(NUM_SLICES, 1); + props.remove(SHARDS_PROP); + createSysConfigSet(h.coreContainer); + } + copyPropertiesWithPrefix(params, props, COLL_PROP_PREFIX); + copyPropertiesWithPrefix(params, props, "router."); + return props; }), + DELETE_OP(DELETE, (req, rsp, h) -> req.getParams().required().getAll(null, NAME)), RELOAD_OP(RELOAD, (req, rsp, h) -> req.getParams().required().getAll(null, NAME)), @@ -447,33 +484,51 @@ public enum CollectionOperation implements CollectionOp { } return null; }), + CREATEALIAS_OP(CREATEALIAS, (req, rsp, h) -> { SolrIdentifierValidator.validateAliasName(req.getParams().get(NAME)); return req.getParams().required().getAll(null, NAME, "collections"); }), + CREATEROUTEDALIAS_OP(CREATEROUTEDALIAS, (req, rsp, h) -> { String alias = req.getParams().get(NAME); SolrIdentifierValidator.validateAliasName(alias); - Map params = req.getParams().required() - .getAll(null, REQUIRED_ROUTING_PARAMS.toArray(new String[REQUIRED_ROUTING_PARAMS.size()])); - req.getParams().getAll(params, NONREQUIRED_ROUTING_PARAMS); - // subset the params to reuse the collection creation/parsing code - ModifiableSolrParams collectionParams = extractPrefixedParams("create-collection.", req.getParams()); - if (collectionParams.get(NAME) != null) { + Map result = req.getParams().required().getAll(null, REQUIRED_ROUTING_PARAMS); + req.getParams().getAll(result, NONREQUIRED_ROUTING_PARAMS); + + ModifiableSolrParams createCollParams = new ModifiableSolrParams(); // without prefix + + // add to result params that start with "create-collection.". + // Additionally, save these without the prefix to createCollParams + + forEach(req.getParams(), (p, v) -> { + if (p.startsWith(CREATE_COLLECTION_PREFIX)) { + // This is what SolrParams#getAll(Map, Collection)} does + if (v.length == 1) { + result.put(p, v[0]); + } else { + result.put(p, v); + } + createCollParams.set(p.substring(CREATE_COLLECTION_PREFIX.length()), v); + } + }); + + // Verify that the create-collection prefix'ed params appear to be valid. + if (createCollParams.get(NAME) != null) { throw new SolrException(BAD_REQUEST, "routed aliases calculate names for their " + "dependent collections, you cannot specify the name."); } - SolrParams v1Params = convertToV1WhenRequired(req, collectionParams); - - // We need to add this temporary name just to pass validation. - collectionParams.add(NAME, "TMP_name_TMP_name_TMP"); - params.putAll(parseCollectionCreationProps(h, v1Params, ROUTED_ALIAS_COLLECTION_PROP_PFX)); + if (createCollParams.get(COLL_CONF) == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "We require an explicit " + COLL_CONF ); + } + // note: could insist on a config name here as well.... or wait to throw at overseer + createCollParams.add(NAME, "TMP_name_TMP_name_TMP"); // just to pass validation + CREATE_OP.execute(new LocalSolrQueryRequest(null, createCollParams), rsp, h); // ignore results - // We will be giving the collection a real name based on the partition scheme later. - params.remove(ROUTED_ALIAS_COLLECTION_PROP_PFX + NAME); - params.remove("create-collection"); // don't include a stringified version now that we've parsed things out. - return params; + return result; }), + DELETEALIAS_OP(DELETEALIAS, (req, rsp, h) -> req.getParams().required().getAll(null, NAME)), /** @@ -930,25 +985,6 @@ public Map execute(SolrQueryRequest req, SolrQueryResponse rsp, }), DELETENODE_OP(DELETENODE, (req, rsp, h) -> req.getParams().required().getAll(null, "node")); - /** - * Extract only the params that have a given prefix. Any params with a different prefix are removed, and - * the params with the prefix have the prefix removed in the result - * - * @param prefix the prefix to select - * @param params the source of parameters that should be searched - * @return a SolrParams object containing only the params that had the prefix, with the prefix removed. - */ - private static ModifiableSolrParams extractPrefixedParams(String prefix, SolrParams params) { - ModifiableSolrParams result = new ModifiableSolrParams(); - for (Iterator i = params.getParameterNamesIterator(); i.hasNext();) { - String name = i.next(); - if (name.startsWith(prefix)) { - result.add(name.substring(prefix.length()), params.get(name)); - } - } - return result; - } - public final CollectionOp fun; CollectionAction action; long timeOut; @@ -980,75 +1016,7 @@ public Map execute(SolrQueryRequest req, SolrQueryResponse rsp, } } - private static SolrParams convertToV1WhenRequired(SolrQueryRequest req, ModifiableSolrParams params) { - SolrParams v1Params = params; // (maybe...) - - // in the v2 world we get a data map based on the json request, from the CommandOperation associated - // with the request, so locate that if we can.. if we find it we have to translate the v2 request - // properties to v1 params, otherwise we're already good to go. - List cmds = req.getCommands(true); - if (cmds.size() > 1) { - // todo: not sure if this is the right thing to do here, but also not sure what to do if there is more than one... - throw new SolrException(BAD_REQUEST, "Only one command is allowed when creating a routed alias"); - } - CommandOperation c = cmds.size() == 0 ? null : cmds.get(0); - if (c != null) { // v2 api, do conversion to v1 - v1Params = new BaseHandlerApiSupport.V2ToV1SolrParams(CollectionApiMapping.Meta.CREATE_COLLECTION, - req.getPathTemplateValues(), true, params, - new CommandOperation("create", c.getDataMap().get("create-collection"))); - } - return v1Params; - } - - private static Map parseCollectionCreationProps(CollectionsHandler h, SolrParams params, String prefix) - throws KeeperException, InterruptedException { - Map props = params.getAll(null, - NAME, - REPLICATION_FACTOR, - COLL_CONF, - NUM_SLICES, - MAX_SHARDS_PER_NODE, - CREATE_NODE_SET, - CREATE_NODE_SET_SHUFFLE, - SHARDS_PROP, - STATE_FORMAT, - AUTO_ADD_REPLICAS, - RULE, - SNITCH, - PULL_REPLICAS, - TLOG_REPLICAS, - NRT_REPLICAS, - POLICY, - WAIT_FOR_FINAL_STATE); - props.put("fromApi", "true"); - if (props.get(STATE_FORMAT) == null) { - props.put(STATE_FORMAT, "2"); - } - addMapObject(props, RULE); - addMapObject(props, SNITCH); - verifyRuleParams(h.coreContainer, props); - final String collectionName = SolrIdentifierValidator.validateCollectionName((String) props.get(NAME)); - final String shardsParam = (String) props.get(SHARDS_PROP); - if (StringUtils.isNotEmpty(shardsParam)) { - verifyShardsParam(shardsParam); - } - if (CollectionAdminParams.SYSTEM_COLL.equals(collectionName)) { - //We must always create a .system collection with only a single shard - props.put(NUM_SLICES, 1); - props.remove(SHARDS_PROP); - createSysConfigSet(h.coreContainer); - } - copyPropertiesWithPrefix(params, props, COLL_PROP_PREFIX); - Map result = copyPropertiesWithPrefix(params, props, "router."); - if (StringUtils.isNotBlank(prefix)) { - result = addPrefix(prefix, result); - } - return result; - } - private static Map addPrefix(String prefix, Map aMap) { - return aMap.entrySet().stream().collect(Collectors.toMap( entry -> prefix + entry.getKey(), Map.Entry::getValue)); - } private static void forceLeaderElection(SolrQueryRequest req, CollectionsHandler handler) { ClusterState clusterState = handler.coreContainer.getZkController().getClusterState(); @@ -1254,4 +1222,19 @@ public Collection getApis() { public Boolean registerV2() { return Boolean.TRUE; } + + /** + * Calls the consumer for each parameter and with all values. + * This may be more convenient than using the iterator. + */ + //TODO put on SolrParams, or maybe SolrParams should implement Iterable + private static void forEach(SolrParams params, BiConsumer consumer) { + //TODO do we add a predicate for the parameter as a filter? It would avoid calling getParams + final Iterator iterator = params.getParameterNamesIterator(); + while (iterator.hasNext()) { + String param = iterator.next(); + String[] values = params.getParams(param); + consumer.accept(param, values); + } + } } diff --git a/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java b/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java index f1a718dd5f61..02a6390b269c 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java +++ b/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java @@ -100,7 +100,7 @@ public Date getNOW() { return now; } - /** The TimeZone specified by the request, or null if none was specified */ + /** The TimeZone specified by the request, or UTC if none was specified */ public TimeZone getClientTimeZone() { if (tz == null) { tz = TimeZoneUtils.parseTimezone(req.getParams().get(CommonParams.TZ)); diff --git a/solr/core/src/java/org/apache/solr/util/DateMathParser.java b/solr/core/src/java/org/apache/solr/util/DateMathParser.java index 2124d1d189c3..f5af734d2051 100644 --- a/solr/core/src/java/org/apache/solr/util/DateMathParser.java +++ b/solr/core/src/java/org/apache/solr/util/DateMathParser.java @@ -217,12 +217,25 @@ private static LocalDateTime round(LocalDateTime t, String unit) { /** * Parses a String which may be a date (in the standard ISO-8601 format) * followed by an optional math expression. - * @param now an optional fixed date to use as "NOW" + * The TimeZone is taken from the {@code TZ} param retrieved via {@link SolrRequestInfo}, defaulting to UTC. + * @param now an optional fixed date to use as "NOW". {@link SolrRequestInfo} is consulted if unspecified. * @param val the string to parse */ + //TODO this API is a bit clumsy. "now" is rarely used. public static Date parseMath(Date now, String val) { + return parseMath(now, val, null); + } + + /** + * Parses a String which may be a date (in the standard ISO-8601 format) + * followed by an optional math expression. + * @param now an optional fixed date to use as "NOW" + * @param val the string to parse + * @param zone the timezone to use + */ + public static Date parseMath(Date now, String val, TimeZone zone) { String math; - final DateMathParser p = new DateMathParser(); + final DateMathParser p = new DateMathParser(zone); if (null != now) p.setNow(now); diff --git a/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java b/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java index 0600a83170f7..9ab49110e180 100644 --- a/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java +++ b/solr/core/src/java/org/apache/solr/util/TimeZoneUtils.java @@ -86,7 +86,7 @@ public static final TimeZone getTimeZone(final String ID) { /** * Parse the specified timezone ID. If null input then return UTC. If we can't resolve it then - * throw an exception. + * throw an exception. Does not return null. */ public static TimeZone parseTimezone(String tzStr) { if (tzStr != null) { diff --git a/solr/core/src/test-files/log4j.properties b/solr/core/src/test-files/log4j.properties index 26972038f9c5..969439a2287b 100644 --- a/solr/core/src/test-files/log4j.properties +++ b/solr/core/src/test-files/log4j.properties @@ -29,6 +29,8 @@ log4j.logger.org.apache.solr.hadoop=INFO #log4j.logger.org.apache.solr.common.cloud.ClusterStateUtil=DEBUG #log4j.logger.org.apache.solr.cloud.OverseerAutoReplicaFailoverThread=DEBUG +#log4j.logger.org.apache.http.wire=DEBUG +#log4j.logger.org.apache.http.headers=DEBUG #log4j.logger.org.apache.http.impl.conn.PoolingHttpClientConnectionManager=DEBUG #log4j.logger.org.apache.http.impl.conn.BasicClientConnectionManager=DEBUG #log4j.logger.org.apache.http=DEBUG diff --git a/solr/core/src/test/org/apache/solr/cloud/ConcurrentCreateRoutedAliasTest.java b/solr/core/src/test/org/apache/solr/cloud/ConcurrentCreateRoutedAliasTest.java index 9b8744de60b6..c750bba68c3b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ConcurrentCreateRoutedAliasTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ConcurrentCreateRoutedAliasTest.java @@ -18,11 +18,11 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.nio.file.Path; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -35,19 +35,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_MAX_SHARDS_PER_NODE; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_NUM_SHARDS; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_PULL_REPLICAS; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_REPLICATION_FACTOR; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_ROUTER_FIELD; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_ROUTER_NAME; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_SHARDS; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_TLOG_REPLICAS; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.ROUTING_FIELD; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.ROUTING_INCREMENT; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.ROUTING_MAX_FUTURE; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.ROUTING_TYPE; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateTimeRoutedAlias.ROUTING_FIELD; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateTimeRoutedAlias.ROUTING_INCREMENT; +import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateTimeRoutedAlias.ROUTING_TYPE; +@LuceneTestCase.Slow public class ConcurrentCreateRoutedAliasTest extends SolrTestCaseJ4 { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -87,7 +79,6 @@ public void testConcurrentCreateRoutedAliasMinimal() throws IOException, KeeperE final AtomicReference failure = new AtomicReference<>(); - final int timeToRunSec = 30; // Note: this number of threads seems to work regularly with the up-tweaked number of retries (50) in // org.apache.solr.common.cloud.ZkStateReader.AliasesManager.applyModificationAndExportToZk() @@ -108,16 +99,13 @@ public void testConcurrentCreateRoutedAliasMinimal() throws IOException, KeeperE int numStart = num; for (; num < threads.length + numStart; num++) { final String aliasName = "testAlias" + num; - uploadConfig(configset("_default"), aliasName); final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); final SolrClient solrClient = getHttpSolrClient(baseUrl); int i = num - numStart; - Map routingParams = getMinimalRoutingCommands(); - Map collectionParams = getMinimalCollectionParams(); threads[i] = new CreateRoutedAliasThread("create-delete-search-" + i, aliasName, "NOW/HOUR", - "UTC", routingParams, collectionParams, timeToRunSec, solrClient, failure, false); + solrClient, failure, false); } startAll(threads); @@ -130,23 +118,19 @@ public void testConcurrentCreateRoutedAliasMinimal() throws IOException, KeeperE @Test public void testConcurrentCreateRoutedAliasComplex() { final AtomicReference failure = new AtomicReference<>(); - final int timeToRunSec = 30; final CreateRoutedAliasThread[] threads = new CreateRoutedAliasThread[1]; int numStart = num; System.out.println("NUM ==> " +num); for (; num < threads.length + numStart; num++) { final String aliasName = "testAliasCplx" + num; - uploadConfig(configset("_default"), aliasName); final String baseUrl = solrCluster.getJettySolrRunners().get(0).getBaseUrl().toString(); final SolrClient solrClient = getHttpSolrClient(baseUrl); int i = num - numStart; - Map routingParams = getMinimalRoutingCommands(); - Map collectionParams = getComplicatedCollectionParams(); threads[i] = new CreateRoutedAliasThread("create-routed-alias-cplx-" + i, - aliasName, "2017-12-25_23_24_25","EST", routingParams, collectionParams, - timeToRunSec, solrClient, failure, true); + aliasName, "2017-12-25T23:24:25Z", + solrClient, failure, true); } startAll(threads); @@ -155,49 +139,6 @@ public void testConcurrentCreateRoutedAliasComplex() { assertNull("concurrent alias creation failed " + failure.get(), failure.get()); } - public Map getComplicatedCollectionParams() { - Map result = getMinimalCollectionParams(); - result.put(CREATE_COLLECTION_ROUTER_NAME, "implicit"); - result.put(CREATE_COLLECTION_ROUTER_FIELD, "implicit_s"); - result.put(CREATE_COLLECTION_SHARDS, "foo,bar"); - result.remove(CREATE_COLLECTION_NUM_SHARDS); - result.remove(CREATE_COLLECTION_REPLICATION_FACTOR); - result.put(CREATE_COLLECTION_REPLICATION_FACTOR,"2" ); - result.put(CREATE_COLLECTION_NUM_SHARDS,"2"); - result.put(CREATE_COLLECTION_MAX_SHARDS_PER_NODE, "4"); - result.put(CREATE_COLLECTION_PULL_REPLICAS, "2"); - result.put(CREATE_COLLECTION_TLOG_REPLICAS, "2"); - return result; - } - - static Map getMinimalCollectionParams() { - // needs to be V1 param name below - - HashMap cparams = new HashMap<>(); - cparams.put("create-collection.collection.configName", "_default"); - cparams.put(CREATE_COLLECTION_NUM_SHARDS,"1" ); - cparams.put(CREATE_COLLECTION_REPLICATION_FACTOR,"1" ); - return cparams; - } - - static Map getMinimalRoutingCommands() { - HashMap rparams = new HashMap<>(); - rparams.put(ROUTING_TYPE, "time"); - rparams.put(ROUTING_FIELD, "routedFoo_dt"); - rparams.put(ROUTING_INCREMENT, "+12HOUR"); - rparams.put(ROUTING_MAX_FUTURE, String.valueOf(1000 * 60 * 60)); - return rparams; - } - - - private void uploadConfig(Path configDir, String configName) { - try { - solrCluster.uploadConfigSet(configDir, configName); - } catch (IOException | KeeperException | InterruptedException e) { - throw new RuntimeException(e); - } - } - private void joinAll(final CreateRoutedAliasThread[] threads) { for (CreateRoutedAliasThread t : threads) { try { @@ -216,30 +157,19 @@ private void startAll(final Thread[] threads) { } private static class CreateRoutedAliasThread extends Thread { - protected final String aliasName; + final String aliasName; protected final String start; - protected final String tz; - protected final Map routingParams; - protected final Map collectionParams; - protected final long timeToRunSec; protected final SolrClient solrClient; protected final AtomicReference failure; - private final boolean v2; - public CreateRoutedAliasThread( - String name, String aliasName, String start, String tz, Map routingParams, Map collectionParams, long timeToRunSec, SolrClient solrClient, + CreateRoutedAliasThread( + String name, String aliasName, String start, SolrClient solrClient, AtomicReference failure, boolean v2) { super(name); this.aliasName = aliasName; this.start = start; - this.tz = tz; - this.routingParams = routingParams; - this.collectionParams = collectionParams; - this.timeToRunSec = timeToRunSec; this.solrClient = solrClient; this.failure = failure; - this.v2 = v2; } @Override @@ -247,11 +177,11 @@ public void run() { doWork(); } - protected void doWork() { + void doWork() { createAlias(); } - protected void addFailure(Exception e) { + void addFailure(Exception e) { log.error("Add Failure", e); synchronized (failure) { if (failure.get() != null) { @@ -264,8 +194,14 @@ protected void addFailure(Exception e) { private void createAlias() { try { - CollectionAdminRequest.CreateRoutedAlias rq = CollectionAdminRequest - .createRoutedAlias(aliasName, start, "UTC", getMinimalRoutingCommands(), getMinimalCollectionParams()); + CollectionAdminRequest.CreateTimeRoutedAlias rq = CollectionAdminRequest + .createTimeRoutedAlias( + aliasName, + start, + "+12HOUR", + "routedFoo_dt", + CollectionAdminRequest.createCollection("_ignored_", "_default", 1, 1) + ); final CollectionAdminResponse response = rq.process(solrClient); if (response.getStatus() != 0) { @@ -278,7 +214,7 @@ private void createAlias() { } - public void joinAndClose() throws InterruptedException { + void joinAndClose() throws InterruptedException { try { super.join(60000); } finally { diff --git a/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java b/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java index 3bfb6c481fef..cbed2ff763b0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java @@ -17,32 +17,32 @@ package org.apache.solr.cloud; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.Date; -import java.util.List; -import java.util.Locale; import java.util.Map; +import java.util.TimeZone; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.entity.ContentType; -import org.apache.http.entity.InputStreamEntity; +import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; import org.apache.lucene.util.IOUtils; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.common.cloud.Aliases; +import org.apache.solr.common.cloud.CompositeIdRouter; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.ImplicitDocRouter; +import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor; import org.apache.solr.util.DateMathParser; @@ -57,25 +57,34 @@ @SolrTestCaseJ4.SuppressSSL public class CreateRoutedAliasTest extends SolrCloudTestCase { - private CloudSolrClient solrClient; - private CloseableHttpClient httpclient; + private CloudSolrClient solrClient; + private CloseableHttpClient httpClient; @BeforeClass public static void setupCluster() throws Exception { configureCluster(2).configure(); + +// final Properties properties = new Properties(); +// properties.setProperty("immutable", "true"); // we won't modify it in this test +// new ConfigSetAdminRequest.Create() +// .setConfigSetName(configName) +// .setBaseConfigSetName("_default") +// .setNewConfigSetProperties(properties) +// .process(cluster.getSolrClient()); } @After - public void finish() throws Exception { - IOUtils.close(solrClient, httpclient); + public void finish() throws Exception { + IOUtils.close(solrClient, httpClient); } @Before public void doBefore() throws Exception { solrClient = getCloudSolrClient(cluster); - httpclient = HttpClients.createDefault(); - // delete aliases first to avoid problems such as: https://issues.apache.org/jira/browse/SOLR-11839 + httpClient = (CloseableHttpClient) solrClient.getHttpClient(); + // delete aliases first since they refer to the collections ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader(); + //TODO create an API to delete collections attached to the routed alias when the alias is removed zkStateReader.aliasesHolder.applyModificationAndExportToZk(aliases -> { Aliases a = zkStateReader.getAliases(); for (String alias : a.getCollectionAliasMap().keySet()) { @@ -83,21 +92,24 @@ public void doBefore() throws Exception { } return a; }); - for (String col : CollectionAdminRequest.listCollections(solrClient)) { - CollectionAdminRequest.deleteCollection(col).process(solrClient); - } + cluster.deleteAllCollections(); } + // This is a fairly complete test where we set many options and see that it both affected the created + // collection and that the alias metadata was saved accordingly @Test public void testV2() throws Exception { - final String aliasName = "testAlias"; - cluster.uploadConfigSet(configset("_default"), aliasName); + // note we don't use TZ in this test, thus it's UTC + final String aliasName = getTestName(); + + String createNode = cluster.getRandomJetty(random()).getNodeName(); + final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); + //TODO fix Solr test infra so that this /____v2/ becomes /api/ HttpPost post = new HttpPost(baseUrl + "/____v2/c"); - post.setHeader("Content-Type", ContentType.APPLICATION_JSON.getMimeType()); - HttpEntity httpEntity = new InputStreamEntity(org.apache.commons.io.IOUtils.toInputStream("{\n" + + post.setEntity(new StringEntity("{\n" + " \"create-routed-alias\" : {\n" + - " \"name\": \"testaliasV2\",\n" + + " \"name\": \"" + aliasName + "\",\n" + " \"router\" : {\n" + " \"name\": \"time\",\n" + " \"field\": \"evt_dt\",\n" + @@ -105,306 +117,252 @@ public void testV2() throws Exception { " \"max-future-ms\":\"14400000\"\n" + " },\n" + " \"start\":\"NOW/DAY\",\n" + // small window for test failure once a day. + //TODO should we use "NOW=" param? Won't work with v2 and is kinda a hack any way since intended for distrib " \"create-collection\" : {\n" + " \"router\": {\n" + " \"name\":\"implicit\",\n" + " \"field\":\"foo_s\"\n" + " },\n" + - " \"shards\":\"foo,bar,baz\",\n" + + " \"shards\":\"foo,bar\",\n" + " \"config\":\"_default\",\n" + - " \"numShards\": 2,\n" + " \"tlogReplicas\":1,\n" + " \"pullReplicas\":1,\n" + - " \"maxShardsPerNode\":3,\n" + + " \"maxShardsPerNode\":4,\n" + // note: we also expect the 'policy' to work fine + " \"nodeSet\": ['" + createNode + "'],\n" + " \"properties\" : {\n" + " \"foobar\":\"bazbam\"\n" + " }\n" + " }\n" + " }\n" + - "}", "UTF-8"), org.apache.http.entity.ContentType.APPLICATION_JSON); - post.setEntity(httpEntity); - try (CloseableHttpResponse response = httpclient.execute(post)) { - assertEquals(200, response.getStatusLine().getStatusCode()); - } - Date date = DateMathParser.parseMath(new Date(), "NOW/DAY"); + "}", ContentType.APPLICATION_JSON)); + assertSuccess(post); + + Date startDate = DateMathParser.parseMath(new Date(), "NOW/DAY"); String initialCollectionName = TimeRoutedAliasUpdateProcessor - .formatCollectionNameFromInstant("testaliasV2", date.toInstant(), + .formatCollectionNameFromInstant(aliasName, startDate.toInstant(), TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER); - HttpGet get = new HttpGet(baseUrl + "/____v2/c/"+ initialCollectionName); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(200, response.getStatusLine().getStatusCode()); - } - + // small chance could fail due to "NOW"; see above + assertCollectionExists(initialCollectionName); + + // Test created collection: + final DocCollection coll = solrClient.getClusterStateProvider().getState(initialCollectionName).get(); + //System.err.println(coll); + //TODO how do we assert the configSet ? + assertEquals(ImplicitDocRouter.class, coll.getRouter().getClass()); + assertEquals("foo_s", ((Map)coll.get("router")).get("field")); + assertEquals(2, coll.getSlices().size()); // numShards + assertEquals(4, coll.getSlices().stream() + .mapToInt(s -> s.getReplicas().size()).sum()); // num replicas + // we didn't ask for any NRT replicas + assertEquals(0, coll.getSlices().stream() + .mapToInt(s -> s.getReplicas(r -> r.getType() == Replica.Type.NRT).size()).sum()); + //assertEquals(1, coll.getNumNrtReplicas().intValue()); // TODO seems to be erroneous; I figured 'null' + assertEquals(1, coll.getNumTlogReplicas().intValue()); // per-shard + assertEquals(1, coll.getNumPullReplicas().intValue()); // per-shard + assertEquals(4, coll.getMaxShardsPerNode()); + //TODO SOLR-11877 assertEquals(2, coll.getStateFormat()); + assertTrue("nodeSet didn't work?", + coll.getSlices().stream().flatMap(s -> s.getReplicas().stream()) + .map(Replica::getNodeName).allMatch(createNode::equals)); + + // Test Alias metadata: Aliases aliases = cluster.getSolrClient().getZkStateReader().getAliases(); Map collectionAliasMap = aliases.getCollectionAliasMap(); - String alias = collectionAliasMap.get("testaliasV2"); - assertNotNull(alias); - Map meta = aliases.getCollectionAliasMetadata("testaliasV2"); - assertNotNull(meta); + assertEquals(initialCollectionName, collectionAliasMap.get(aliasName)); + Map meta = aliases.getCollectionAliasMetadata(aliasName); + //System.err.println(new TreeMap(meta)); assertEquals("evt_dt",meta.get("router.field")); - assertEquals("foo_s",meta.get("collection-create.router.field")); - assertEquals("bazbam",meta.get("collection-create.property.foobar")); + assertEquals("_default",meta.get("create-collection.collection.configName")); + assertEquals("foo_s",meta.get("create-collection.router.field")); + assertEquals("bazbam",meta.get("create-collection.property.foobar")); + assertEquals(createNode,meta.get("create-collection.createNodeSet")); } @Test public void testV1() throws Exception { - - final String aliasName = "testAlias"; - cluster.uploadConfigSet(configset("_default"), aliasName); + final String aliasName = getTestName(); final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis - String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); + Instant start = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + "&wt=xml" + - "&name=testalias" + + "&name=" + aliasName + "&router.field=evt_dt" + "&router.name=time" + - "&start=" + timestamp + + "&start=" + start + "&router.interval=%2B30MINUTE" + - "&router.max-future-ms=60000" + "&create-collection.collection.configName=_default" + - "&create-collection.numShards=2"); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(200, response.getStatusLine().getStatusCode()); - } + "&create-collection.router.field=foo_s" + + "&create-collection.numShards=1" + + "&create-collection.replicationFactor=2"); + assertSuccess(get); + String initialCollectionName = TimeRoutedAliasUpdateProcessor - .formatCollectionNameFromInstant("testalias", instant, + .formatCollectionNameFromInstant(aliasName, start, TimeRoutedAliasUpdateProcessor.DATE_TIME_FORMATTER); - get = new HttpGet(baseUrl + "/____v2/c/"+ initialCollectionName); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(200, response.getStatusLine().getStatusCode()); - } - + assertCollectionExists(initialCollectionName); + + // Test created collection: + final DocCollection coll = solrClient.getClusterStateProvider().getState(initialCollectionName).get(); + //TODO how do we assert the configSet ? + assertEquals(CompositeIdRouter.class, coll.getRouter().getClass()); + assertEquals("foo_s", ((Map)coll.get("router")).get("field")); + assertEquals(1, coll.getSlices().size()); // numShards + assertEquals(2, coll.getReplicationFactor().intValue()); // num replicas + //TODO SOLR-11877 assertEquals(2, coll.getStateFormat()); + + // Test Alias metadata Aliases aliases = cluster.getSolrClient().getZkStateReader().getAliases(); Map collectionAliasMap = aliases.getCollectionAliasMap(); - String alias = collectionAliasMap.get("testalias"); + String alias = collectionAliasMap.get(aliasName); assertNotNull(alias); - Map meta = aliases.getCollectionAliasMetadata("testalias"); + Map meta = aliases.getCollectionAliasMetadata(aliasName); assertNotNull(meta); assertEquals("evt_dt",meta.get("router.field")); - assertEquals("_default",meta.get("collection-create.collection.configName")); - assertEquals("2",meta.get("collection-create.numShards")); - assertEquals(null ,meta.get("start")); + assertEquals("_default",meta.get("create-collection.collection.configName")); + assertEquals(null,meta.get("start")); } - + // TZ should not affect the first collection name if absolute date given for start @Test public void testTimezoneAbsoluteDate() throws Exception { - - final String aliasName = "testAlias"; - cluster.uploadConfigSet(configset("_default"), aliasName); - final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis - String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); - HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + - "&wt=xml" + - "&name=testalias" + - "&router.field=evt_dt" + - "&router.name=time" + - "&start=2018-01-15T00:00:00Z" + - "&TZ=GMT-10" + - "&router.interval=%2B30MINUTE" + - "&router.max-future-ms=60000" + - "&create-collection.collection.configName=_default" + - "&create-collection.numShards=2"); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(200, response.getStatusLine().getStatusCode()); - } - //TODO: name like this because interval is in hours? - //assertInitialCollectionNameExists("testalias_2018-01-15_00_00"); - assertInitialCollectionNameExists("testalias_2018-01-15"); - //get = new HttpGet(baseUrl + "/____v2/c/testalias_2018-01-15_00_00"); - get = new HttpGet(baseUrl + "/____v2/c/testalias_2018-01-15"); - try (CloseableHttpResponse response = httpclient.execute(get)) { - System.out.println(responseToMap(response)); - System.out.println(getCollectionList()); - assertEquals(200, response.getStatusLine().getStatusCode()); + final String aliasName = getTestName(); + try (SolrClient client = getCloudSolrClient(cluster)) { + CollectionAdminRequest.createTimeRoutedAlias( + aliasName, + "2018-01-15T00:00:00Z", + "+30MINUTE", + "evt_dt", + CollectionAdminRequest.createCollection("_ignored_", "_default", 1, 1) + ) + .setTimeZone(TimeZone.getTimeZone("GMT-10")) + .process(client); } + assertCollectionExists(aliasName + "_2018-01-15"); } @Test public void testAliasNameMustBeValid() throws Exception { - - final String aliasName = "testAlias"; - cluster.uploadConfigSet(configset("_default"), aliasName); final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis - String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); - HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + "&wt=json" + "&name=735741!45" + // ! not allowed "&router.field=evt_dt" + "&router.name=time" + - "&start=" + timestamp + + "&start=2018-01-15T00:00:00Z" + "&router.interval=%2B30MINUTE" + - "&router.max-future-ms=60000" + "&create-collection.collection.configName=_default" + - "&create-collection.numShards=2"); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(400, response.getStatusLine().getStatusCode()); - assertErrorStartsWith(response,"Invalid alias"); - - } + "&create-collection.numShards=1"); + assertFailure(get, "Invalid alias"); } + @Test public void testRandomRouterNameFails() throws Exception { - - final String aliasName = "testAlias"; - cluster.uploadConfigSet(configset("_default"), aliasName); + final String aliasName = getTestName(); final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis - String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + "&wt=json" + - "&name=testalias" + + "&name=" + aliasName + "&router.field=evt_dt" + - "&router.name=tiafasme" + - "&start=" + timestamp + + "&router.name=tiafasme" + //bad + "&start=2018-01-15T00:00:00Z" + "&router.interval=%2B30MINUTE" + - "&router.max-future-ms=60000" + "&create-collection.collection.configName=_default" + - "&create-collection.numShards=2"); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(400, response.getStatusLine().getStatusCode()); - assertErrorStartsWith(response,"Only time based routing is supported"); - } + "&create-collection.numShards=1"); + assertFailure(get, "Only time based routing is supported"); } @Test public void testTimeStampWithMsFails() throws Exception { - final String aliasName = "testAlias"; - cluster.uploadConfigSet(configset("_default"), aliasName); + final String aliasName = getTestName(); final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS).plus(123,ChronoUnit.MILLIS); // mostly make sure no millis - String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + "&wt=json" + - "&name=testalias" + + "&name=" + aliasName + "&router.field=evt_dt" + "&router.name=time" + - "&start=" + timestamp + + "&start=2018-01-15T00:00:00.001Z" + // bad: no milliseconds permitted "&router.interval=%2B30MINUTE" + - "&router.max-future-ms=60000" + "&create-collection.collection.configName=_default" + - "&create-collection.numShards=2"); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(400, response.getStatusLine().getStatusCode()); - assertErrorStartsWith(response, "Date or date math for start time includes milliseconds"); - } - } - - - - private String getStringEntity(CloseableHttpResponse response) throws IOException { - ByteArrayOutputStream outstream = new ByteArrayOutputStream(); - response.getEntity().writeTo(outstream); - return new String(outstream.toByteArray(), StandardCharsets.UTF_8); + "&create-collection.numShards=1"); + assertFailure(get, "Date or date math for start time includes milliseconds"); } @Test public void testBadDateMathIntervalFails() throws Exception { - - final String aliasName = "testAlias"; - cluster.uploadConfigSet(configset("_default"), aliasName); + final String aliasName = getTestName(); final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis - String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + "&wt=json" + - "&name=testalias" + + "&name=" + aliasName + "&router.field=evt_dt" + "&router.name=time" + - "&start=" + timestamp + - "&router.interval=%2B30MINUTEx" + + "&start=2018-01-15T00:00:00Z" + + "&router.interval=%2B30MINUTEx" + // bad; trailing 'x' "&router.max-future-ms=60000" + "&create-collection.collection.configName=_default" + - "&create-collection.numShards=2"); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(400, response.getStatusLine().getStatusCode()); - assertErrorStartsWith(response,"Unit not recognized"); - } + "&create-collection.numShards=1"); + assertFailure(get, "Unit not recognized"); } @Test public void testNegativeFutureFails() throws Exception { - - final String aliasName = "testAlias"; - cluster.uploadConfigSet(configset("_default"), aliasName); + final String aliasName = getTestName(); final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis - String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + "&wt=json" + - "&name=testalias" + + "&name=" + aliasName + "&router.field=evt_dt" + "&router.name=time" + - "&start=" + timestamp + + "&start=2018-01-15T00:00:00Z" + "&router.interval=%2B30MINUTE" + - "&router.max-future-ms=-60000" + + "&router.max-future-ms=-60000" + // bad: negative "&create-collection.collection.configName=_default" + - "&create-collection.numShards=2"); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(400, response.getStatusLine().getStatusCode()); - assertErrorStartsWith(response, "router.max-future-ms must be a valid long integer"); - } + "&create-collection.numShards=1"); + assertFailure(get, "router.max-future-ms must be a valid long integer"); } @Test public void testUnParseableFutureFails() throws Exception { - final String aliasName = "testAlias"; - cluster.uploadConfigSet(configset("_default"), aliasName); final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - Instant instant = Instant.now().truncatedTo(ChronoUnit.HOURS); // mostly make sure no millis - String timestamp = DateTimeFormatter.ISO_INSTANT.format(instant); HttpGet get = new HttpGet(baseUrl + "/admin/collections?action=CREATEROUTEDALIAS" + "&wt=json" + - "&name=testalias" + + "&name=" + aliasName + "&router.field=evt_dt" + "&router.name=time" + - "&start=" + timestamp + + "&start=2018-01-15T00:00:00Z" + "&router.interval=%2B30MINUTE" + - "&router.max-future-ms=SixtyThousandMiliseconds" + + "&router.max-future-ms=SixtyThousandMiliseconds" + // bad "&create-collection.collection.configName=_default" + - "&create-collection.numShards=2"); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(400, response.getStatusLine().getStatusCode()); - assertErrorStartsWith(response, "router.max-future-ms must be a valid long integer"); - } + "&create-collection.numShards=1"); + assertFailure(get, "router.max-future-ms must be a valid long integer"); } - private void assertErrorStartsWith(CloseableHttpResponse response, String prefix) throws IOException { - Map map = responseToMap(response); - Map exception = (Map) map.get("exception"); - if (exception == null) { - exception = (Map) map.get("error"); + private void assertSuccess(HttpUriRequest msg) throws IOException { + try (CloseableHttpResponse response = httpClient.execute(msg)) { + if (200 != response.getStatusLine().getStatusCode()) { + System.err.println(EntityUtils.toString(response.getEntity())); + fail("Unexpected status: " + response.getStatusLine()); + } } - String msg = (String) exception.get("msg"); - assertTrue(msg.toLowerCase(Locale.ROOT).startsWith(prefix.toLowerCase(Locale.ROOT))); } - private Map responseToMap(CloseableHttpResponse response) throws IOException { - String entity = getStringEntity(response); - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(entity, new TypeReference>() {}); + private void assertFailure(HttpUriRequest msg, String expectedErrorSubstring) throws IOException { + try (CloseableHttpResponse response = httpClient.execute(msg)) { + assertEquals(400, response.getStatusLine().getStatusCode()); + String entity = EntityUtils.toString(response.getEntity()); + assertTrue("Didn't find expected error string within response: " + entity, + entity.contains(expectedErrorSubstring)); + } } - private void assertInitialCollectionNameExists(String name) throws IOException { - List collections; - collections = getCollectionList(); - assertTrue(name + " not found among existing collections:" + collections,collections.contains(name)); - } + private void assertCollectionExists(String name) throws IOException, SolrServerException { + solrClient.getClusterStateProvider().connect(); // TODO get rid of this + // https://issues.apache.org/jira/browse/SOLR-9784?focusedCommentId=16332729 - private List getCollectionList() throws IOException { - List collections; - final String baseUrl = cluster.getRandomJetty(random()).getBaseUrl().toString(); - HttpGet get = new HttpGet(baseUrl + "/____v2/c"); - try (CloseableHttpResponse response = httpclient.execute(get)) { - assertEquals(200, response.getStatusLine().getStatusCode()); - Map map = responseToMap(response); - collections = (List) map.get("collections"); - } - return collections; + assertNotNull(name + " not found", solrClient.getClusterStateProvider().getState(name)); + // note: could also do: + //List collections = CollectionAdminRequest.listCollections(solrClient); } + // not testing collection parameters, those should inherit error checking from the collection creation code. } diff --git a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java index 2ba380eeea83..943996c2311d 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java @@ -78,23 +78,6 @@ public static void finish() throws Exception { IOUtils.close(solrClient); } - //TODO this is necessary when -Dtests.iters but why? Some other tests aren't affected - @Before - public void doBefore() throws Exception { - // delete aliases first to avoid problems such as: https://issues.apache.org/jira/browse/SOLR-11839 - ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader(); - zkStateReader.aliasesHolder.applyModificationAndExportToZk(aliases -> { - Aliases a = zkStateReader.getAliases(); - for (String alias : a.getCollectionAliasMap().keySet()) { - a = a.cloneWithCollectionAlias(alias,null); // remove - } - return a; - }); - for (String col : CollectionAdminRequest.listCollections(solrClient)) { - CollectionAdminRequest.deleteCollection(col).process(solrClient); - } - } - @Test public void test() throws Exception { // First create a config using REST API. To do this, we create a collection with the name of the eventual config. diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java index b623157dad20..deb8fbc123c1 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java @@ -139,6 +139,7 @@ private ClusterState fetchClusterState(SolrClient client, String collection, Map Set liveNodes = new HashSet((List)(cluster.get("live_nodes"))); this.liveNodes = liveNodes; liveNodesTimestamp = System.nanoTime(); + //TODO SOLR-11877 we don't know the znode path; CLUSTER_STATE is probably wrong leading to bad stateFormat ClusterState cs = ClusterState.load(znodeVersion, collectionsMap, liveNodes, ZkStateReader.CLUSTER_STATE); if (clusterProperties != null) { Map properties = (Map) cluster.get("properties"); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index 9fdc6d3f7714..a525525f3fde 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -18,9 +18,11 @@ import java.io.IOException; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.TimeZone; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -43,6 +45,7 @@ import org.apache.solr.common.params.CommonAdminParams; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; @@ -1359,97 +1362,91 @@ public SolrParams getParams() { /** * Returns a SolrRequest to create a routed alias. Only time based routing is supported presently, - * For time based routing, the start is a timestamp or date math. + * For time based routing, the start a standard Solr timestamp string (possibly with "date math"). * * @param aliasName the name of the alias to create. - * @param start the the start of the routing. - * @param tz the timezone on which to base date math (for time based routing) - * @param routingParams all router.* parameters, including the 'router.' prefix - * @param collectionParams all create-collection.* parameters including the 'create-collection.' prefix - * + * @param start the start of the routing. A standard Solr date: ISO-8601 or NOW with date math. + * @param interval date math representing the time duration of each collection (e.g. {@code +1DAY}) + * @param routerField the document field to contain the timestamp to route on + * @param createCollTemplate Holds options to create a collection. The "name" is ignored. */ - public static CreateRoutedAlias createRoutedAlias(String aliasName, String start, String tz, - Map routingParams, - Map collectionParams) { - return new CreateRoutedAlias(aliasName,start,tz,routingParams,collectionParams); + public static CreateTimeRoutedAlias createTimeRoutedAlias(String aliasName, String start, + String interval, + String routerField, + Create createCollTemplate) { + + return new CreateTimeRoutedAlias(aliasName, routerField, start, interval, createCollTemplate); } - public static class CreateRoutedAlias extends AsyncCollectionAdminRequest { + public static class CreateTimeRoutedAlias extends AsyncCollectionAdminRequest { // TODO: This and other commands in this file seem to need to share some sort of constants class with core // to allow this stuff not to be duplicated. (this is pasted from CreateAliasCmd.java), however I think // a comprehensive cleanup of this for all the requests in this class should be done as a separate ticket. - public static final String ROUTING_TYPE = "router.name"; public static final String ROUTING_FIELD = "router.field"; public static final String ROUTING_INCREMENT = "router.interval"; public static final String ROUTING_MAX_FUTURE = "router.max-future-ms"; public static final String START = "start"; - public static final String CREATE_COLLECTION_CONFIG = "create-collection.config"; - public static final String CREATE_COLLECTION_ROUTER_NAME = "create-collection.router.name"; - public static final String CREATE_COLLECTION_ROUTER_FIELD = "create-collection.router.field"; - public static final String CREATE_COLLECTION_NUM_SHARDS = "create-collection.numShards"; - public static final String CREATE_COLLECTION_SHARDS = "create-collection.shards"; - public static final String CREATE_COLLECTION_REPLICATION_FACTOR = "create-collection.replicationFactor"; - public static final String CREATE_COLLECTION_NRT_REPLICAS = "create-collection.nrtReplicas"; - public static final String CREATE_COLLECTION_TLOG_REPLICAS = "create-collection.tlogReplicas"; - public static final String CREATE_COLLECTION_PULL_REPLICAS = "create-collection.pullReplicas"; - public static final String CREATE_COLLECTION_NODE_SET = "create-collection.nodeSet"; - public static final String CREATE_COLLECTION_SHUFFLE_NODES = "create-collection.shuffleNodes"; - public static final String CREATE_COLLECTION_MAX_SHARDS_PER_NODE = "create-collection.maxShardsPerNode"; - public static final String CREATE_COLLECTION_AUTO_ADD_REPLICAS = "create-collection.autoAddReplicas"; - public static final String CREATE_COLLECTION_RULE = "create-collection.rule"; - public static final String CREATE_COLLECTION_SNITCH = "create-collection.snitch"; - public static final String CREATE_COLLECTION_POLICY = "create-collection.policy"; - public static final String CREATE_COLLECTION_PROPERTIES = "create-collection.properties"; private final String aliasName; + private final String routerField; private final String start; - private final String tz; - private final Map routingParams; - private final Map collectionParams; + private final String interval; + //Optional: + private TimeZone tz; + private Integer maxFutureMs; + private final Create createCollTemplate; - public CreateRoutedAlias(String aliasName, String start, String tz, - Map routingParams, - Map collectionParams) { + public CreateTimeRoutedAlias(String aliasName, String routerField, String start, String interval, Create createCollTemplate) { super(CollectionAction.CREATEROUTEDALIAS); this.aliasName = aliasName; this.start = start; + this.interval = interval; + this.routerField = routerField; + this.createCollTemplate = createCollTemplate; + } + + /** Sets the timezone for interpreting any Solr "date math. */ + public CreateTimeRoutedAlias setTimeZone(TimeZone tz) { this.tz = tz; - this.routingParams = routingParams; - this.collectionParams = collectionParams; + return this; + } + + /** Sets how long into the future (millis) that we will allow a document to pass. */ + public CreateTimeRoutedAlias setMaxFutureMs(Integer maxFutureMs) { + this.maxFutureMs = maxFutureMs; + return this; } @Override public SolrParams getParams() { ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); params.add(CommonParams.NAME, aliasName); + params.add(ROUTING_TYPE, "time"); + params.add(ROUTING_FIELD, routerField); params.add(START, start); - params.add(CommonParams.TZ, tz ); - // these need to be V1 params, not V2 - params.add(ROUTING_FIELD, routingParams.get(ROUTING_FIELD)); - params.add(ROUTING_INCREMENT, routingParams.get(ROUTING_INCREMENT)); - params.add(ROUTING_MAX_FUTURE, routingParams.get(ROUTING_MAX_FUTURE)); - params.add(ROUTING_TYPE, routingParams.get(ROUTING_TYPE)); - params.add("create-collection.collection.configName", collectionParams.get("create-collection.collection.configName")); - params.add(CREATE_COLLECTION_ROUTER_NAME, collectionParams.get(CREATE_COLLECTION_ROUTER_NAME)); - params.add(CREATE_COLLECTION_ROUTER_FIELD, collectionParams.get(CREATE_COLLECTION_ROUTER_FIELD)); - params.add(CREATE_COLLECTION_AUTO_ADD_REPLICAS, collectionParams.get(CREATE_COLLECTION_AUTO_ADD_REPLICAS)); - params.add(CREATE_COLLECTION_MAX_SHARDS_PER_NODE, collectionParams.get(CREATE_COLLECTION_MAX_SHARDS_PER_NODE)); - params.add(CREATE_COLLECTION_NODE_SET, collectionParams.get(CREATE_COLLECTION_NODE_SET)); - params.add(CREATE_COLLECTION_NRT_REPLICAS, collectionParams.get(CREATE_COLLECTION_NRT_REPLICAS)); - params.add(CREATE_COLLECTION_NUM_SHARDS, collectionParams.get(CREATE_COLLECTION_NUM_SHARDS)); - params.add(CREATE_COLLECTION_POLICY, collectionParams.get(CREATE_COLLECTION_POLICY)); - params.add(CREATE_COLLECTION_PROPERTIES, collectionParams.get(CREATE_COLLECTION_PROPERTIES)); - params.add(CREATE_COLLECTION_PULL_REPLICAS, collectionParams.get(CREATE_COLLECTION_PULL_REPLICAS)); - params.add(CREATE_COLLECTION_REPLICATION_FACTOR, collectionParams.get(CREATE_COLLECTION_REPLICATION_FACTOR)); - params.add(CREATE_COLLECTION_RULE, collectionParams.get(CREATE_COLLECTION_RULE)); - params.add(CREATE_COLLECTION_SHARDS, collectionParams.get(CREATE_COLLECTION_SHARDS)); - params.add(CREATE_COLLECTION_SNITCH, collectionParams.get(CREATE_COLLECTION_SNITCH)); - params.add(CREATE_COLLECTION_SHUFFLE_NODES, collectionParams.get(CREATE_COLLECTION_SHUFFLE_NODES)); - params.add(CREATE_COLLECTION_TLOG_REPLICAS, collectionParams.get(CREATE_COLLECTION_TLOG_REPLICAS)); - return params; + params.add(ROUTING_INCREMENT, interval); + if (tz != null) { + params.add(CommonParams.TZ, tz.getID()); + } + if (maxFutureMs != null) { + params.add(ROUTING_MAX_FUTURE, ""+maxFutureMs); + } + + // merge the above with collectionParams. Above takes precedence. + ModifiableSolrParams createCollParams = new ModifiableSolrParams(); // output target + final SolrParams collParams = createCollTemplate.getParams(); + final Iterator pIter = collParams.getParameterNamesIterator(); + while (pIter.hasNext()) { + String key = pIter.next(); + if (key.equals(CollectionParams.ACTION) || key.equals("name")) { + continue; + } + createCollParams.set("create-collection." + key, collParams.getParams(key)); + } + return SolrParams.wrapDefaults(params, createCollParams); } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java index 46ce712557f0..e780608d9081 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java @@ -21,8 +21,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.common.params.CollectionParams.CollectionAction; @@ -34,9 +37,6 @@ import static org.apache.solr.client.solrj.SolrRequest.METHOD.DELETE; import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET; import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_CONFIG; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_NODE_SET; -import static org.apache.solr.client.solrj.request.CollectionAdminRequest.CreateRoutedAlias.CREATE_COLLECTION_SHUFFLE_NODES; import static org.apache.solr.client.solrj.request.CollectionApiMapping.ConfigSetEndPoint.CONFIG_COMMANDS; import static org.apache.solr.client.solrj.request.CollectionApiMapping.ConfigSetEndPoint.CONFIG_DEL; import static org.apache.solr.client.solrj.request.CollectionApiMapping.ConfigSetEndPoint.LIST_CONFIG; @@ -120,11 +120,15 @@ public enum Meta implements CommandMeta { POST, CREATEROUTEDALIAS, "create-routed-alias", - Utils.makeMap( - "create-collection.collection.configName", CREATE_COLLECTION_CONFIG, - "createNodeSet",CREATE_COLLECTION_NODE_SET, - "create-collection.createNodeSet.shuffle", CREATE_COLLECTION_SHUFFLE_NODES + // same as the CREATE_COLLECTION but with "create-collection" prefix + CREATE_COLLECTION.paramstoAttr.entrySet().stream().collect(Collectors.toMap( + entry -> "create-collection." + entry.getKey(), + entry -> "create-collection." + entry.getValue() )), + CREATE_COLLECTION.prefixSubstitutes.entrySet().stream().collect(Collectors.toMap( + entry -> "create-collection." + entry.getKey(), + entry -> "create-collection." + entry.getValue() + ))), DELETE_ALIAS(COLLECTIONS_COMMANDS, POST, @@ -215,10 +219,11 @@ public String getParamSubstitute(String param) { public final String commandName; public final EndPoint endPoint; public final SolrRequest.METHOD method; - //mapping of http param name to json attribute + //mapping of http param name to json attribute (v1 -> v2) public final Map paramstoAttr; + public final Map attrToParams; //mapping of old prefix to new for instance properties.a=val can be substituted with property:{a:val} - public final Map prefixSubstitutes; + public final Map prefixSubstitutes; // v2 -> v1 public final CollectionAction action; public SolrRequest.METHOD getMethod() { @@ -243,6 +248,10 @@ public SolrRequest.METHOD getMethod() { this.endPoint = endPoint; this.method = method; this.paramstoAttr = paramstoAttr == null ? Collections.EMPTY_MAP : Collections.unmodifiableMap(paramstoAttr); + // flip around + this.attrToParams = Collections.unmodifiableMap( + this.paramstoAttr.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)) + ); this.prefixSubstitutes = Collections.unmodifiableMap(prefixSubstitutes); } @@ -262,30 +271,29 @@ public V2EndPoint getEndPoint() { return endPoint; } - @Override - public Collection getParamNames(CommandOperation op) { + public Iterator getParamNamesIterator(CommandOperation op) { Collection paramNames = getParamNames_(op, this); + Stream pStream = paramNames.stream(); + if (!attrToParams.isEmpty()) { + pStream = pStream.map(paramName -> attrToParams.getOrDefault(paramName, paramName)); + } if (!prefixSubstitutes.isEmpty()) { - Collection result = new ArrayList<>(paramNames.size()); - for (Map.Entry e : prefixSubstitutes.entrySet()) { - for (String paramName : paramNames) { + pStream = pStream.map(paramName -> { + for (Map.Entry e : prefixSubstitutes.entrySet()) { if (paramName.startsWith(e.getKey())) { - result.add(paramName.replace(e.getKey(), e.getValue())); - } else { - result.add(paramName); + return paramName.replace(e.getKey(), e.getValue()); } } - paramNames = result; - } + return paramName; + }); } - - return paramNames; + return pStream.iterator(); } @Override public String getParamSubstitute(String param) { - String s = paramstoAttr.containsKey(param) ? paramstoAttr.get(param) : param; + String s = paramstoAttr.getOrDefault(param, param); if (prefixSubstitutes != null) { for (Map.Entry e : prefixSubstitutes.entrySet()) { if (s.startsWith(e.getValue())) return s.replace(e.getValue(), e.getKey()); @@ -294,7 +302,7 @@ public String getParamSubstitute(String param) { return s; } public Object getReverseParamSubstitute(String param) { - String s = paramstoAttr.containsKey(param) ? paramstoAttr.get(param) : param; + String s = paramstoAttr.getOrDefault(param, param); if (prefixSubstitutes != null) { for (Map.Entry e : prefixSubstitutes.entrySet()) { @@ -398,14 +406,15 @@ public String getSpecName() { private static Collection getParamNames_(CommandOperation op, CommandMeta command) { - List result = new ArrayList<>(); Object o = op.getCommandData(); if (o instanceof Map) { Map map = (Map) o; + List result = new ArrayList<>(); collectKeyNames(map, result, ""); + return result; + } else { + return Collections.emptySet(); } - return result; - } public static void collectKeyNames(Map map, List result, String prefix) { @@ -427,11 +436,10 @@ public interface CommandMeta { V2EndPoint getEndPoint(); - default Collection getParamNames(CommandOperation op) { - return getParamNames_(op, CommandMeta.this); + default Iterator getParamNamesIterator(CommandOperation op) { + return getParamNames_(op, CommandMeta.this).iterator(); } - default String getParamSubstitute(String name) { return name; } diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index f11339ce533d..91d9758f8806 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -1460,7 +1460,7 @@ public void applyModificationAndExportToZk(UnaryOperator op) { final long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); // up-tweaked to better handle ConcurrentCreateRoutedAliasTest.testConcurrentCreateRoutedAliasMinimal() // This may be too aggressive but the impact on that test should be validated before changing this. - int triesLeft = 50; + int triesLeft = 50;//nocommit while (triesLeft > 0) { triesLeft--; // we could synchronize on "this" but there doesn't seem to be a point; we have a retry loop. diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java b/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java index 9f9231417345..2a8d2d179d79 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/JsonSchemaValidator.java @@ -236,7 +236,8 @@ boolean validate( Object o, List errs, Set requiredProps) { for (String requiredProp : requiredProps) { if (requiredProp.contains(".")) { if (requiredProp.endsWith(".")) { - errs.add("Illegal required attribute name (ends with '.'" + requiredProp + ") This is a bug."); + errs.add("Illegal required attribute name (ends with '.': " + requiredProp + "). This is a bug."); + return false; } String subprop = requiredProp.substring(requiredProp.indexOf(".") + 1); if (!validate(((Map)o).get(requiredProp), errs, Collections.singleton(subprop))) {