Skip to content

Commit

Permalink
Add exactMatch comparison method for work buckets
Browse files Browse the repository at this point in the history
  • Loading branch information
mederly committed Jun 13, 2019
1 parent 9d86044 commit 80f5979
Show file tree
Hide file tree
Showing 7 changed files with 292 additions and 13 deletions.
Expand Up @@ -1501,7 +1501,7 @@
<xsd:annotation>
<xsd:documentation>
Work state management strategy that divides the processing space (represented by a set of string
values) into a number of work buckets; either based on intervals or prefixes.
values) into a number of work buckets; either based on intervals, prefixes or exact value matching.
</xsd:documentation>
<xsd:appinfo>
<a:since>3.8</a:since>
Expand All @@ -1513,7 +1513,7 @@
<xsd:element name="boundaryCharacters" type="xsd:string" minOccurs="0" maxOccurs="unbounded">
<xsd:annotation>
<xsd:documentation>
Characters that make up the prefix (or interval). These characters must be sorted.
Characters that make up the boundaries. These characters must be sorted.
Reserved characters: '-', '$' (to be implemented later)
Escaping character: '\'
</xsd:documentation>
Expand All @@ -1531,7 +1531,7 @@
<xsd:element name="comparisonMethod" type="tns:StringWorkBucketsBoundaryMarkingType" minOccurs="0">
<xsd:annotation>
<xsd:documentation>
Whether to use intervals or prefixes. Former is the default.
Whether to use intervals, prefixes or exact value match. Interval matching is the default.
</xsd:documentation>
</xsd:annotation>
</xsd:element>
Expand Down Expand Up @@ -1561,7 +1561,7 @@
<xsd:simpleType name="StringWorkBucketsBoundaryMarkingType">
<xsd:annotation>
<xsd:documentation>
TODO
How to construct work bucket filters based on defined boundary values.
</xsd:documentation>
<xsd:appinfo>
<a:since>3.8</a:since>
Expand All @@ -1571,7 +1571,7 @@
<xsd:enumeration value="interval">
<xsd:annotation>
<xsd:documentation>
TODO
Use greater-than/less-than comparison.
</xsd:documentation>
<xsd:appinfo>
<jaxb:typesafeEnumMember name="INTERVAL"/>
Expand All @@ -1581,13 +1581,25 @@
<xsd:enumeration value="prefix">
<xsd:annotation>
<xsd:documentation>
TODO
Use prefix matching.
</xsd:documentation>
<xsd:appinfo>
<jaxb:typesafeEnumMember name="PREFIX"/>
</xsd:appinfo>
</xsd:annotation>
</xsd:enumeration>
<xsd:enumeration value="exactMatch">
<xsd:annotation>
<xsd:documentation>
Use exact value matching. This is quite risky and should be used only when you are absolutely sure that
boundary values cover all possible values of the discriminator.
</xsd:documentation>
<xsd:appinfo>
<jaxb:typesafeEnumMember name="EXACT_MATCH"/>
<a:since>4.0</a:since>
</xsd:appinfo>
</xsd:annotation>
</xsd:enumeration>
</xsd:restriction>
</xsd:simpleType>

Expand Down Expand Up @@ -1747,6 +1759,32 @@
</xsd:complexContent>
</xsd:complexType>

<!-- Currently the same content as StringPrefixWorkBucketContentType but let's keep these separate fro conceptual
as well as for practical reasons (the latter ones being mainly backwards compatibility). -->
<xsd:complexType name="StringValueWorkBucketContentType">
<xsd:annotation>
<xsd:documentation>
Work bucket content defined using string value.
</xsd:documentation>
<xsd:appinfo>
<a:since>4.0</a:since>
</xsd:appinfo>
</xsd:annotation>
<xsd:complexContent>
<xsd:extension base="tns:AbstractWorkBucketContentType">
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" maxOccurs="unbounded">
<xsd:annotation>
<xsd:documentation>
Value(s) that are part of this bucket.
</xsd:documentation>
</xsd:annotation>
</xsd:element>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>

<xsd:complexType name="StringIntervalWorkBucketContentType">
<xsd:annotation>
<xsd:documentation>
Expand Down
Expand Up @@ -5294,4 +5294,25 @@ public void test1434QuerySearchForObjectType() throws Exception {
}
}

@Test
public void test1435QueryNameNormAsString() throws Exception {
Session session = open();

try {
ObjectQuery query = prismContext.queryFor(UserType.class)
.item(F_NAME).eq("asdf").matchingNorm().build();

String expected = "select\n" +
" u.oid, u.fullObject, u.stringsCount, u.longsCount, u.datesCount, u.referencesCount, u.polysCount, u.booleansCount\n" +
"from\n" +
" RUser u\n" +
"where\n" +
" u.nameCopy.norm = :norm";

String real = getInterpretedQuery2(session, UserType.class, query);
assertEqualsIgnoreWhitespace(expected, real);
} finally {
close(session);
}
}
}
Expand Up @@ -30,7 +30,6 @@
import java.util.stream.Collectors;

import static com.evolveum.midpoint.xml.ns._public.common.common_3.StringWorkBucketsBoundaryMarkingType.INTERVAL;
import static com.evolveum.midpoint.xml.ns._public.common.common_3.StringWorkBucketsBoundaryMarkingType.PREFIX;
import static java.util.Collections.singletonList;
import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;

Expand All @@ -57,12 +56,11 @@ public StringWorkSegmentationStrategy(@NotNull TaskWorkManagementType configurat

@Override
protected AbstractWorkBucketContentType createAdditionalBucket(AbstractWorkBucketContentType lastBucketContent, Integer lastBucketSequentialNumber) {
if (marking == INTERVAL) {
return createAdditionalIntervalBucket(lastBucketContent, lastBucketSequentialNumber);
} else if (marking == PREFIX) {
return createAdditionalPrefixBucket(lastBucketContent, lastBucketSequentialNumber);
} else {
throw new AssertionError("unsupported marking: " + marking);
switch (marking) {
case INTERVAL: return createAdditionalIntervalBucket(lastBucketContent, lastBucketSequentialNumber);
case PREFIX: return createAdditionalPrefixBucket(lastBucketContent, lastBucketSequentialNumber);
case EXACT_MATCH: return createAdditionalExactMatchBucket(lastBucketContent, lastBucketSequentialNumber);
default: throw new AssertionError("unsupported marking: " + marking);
}
}

Expand Down Expand Up @@ -111,6 +109,32 @@ private AbstractWorkBucketContentType createAdditionalPrefixBucket(AbstractWorkB
}
}

private AbstractWorkBucketContentType createAdditionalExactMatchBucket(AbstractWorkBucketContentType lastBucketContent, Integer lastBucketSequentialNumber) {
String lastBoundary;
if (lastBucketSequentialNumber != null) {
if (!(lastBucketContent instanceof StringValueWorkBucketContentType)) {
throw new IllegalStateException("Null or unsupported bucket content: " + lastBucketContent);
}
StringValueWorkBucketContentType lastContent = (StringValueWorkBucketContentType) lastBucketContent;
if (lastContent.getValue().size() > 1) {
throw new IllegalStateException("Multiple values are not supported now: " + lastContent);
} else if (lastContent.getValue().isEmpty()) {
return null;
} else {
lastBoundary = lastContent.getValue().get(0);
}
} else {
lastBoundary = null;
}
String nextBoundary = computeNextBoundary(lastBoundary);
if (nextBoundary != null) {
return new StringValueWorkBucketContentType()
.value(nextBoundary);
} else {
return null;
}
}

private String computeNextBoundary(String lastBoundary) {
List<Integer> currentIndices = stringToIndices(lastBoundary);
if (incrementIndices(currentIndices)) {
Expand Down
Expand Up @@ -45,6 +45,7 @@ public void register() {
registry.registerHandler(StringPrefixWorkBucketContentType.class, this);
}

@SuppressWarnings("Duplicates")
@NotNull
@Override
public List<ObjectFilter> createSpecificFilters(@NotNull WorkBucketType bucket, AbstractWorkSegmentationType configuration,
Expand Down
@@ -0,0 +1,85 @@
/*
* Copyright (c) 2010-2019 Evolveum
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.evolveum.midpoint.task.quartzimpl.work.segmentation.content;

import com.evolveum.midpoint.prism.ItemDefinition;
import com.evolveum.midpoint.prism.PrismConstants;
import com.evolveum.midpoint.prism.path.ItemPath;
import com.evolveum.midpoint.prism.query.ObjectFilter;
import com.evolveum.midpoint.util.QNameUtil;
import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import javax.xml.namespace.QName;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.function.Function;

import static java.util.Collections.emptyList;

/**
*
*/
@Component
public class StringValueWorkBucketContentHandler extends BaseWorkBucketContentHandler {

@PostConstruct
public void register() {
registry.registerHandler(StringValueWorkBucketContentType.class, this);
}

@SuppressWarnings("Duplicates")
@NotNull
@Override
public List<ObjectFilter> createSpecificFilters(@NotNull WorkBucketType bucket, AbstractWorkSegmentationType configuration,
Class<? extends ObjectType> type, Function<ItemPath, ItemDefinition<?>> itemDefinitionProvider) {

StringValueWorkBucketContentType content = (StringValueWorkBucketContentType) bucket.getContent();

if (content == null || content.getValue().isEmpty()) {
return emptyList();
}
if (configuration == null) {
throw new IllegalStateException("No buckets configuration but having defined bucket content: " + content);
}
if (configuration.getDiscriminator() == null) {
throw new IllegalStateException("No buckets discriminator defined; bucket content = " + content);
}
ItemPath discriminator = configuration.getDiscriminator().getItemPath();
ItemDefinition<?> discriminatorDefinition = itemDefinitionProvider != null ? itemDefinitionProvider.apply(discriminator) : null;

QName matchingRuleName = configuration.getMatchingRule() != null
? QNameUtil.uriToQName(configuration.getMatchingRule(), PrismConstants.NS_MATCHING_RULE)
: null;

List<ObjectFilter> prefixFilters = new ArrayList<>();
for (String prefix : content.getValue()) {
prefixFilters.add(prismContext.queryFor(type)
.item(discriminator, discriminatorDefinition).eq(prefix).matching(matchingRuleName)
.buildFilter());
}
assert !prefixFilters.isEmpty();
if (prefixFilters.size() > 1) {
return Collections.singletonList(prismContext.queryFactory().createOr(prefixFilters));
} else {
return prefixFilters;
}
}
}
Expand Up @@ -338,6 +338,62 @@ public void test120StringPrefixBuckets() throws Exception {
suspendAndDeleteTasks(task.getOid());
}

@Test
public void test125StringExactValueBuckets() throws Exception {
final String TEST_NAME = "test125StringExactValueBuckets";
OperationResult result = createResult(TEST_NAME, LOGGER);
addObjectFromFile(taskFilename(TEST_NAME));

TaskQuartzImpl task = taskManager.getTask(taskOid(TEST_NAME), result);

// WHEN
WorkSegmentationStrategy segmentationStrategy = strategyFactory.createStrategy(task.getWorkManagement());
TaskWorkStateType workState = new TaskWorkStateType(prismContext);

// WHEN+THEN
// a, 01abc, 01abc
StringWorkSegmentationStrategy stringStrategy = (StringWorkSegmentationStrategy) segmentationStrategy;
assertEquals("Wrong expanded boundaries", Arrays.asList("a", "01abc", "01abc"), stringStrategy.getBoundaries());
assertEquals("Wrong # of estimated buckets", Integer.valueOf(25), segmentationStrategy.estimateNumberOfBuckets(null));

WorkBucketType bucket = assumeNextValue(segmentationStrategy, workState, "a00", 1);
ObjectQuery narrowedQuery = workStateManager
.narrowQueryForWorkBucket(task, null, UserType.class, null, bucket, result);
display("narrowed query (1)", narrowedQuery);
ObjectQuery expectedQuery = prismContext.queryFor(UserType.class)
.item(UserType.F_NAME).eq("a00").matchingNorm()
.build();
PrismAsserts.assertQueriesEquivalent("Wrong narrowed query (1)", expectedQuery, narrowedQuery);

assumeNextValue(segmentationStrategy, workState, "a01", 2);
assumeNextValue(segmentationStrategy, workState, "a0a", 3);
assumeNextValue(segmentationStrategy, workState, "a0b", 4);
assumeNextValue(segmentationStrategy, workState, "a0c", 5);
assumeNextValue(segmentationStrategy, workState, "a10", 6);
assumeNextValue(segmentationStrategy, workState, "a11", 7);
assumeNextValue(segmentationStrategy, workState, "a1a", 8);
assumeNextValue(segmentationStrategy, workState, "a1b", 9);
assumeNextValue(segmentationStrategy, workState, "a1c", 10);
assumeNextValue(segmentationStrategy, workState, "aa0", 11);
assumeNextValue(segmentationStrategy, workState, "aa1", 12);
assumeNextValue(segmentationStrategy, workState, "aaa", 13);
assumeNextValue(segmentationStrategy, workState, "aab", 14);
assumeNextValue(segmentationStrategy, workState, "aac", 15);
assumeNextValue(segmentationStrategy, workState, "ab0", 16);
assumeNextValue(segmentationStrategy, workState, "ab1", 17);
assumeNextValue(segmentationStrategy, workState, "aba", 18);
assumeNextValue(segmentationStrategy, workState, "abb", 19);
assumeNextValue(segmentationStrategy, workState, "abc", 20);
assumeNextValue(segmentationStrategy, workState, "ac0", 21);
assumeNextValue(segmentationStrategy, workState, "ac1", 22);
assumeNextValue(segmentationStrategy, workState, "aca", 23);
assumeNextValue(segmentationStrategy, workState, "acb", 24);
assumeNextValue(segmentationStrategy, workState, "acc", 25);
assumeNoNextBucket(segmentationStrategy, workState);

suspendAndDeleteTasks(task.getOid());
}

@Test
public void test130StringIntervalBuckets() throws Exception {
final String TEST_NAME = "test130StringIntervalBuckets";
Expand Down Expand Up @@ -426,6 +482,19 @@ public void test150OidBucketsTwice() throws Exception {
suspendAndDeleteTasks(task.getOid());
}

private WorkBucketType assumeNextValue(WorkSegmentationStrategy segmentationStrategy, TaskWorkStateType workState,
String expectedNextValue, int expectedSequentialNumber) throws SchemaException {
WorkBucketType newBucket = getNextBucket(segmentationStrategy, workState, expectedSequentialNumber);
AbstractWorkBucketContentType content = newBucket.getContent();
assertEquals("Wrong content class", StringValueWorkBucketContentType.class, content.getClass());
StringValueWorkBucketContentType prefixContent = (StringValueWorkBucketContentType) content;
assertEquals("Wrong # of values generated", 1, prefixContent.getValue().size());
assertEquals("Wrong next value", expectedNextValue, prefixContent.getValue().get(0));

workState.getBucket().add(newBucket.clone().state(WorkBucketStateType.COMPLETE));
return newBucket;
}

private WorkBucketType assumeNextPrefix(WorkSegmentationStrategy segmentationStrategy, TaskWorkStateType workState,
String expectedNextPrefix, int expectedSequentialNumber) throws SchemaException {
WorkBucketType newBucket = getNextBucket(segmentationStrategy, workState, expectedSequentialNumber);
Expand Down

0 comments on commit 80f5979

Please sign in to comment.